diff --git a/dedal/docs/resources/dedal_UML_facade.png b/dedal/docs/resources/dedal_UML_facade.png new file mode 100644 index 0000000000000000000000000000000000000000..d16e40ba7b2cee0c2ad78cae626d28bcdc704010 Binary files /dev/null and b/dedal/docs/resources/dedal_UML_facade.png differ diff --git a/dedal/error_handling/exceptions.py b/dedal/error_handling/exceptions.py index 76844ac6fbc5e071f444da881f4fdb3caeee03ef..d0a383dd26d1f57e7d17596e18e29250230389b6 100644 --- a/dedal/error_handling/exceptions.py +++ b/dedal/error_handling/exceptions.py @@ -87,4 +87,9 @@ class SpackConfigException(BashCommandException): class SpackFindException(BashCommandException): """ To be thrown when the spack find command fails + """ + +class MissingAttributeException(BashCommandException): + """ + To be thrown when a missing attribute for a class is missing """ \ No newline at end of file diff --git a/dedal/spack_factory/SpackCacheOperation.py b/dedal/spack_factory/SpackCacheOperation.py new file mode 100644 index 0000000000000000000000000000000000000000..91a69c1e7fba63584c4ad175079b7ba5a70c2bf3 --- /dev/null +++ b/dedal/spack_factory/SpackCacheOperation.py @@ -0,0 +1,174 @@ +# Dedal library - Wrapper over Spack for building multiple target +# environments: ESD, Virtual Boxes, HPC compatible kernels, etc. + +# (c) Copyright 2025 Dedal developers + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +from pathlib import Path + +from dedal.utils.utils import run_command, get_first_word + +from dedal.wrapper.spack_wrapper import check_spack_env + +from dedal.error_handling.exceptions import MissingAttributeException, SpackGpgException, SpackMirrorException, \ + SpackReindexException +from dedal.logger.logger_builder import get_logger + +from dedal.configuration.SpackConfig import SpackConfig + + +class SpackCacheOperation: + def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__), spack_setup_script=None, + spack_dir=Path('./').resolve(), spack_command_on_env=None): + self.spack_config = spack_config + self.logger = logger + if spack_setup_script and spack_command_on_env: + self.spack_setup_script = spack_setup_script + self.spack_command_on_env = spack_command_on_env + else: + raise MissingAttributeException(f'Missing attribute for class {__name__}') + self.spack_dir = spack_dir + + def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False): + """Adds a Spack mirror. + Adds a new mirror to the Spack configuration, either globally or to a specific environment. + Args: + mirror_name (str): The name of the mirror. + mirror_path (str): The path or URL of the mirror. + signed (bool): Whether to require signed packages from the mirror. + autopush (bool): Whether to enable autopush for the mirror. + global_mirror (bool): Whether to add the mirror globally (True) or to the current environment (False). + Raises: + ValueError: If mirror_name or mirror_path are empty. + NoSpackEnvironmentException: If global_mirror is False and no environment is defined. + """ + autopush = '--autopush' if autopush else '' + signed = '--signed' if signed else '' + spack_add_mirror = f'spack mirror add {autopush} {signed} {mirror_name} {mirror_path}' + if global_mirror: + run_command("bash", "-c", + f'{self.spack_setup_script} && {spack_add_mirror}', + check=True, + logger=self.logger, + info_msg=f'Added mirror {mirror_name}', + exception_msg=f'Failed to add mirror {mirror_name}', + exception=SpackMirrorException) + else: + check_spack_env( + run_command("bash", "-c", + f'{self.spack_command_on_env} && {spack_add_mirror}', + check=True, + logger=self.logger, + info_msg=f'Added mirror {mirror_name}', + exception_msg=f'Failed to add mirror {mirror_name}', + exception=SpackMirrorException)) + + def trust_gpg_key(self, public_key_path: str): + """Adds a GPG public key to the trusted keyring. + This method attempts to add the provided GPG public key to the + Spack trusted keyring. + Args: + public_key_path (str): Path to the GPG public key file. + Returns: + bool: True if the key was added successfully, False otherwise. + Raises: + ValueError: If public_key_path is empty. + NoSpackEnvironmentException: If the spack environment is not set up. + """ + if not public_key_path: + raise ValueError("public_key_path is required") + + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack gpg trust {public_key_path}', + check=True, + logger=self.logger, + info_msg=f'Trusted GPG key for {self.spack_config.env.name}', + exception_msg=f'Failed to trust GPG key for {self.spack_config.env.name}', + exception=SpackGpgException) + + def mirror_list(self): + """Returns of available mirrors. When an environment is activated it will return the mirrors associated with it, + otherwise the mirrors set globally""" + mirrors = run_command("bash", "-c", + f'{self.spack_command_on_env} && spack mirror list', + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + logger=self.logger, + info_msg=f'Listing mirrors', + exception_msg=f'Failed list mirrors', + exception=SpackMirrorException).stdout + return list(map(get_first_word, list(mirrors.strip().splitlines()))) + + def remove_mirror(self, mirror_name: str): + """Removes a mirror from an environment (if it is activated), otherwise removes the mirror globally.""" + if not mirror_name: + raise ValueError("mirror_name is required") + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack mirror rm {mirror_name}', + check=True, + logger=self.logger, + info_msg=f'Removing mirror {mirror_name}', + exception_msg=f'Failed to remove mirror {mirror_name}', + exception=SpackMirrorException) + + def update_buildcache_index(self, mirror_path: str): + """Updates buildcache index""" + if not mirror_path: + raise ValueError("mirror_path is required") + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack buildcache update-index {mirror_path}', + check=True, + logger=self.logger, + info_msg=f'Updating build cache index for mirror {mirror_path}', + exception_msg=f'Failed to update build cache index for mirror {mirror_path}', + exception=SpackMirrorException) + + def install_gpg_keys(self): + """Install gpg keys""" + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack buildcache keys --install --trust', + check=True, + logger=self.logger, + info_msg=f'Installing gpg keys from mirror', + exception_msg=f'Failed to install gpg keys from mirror', + exception=SpackGpgException) + + def create_gpg_keys(self): + """Creates GPG keys (which can be used when creating binary cashes) and adds it to the trusted keyring.""" + if self.spack_config.gpg: + run_command("bash", "-c", + f'{self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}', + check=True, + logger=self.logger, + info_msg=f'Created pgp keys for {self.spack_config.env.name}', + exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.name}', + exception=SpackGpgException) + else: + raise SpackGpgException('No GPG configuration was defined is spack configuration') + + def reindex(self): + """Reindex step for a spack environment + Raises: + SpackReindexException: If the spack reindex command fails. + """ + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack reindex', + check=True, + logger=self.logger, + info_msg=f'Reindex step.', + exception_msg=f'Failed the reindex.', + exception=SpackReindexException) diff --git a/dedal/spack_factory/SpackEnvOperation.py b/dedal/spack_factory/SpackEnvOperation.py new file mode 100644 index 0000000000000000000000000000000000000000..01955aff1a901b9b35cb191a1c083f0a7f7a3c8d --- /dev/null +++ b/dedal/spack_factory/SpackEnvOperation.py @@ -0,0 +1,189 @@ +# Dedal library - Wrapper over Spack for building multiple target +# environments: ESD, Virtual Boxes, HPC compatible kernels, etc. + +# (c) Copyright 2025 Dedal developers + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import os +import subprocess +from pathlib import Path +from dedal.enum.SpackConfigCommand import SpackConfigCommand +from dedal.wrapper.spack_wrapper import check_spack_env +from dedal.error_handling.exceptions import BashCommandException, MissingAttributeException, \ + NoSpackEnvironmentException, SpackRepoException, SpackConfigException, SpackFindException, SpackSpecException +from dedal.utils.utils import git_clone_repo, run_command +from dedal.logger.logger_builder import get_logger +from dedal.configuration.SpackConfig import SpackConfig + + +class SpackEnvOperation: + def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__), spack_setup_script=None, + env_path=None, spack_command_on_env=None): + self.spack_config = spack_config + self.logger = logger + self.env_path = env_path + if spack_setup_script and spack_command_on_env: + self.spack_setup_script = spack_setup_script + self.spack_command_on_env = spack_command_on_env + else: + raise MissingAttributeException(f'Missing attribute for class {__name__}') + + def create_fetch_spack_environment(self): + """Fetches a spack environment if the git path is defined, otherwise creates it.""" + if self.spack_config.env and self.spack_config.env.git_path: + git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name, + self.spack_config.env.git_path, + logger=self.logger) + else: + os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True) + if self.env_path: + run_command("bash", "-c", + f'{self.spack_setup_script} && spack env create -d {self.env_path}', + check=True, logger=self.logger, + info_msg=f"Created {self.spack_config.env.name} spack environment", + exception_msg=f"Failed to create {self.spack_config.env.name} spack environment", + exception=BashCommandException) + else: + raise MissingAttributeException(f'Missing env_path attribute class {__name__}') + + def spack_repo_exists(self, repo_name: str) -> bool | None: + """Check if the given Spack repository exists. + Returns: + True if spack repository exists, False otherwise. + """ + if self.spack_config.env is None: + result = run_command("bash", "-c", + f'{self.spack_setup_script} && spack repo list', + check=True, + capture_output=True, text=True, logger=self.logger, + info_msg=f'Checking if {repo_name} exists') + if result is None: + return False + else: + if self.spack_env_exists(): + result = run_command("bash", "-c", + f'{self.spack_command_on_env} && spack repo list', + check=True, + capture_output=True, text=True, logger=self.logger, + info_msg=f'Checking if repository {repo_name} was added').stdout + else: + self.logger.debug('No spack environment defined') + raise NoSpackEnvironmentException('No spack environment defined') + if result is None: + return False + return any(line.strip().endswith(repo_name) for line in result.splitlines()) + + def spack_env_exists(self): + """Checks if a spack environments exists. + Returns: + True if spack environments exists, False otherwise. + """ + result = run_command("bash", "-c", + self.spack_command_on_env, + check=True, + capture_output=True, text=True, logger=self.logger, + info_msg=f'Checking if environment {self.spack_config.env.name} exists') + return result is not None + + def add_spack_repo(self, repo_path: Path, repo_name: str): + """Add the Spack repository if it does not exist.""" + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}', + check=True, logger=self.logger, + info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.name}", + exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.name}", + exception=SpackRepoException) + + def get_compiler_version(self): + """Returns the compiler version + Raises: + NoSpackEnvironmentException: If the spack environment is not set up. + """ + result = run_command("bash", "-c", + f'{self.spack_command_on_env} && spack compiler list', + check=True, logger=self.logger, + capture_output=True, text=True, + info_msg=f"Checking spack environment compiler version for {self.spack_config.env.name}", + exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.name}", + exception=BashCommandException) + + if result.stdout is None: + self.logger.debug(f'No gcc found for {self.spack_config.env.name}') + return None + + # Find the first occurrence of a GCC compiler using regex + match = re.search(r"gcc@([\d\.]+)", result.stdout) + gcc_version = match.group(1) + self.logger.debug(f'Found gcc for {self.spack_config.env.name}: {gcc_version}') + return gcc_version + + def config(self, config_type: SpackConfigCommand, config_parameter): + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack config {config_type.value} \"{config_parameter}\"', + check=True, + logger=self.logger, + info_msg='Spack config command', + exception_msg='Spack config command failed', + exception=SpackConfigException) + + def find_packages(self): + """Returns a dictionary of installed Spack packages in the current environment. + Each key is the name of a Spack package, and the corresponding value is a list of + installed versions for that package. + Raises: + NoSpackEnvironmentException: If the spack environment is not set up. + """ + packages = run_command("bash", "-c", + f'{self.spack_command_on_env} && spack find -c', + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + logger=self.logger, + info_msg=f'Listing installed packages.', + exception_msg=f'Failed to list installed packages', + exception=SpackFindException).stdout + dict_packages = {} + for package in packages.strip().splitlines(): + if package.startswith('[+]'): + package = package.replace('@', ' ').split() + if len(package) == 3: + _, name, version = package + dict_packages.setdefault(name, []).append(version) + return dict_packages + + def spec_pacakge(self, package_name: str): + """Reindex step for a spack environment + Raises: + SpackSpecException: If the spack spec command fails. + """ + try: + spec_output = run_command("bash", "-c", + f'{self.spack_command_on_env} && spack spec {package_name}', + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + logger=self.logger, + info_msg=f'Spack spec {package_name}.', + exception_msg=f'Failed to spack spec {package_name}.', + exception=SpackSpecException).stdout + pattern = r'^\s*-\s*([\w.-]+@[\d.]+)' + match = re.search(pattern, spec_output) + if match: + return match.group(1) + return None + except SpackSpecException: + return None diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py index 52d1a200089ded717c38a3f3a3f9963f1b6646c2..3b3645bb44a1088311f92b39b304f6e72b816158 100644 --- a/dedal/spack_factory/SpackOperation.py +++ b/dedal/spack_factory/SpackOperation.py @@ -16,19 +16,19 @@ # limitations under the License. import os -import re import subprocess from pathlib import Path from dedal.configuration.SpackConfig import SpackConfig from dedal.enum.SpackConfigCommand import SpackConfigCommand -from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \ - SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException, \ - SpackRepoException, SpackReindexException, SpackSpecException, SpackConfigException, SpackFindException +from dedal.error_handling.exceptions import SpackInstallPackagesException, SpackConcertizeException, SpackGpgException, \ + SpackReindexException from dedal.logger.logger_builder import get_logger +from dedal.spack_factory.SpackCacheOperation import SpackCacheOperation +from dedal.spack_factory.SpackEnvOperation import SpackEnvOperation +from dedal.spack_factory.SpackToolOperation import SpackToolOperation from dedal.tests.testing_variables import SPACK_VERSION -from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable, get_first_word +from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable from dedal.wrapper.spack_wrapper import check_spack_env -import glob class SpackOperation: @@ -47,12 +47,12 @@ class SpackOperation: def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)): self.spack_config = spack_config + self.logger = logger self.spack_config.install_dir = spack_config.install_dir os.makedirs(self.spack_config.install_dir, exist_ok=True) self.spack_dir = self.spack_config.install_dir / 'spack' - + self.env_path = None self.spack_setup_script = "" if self.spack_config.use_spack_global else f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}" - self.logger = logger self.spack_config.concretization_dir = spack_config.concretization_dir if self.spack_config.concretization_dir: os.makedirs(self.spack_config.concretization_dir, exist_ok=True) @@ -69,21 +69,17 @@ class SpackOperation: self.spack_command_on_env = self.spack_setup_script if self.spack_config.env and spack_config.env.path: self.spack_config.env.path.mkdir(parents=True, exist_ok=True) - - def create_fetch_spack_environment(self): - """Fetches a spack environment if the git path is defined, otherwise creates it.""" - if self.spack_config.env and self.spack_config.env.git_path: - git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name, - self.spack_config.env.git_path, - logger=self.logger) - else: - os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True) - run_command("bash", "-c", - f'{self.spack_setup_script} && spack env create -d {self.env_path}', - check=True, logger=self.logger, - info_msg=f"Created {self.spack_config.env.name} spack environment", - exception_msg=f"Failed to create {self.spack_config.env.name} spack environment", - exception=BashCommandException) + self.spack_tool_operation = SpackToolOperation(spack_config=spack_config, + spack_setup_script=self.spack_setup_script, + spack_dir=self.spack_dir) + self.spack_env_operation = SpackEnvOperation(spack_config=spack_config, + spack_setup_script=self.spack_setup_script, + env_path=self.env_path, + spack_command_on_env=self.spack_command_on_env) + self.spack_cache_operation = SpackCacheOperation(spack_config=spack_config, + spack_setup_script=self.spack_setup_script, + spack_command_on_env=self.spack_command_on_env, + spack_dir=self.spack_dir) def setup_spack_env(self): """ @@ -103,7 +99,7 @@ class SpackOperation: self.logger.error(f'Invalid installation path: {self.spack_dir}') # Restart the bash after adding environment variables if self.spack_config.env: - self.create_fetch_spack_environment() + self.spack_env_operation.create_fetch_spack_environment() if self.spack_config.install_dir.exists(): for repo in self.spack_config.repos: repo_dir = self.spack_config.install_dir / repo.path / repo.name @@ -114,89 +110,6 @@ class SpackOperation: else: self.logger.debug(f'Spack repository {repo.name} already added') - def spack_repo_exists(self, repo_name: str) -> bool | None: - """Check if the given Spack repository exists. - Returns: - True if spack repository exists, False otherwise. - """ - if self.spack_config.env is None: - result = run_command("bash", "-c", - f'{self.spack_setup_script} && spack repo list', - check=True, - capture_output=True, text=True, logger=self.logger, - info_msg=f'Checking if {repo_name} exists') - if result is None: - return False - else: - if self.spack_env_exists(): - result = run_command("bash", "-c", - f'{self.spack_command_on_env} && spack repo list', - check=True, - capture_output=True, text=True, logger=self.logger, - info_msg=f'Checking if repository {repo_name} was added').stdout - else: - self.logger.debug('No spack environment defined') - raise NoSpackEnvironmentException('No spack environment defined') - if result is None: - return False - return any(line.strip().endswith(repo_name) for line in result.splitlines()) - - def spack_env_exists(self): - """Checks if a spack environments exists. - Returns: - True if spack environments exists, False otherwise. - """ - result = run_command("bash", "-c", - self.spack_command_on_env, - check=True, - capture_output=True, text=True, logger=self.logger, - info_msg=f'Checking if environment {self.spack_config.env.name} exists') - return result is not None - - def add_spack_repo(self, repo_path: Path, repo_name: str): - """Add the Spack repository if it does not exist.""" - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}', - check=True, logger=self.logger, - info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.name}", - exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.name}", - exception=SpackRepoException) - - @check_spack_env - def get_compiler_version(self): - """Returns the compiler version - Raises: - NoSpackEnvironmentException: If the spack environment is not set up. - """ - result = run_command("bash", "-c", - f'{self.spack_command_on_env} && spack compiler list', - check=True, logger=self.logger, - capture_output=True, text=True, - info_msg=f"Checking spack environment compiler version for {self.spack_config.env.name}", - exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.name}", - exception=BashCommandException) - - if result.stdout is None: - self.logger.debug(f'No gcc found for {self.spack_config.env.name}') - return None - - # Find the first occurrence of a GCC compiler using regex - match = re.search(r"gcc@([\d\.]+)", result.stdout) - gcc_version = match.group(1) - self.logger.debug(f'Found gcc for {self.spack_config.env.name}: {gcc_version}') - return gcc_version - - def get_spack_installed_version(self): - """Returns the spack installed version""" - spack_version = run_command("bash", "-c", f'{self.spack_setup_script} && spack --version', - capture_output=True, text=True, check=True, - logger=self.logger, - info_msg=f"Getting spack version", - exception_msg=f"Error retrieving Spack version") - if spack_version: - return spack_version.stdout.strip().split()[0] - return None - @check_spack_env def concretize_spack_env(self, force=True, test=None): """Concretization step for a spack environment @@ -216,172 +129,6 @@ class SpackOperation: exception_msg=f'Failed the concertization step for {self.spack_config.env.name}', exception=SpackConcertizeException) - def reindex(self): - """Reindex step for a spack environment - Raises: - SpackReindexException: If the spack reindex command fails. - """ - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack reindex', - check=True, - logger=self.logger, - info_msg=f'Reindex step.', - exception_msg=f'Failed the reindex.', - exception=SpackReindexException) - - def spec_pacakge(self, package_name: str): - """Reindex step for a spack environment - Raises: - SpackSpecException: If the spack spec command fails. - """ - try: - spec_output = run_command("bash", "-c", - f'{self.spack_command_on_env} && spack spec {package_name}', - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - logger=self.logger, - info_msg=f'Spack spec {package_name}.', - exception_msg=f'Failed to spack spec {package_name}.', - exception=SpackSpecException).stdout - pattern = r'^\s*-\s*([\w.-]+@[\d.]+)' - match = re.search(pattern, spec_output) - if match: - return match.group(1) - return None - except SpackSpecException: - return None - - def create_gpg_keys(self): - """Creates GPG keys (which can be used when creating binary cashes) and adds it to the trusted keyring.""" - if self.spack_config.gpg: - run_command("bash", "-c", - f'{self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}', - check=True, - logger=self.logger, - info_msg=f'Created pgp keys for {self.spack_config.env.name}', - exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.name}', - exception=SpackGpgException) - else: - raise SpackGpgException('No GPG configuration was defined is spack configuration') - - def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False): - """Adds a Spack mirror. - Adds a new mirror to the Spack configuration, either globally or to a specific environment. - Args: - mirror_name (str): The name of the mirror. - mirror_path (str): The path or URL of the mirror. - signed (bool): Whether to require signed packages from the mirror. - autopush (bool): Whether to enable autopush for the mirror. - global_mirror (bool): Whether to add the mirror globally (True) or to the current environment (False). - Raises: - ValueError: If mirror_name or mirror_path are empty. - NoSpackEnvironmentException: If global_mirror is False and no environment is defined. - """ - autopush = '--autopush' if autopush else '' - signed = '--signed' if signed else '' - spack_add_mirror = f'spack mirror add {autopush} {signed} {mirror_name} {mirror_path}' - if global_mirror: - run_command("bash", "-c", - f'{self.spack_setup_script} && {spack_add_mirror}', - check=True, - logger=self.logger, - info_msg=f'Added mirror {mirror_name}', - exception_msg=f'Failed to add mirror {mirror_name}', - exception=SpackMirrorException) - else: - check_spack_env( - run_command("bash", "-c", - f'{self.spack_command_on_env} && {spack_add_mirror}', - check=True, - logger=self.logger, - info_msg=f'Added mirror {mirror_name}', - exception_msg=f'Failed to add mirror {mirror_name}', - exception=SpackMirrorException)) - - @check_spack_env - def trust_gpg_key(self, public_key_path: str): - """Adds a GPG public key to the trusted keyring. - This method attempts to add the provided GPG public key to the - Spack trusted keyring. - Args: - public_key_path (str): Path to the GPG public key file. - Returns: - bool: True if the key was added successfully, False otherwise. - Raises: - ValueError: If public_key_path is empty. - NoSpackEnvironmentException: If the spack environment is not set up. - """ - if not public_key_path: - raise ValueError("public_key_path is required") - - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack gpg trust {public_key_path}', - check=True, - logger=self.logger, - info_msg=f'Trusted GPG key for {self.spack_config.env.name}', - exception_msg=f'Failed to trust GPG key for {self.spack_config.env.name}', - exception=SpackGpgException) - - def config(self, config_type: SpackConfigCommand, config_parameter): - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack config {config_type.value} \"{config_parameter}\"', - check=True, - logger=self.logger, - info_msg='Spack config command', - exception_msg='Spack config command failed', - exception=SpackConfigException) - - def mirror_list(self): - """Returns of available mirrors. When an environment is activated it will return the mirrors associated with it, - otherwise the mirrors set globally""" - mirrors = run_command("bash", "-c", - f'{self.spack_command_on_env} && spack mirror list', - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - logger=self.logger, - info_msg=f'Listing mirrors', - exception_msg=f'Failed list mirrors', - exception=SpackMirrorException).stdout - return list(map(get_first_word, list(mirrors.strip().splitlines()))) - - def remove_mirror(self, mirror_name: str): - """Removes a mirror from an environment (if it is activated), otherwise removes the mirror globally.""" - if not mirror_name: - raise ValueError("mirror_name is required") - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack mirror rm {mirror_name}', - check=True, - logger=self.logger, - info_msg=f'Removing mirror {mirror_name}', - exception_msg=f'Failed to remove mirror {mirror_name}', - exception=SpackMirrorException) - - def update_buildcache_index(self, mirror_path: str): - """Updates buildcache index""" - if not mirror_path: - raise ValueError("mirror_path is required") - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack buildcache update-index {mirror_path}', - check=True, - logger=self.logger, - info_msg=f'Updating build cache index for mirror {mirror_path}', - exception_msg=f'Failed to update build cache index for mirror {mirror_path}', - exception=SpackMirrorException) - - def install_gpg_keys(self): - """Install gpg keys""" - run_command("bash", "-c", - f'{self.spack_command_on_env} && spack buildcache keys --install --trust', - check=True, - logger=self.logger, - info_msg=f'Installing gpg keys from mirror', - exception_msg=f'Failed to install gpg keys from mirror', - exception=SpackGpgException) - @check_spack_env def install_packages(self, jobs: int, signed=True, fresh=False, debug=False, test=None): """Installs all spack packages. @@ -408,83 +155,60 @@ class SpackOperation: self.logger.error(f'Something went wrong during installation. Please check the logs.') return install_result + def create_fetch_spack_environment(self): + self.spack_env_operation.create_fetch_spack_environment() + + def add_spack_repo(self, repo_path: Path, repo_name: str): + self.spack_env_operation.add_spack_repo(repo_path, repo_name) + + def spack_repo_exists(self, repo_name: str) -> bool | None: + return self.spack_env_operation.spack_repo_exists(repo_name) + + def spack_env_exists(self): + return self.spack_env_operation.spack_env_exists() + + @check_spack_env + def get_compiler_version(self): + return self.spack_env_operation.get_compiler_version() + + def get_spack_installed_version(self): + return self.spack_tool_operation.get_spack_installed_version() + + def reindex(self): + self.spack_cache_operation.reindex() + + def spec_pacakge(self, package_name: str): + return self.spack_env_operation.spec_pacakge(package_name) + + def create_gpg_keys(self): + self.spack_cache_operation.create_gpg_keys() + + def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False): + return self.spack_cache_operation.add_mirror(mirror_name, mirror_path, signed, autopush, global_mirror) + + @check_spack_env + def trust_gpg_key(self, public_key_path: str): + self.spack_cache_operation.trust_gpg_key(public_key_path) + + def config(self, config_type: SpackConfigCommand, config_parameter): + self.spack_env_operation.config(config_type, config_parameter) + + def mirror_list(self): + return self.spack_cache_operation.mirror_list() + + def remove_mirror(self, mirror_name: str): + self.spack_cache_operation.remove_mirror(mirror_name) + + def update_buildcache_index(self, mirror_path: str): + self.spack_cache_operation.update_buildcache_index(mirror_path) + + def install_gpg_keys(self): + self.spack_cache_operation.install_gpg_keys() + @check_spack_env def find_packages(self): - """Returns a dictionary of installed Spack packages in the current environment. - Each key is the name of a Spack package, and the corresponding value is a list of - installed versions for that package. - Raises: - NoSpackEnvironmentException: If the spack environment is not set up. - """ - packages = run_command("bash", "-c", - f'{self.spack_command_on_env} && spack find -c', - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - logger=self.logger, - info_msg=f'Listing installed packages.', - exception_msg=f'Failed to list installed packages', - exception=SpackFindException).stdout - dict_packages = {} - for package in packages.strip().splitlines(): - if package.startswith('[+]'): - package = package.replace('@', ' ').split() - if len(package) == 3: - _, name, version = package - dict_packages.setdefault(name, []).append(version) - return dict_packages + return self.spack_env_operation.find_packages() def install_spack(self, spack_version=f'{SPACK_VERSION}', spack_repo='https://github.com/spack/spack', bashrc_path=os.path.expanduser("~/.bashrc")): - """Install spack. - Args: - spack_version (str): spack version - spack_repo (str): Git path to the Spack repository. - bashrc_path (str): Path to the .bashrc file. - """ - spack_version = f'v{spack_version}' - try: - user = os.getlogin() - except OSError: - user = None - - self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}") - if not self.spack_dir.exists(): - run_command( - "git", "clone", "--depth", "1", - "-c", "advice.detachedHead=false", - "-c", "feature.manyFiles=true", - "--branch", spack_version, spack_repo, self.spack_dir - , check=True, logger=self.logger) - self.logger.debug("Cloned spack") - else: - self.logger.debug("Spack already cloned.") - - if bashrc_path: - # ensure the file exists before opening it - if not os.path.exists(bashrc_path): - open(bashrc_path, "w").close() - # add spack setup commands to .bashrc - with open(bashrc_path, "a") as bashrc: - bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n') - spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}" - bashrc.write(f"{spack_setup_script}\n") - self.logger.info("Added Spack PATH to .bashrc") - if user: - run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger, - info_msg='Adding permissions to the logged in user') - self.logger.info("Spack install completed") - if self.spack_config.use_spack_global is True and bashrc_path is not None: - # Restart the bash only of the spack is used globally - self.logger.info('Restarting bash') - run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash') - os.system("exec bash") - # Configure upstream Spack instance if specified - if self.spack_config.upstream_instance: - search_path = os.path.join(self.spack_config.upstream_instance, 'spack', 'opt', 'spack', '**', '.spack-db') - spack_db_dirs = glob.glob(search_path, recursive=True) - upstream_prefix = [os.path.dirname(dir) for dir in spack_db_dirs] - for prefix in upstream_prefix: - self.config(SpackConfigCommand.ADD, f':upstream-spack-instance:install_tree:{prefix}') - self.logger.info("Added upstream spack instance") + self.spack_tool_operation.install_spack(spack_version, spack_repo, bashrc_path) diff --git a/dedal/spack_factory/SpackToolOperation.py b/dedal/spack_factory/SpackToolOperation.py new file mode 100644 index 0000000000000000000000000000000000000000..996824800d3da06d588a064a478e83058b47e748 --- /dev/null +++ b/dedal/spack_factory/SpackToolOperation.py @@ -0,0 +1,107 @@ +# Dedal library - Wrapper over Spack for building multiple target +# environments: ESD, Virtual Boxes, HPC compatible kernels, etc. + +# (c) Copyright 2025 Dedal developers + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +from pathlib import Path +from dedal.enum.SpackConfigCommand import SpackConfigCommand +from dedal.error_handling.exceptions import MissingAttributeException +from dedal.spack_factory.SpackEnvOperation import SpackEnvOperation +from dedal.tests.testing_variables import SPACK_VERSION +from dedal.utils.utils import run_command +from dedal.logger.logger_builder import get_logger +from dedal.configuration.SpackConfig import SpackConfig + + +class SpackToolOperation: + def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__), spack_setup_script=None, + spack_dir=Path('./').resolve()): + self.spack_config = spack_config + self.logger = logger + if spack_setup_script: + self.spack_setup_script = spack_setup_script + else: + raise MissingAttributeException(f'Missing attribute for class {__name__}') + self.spack_dir = spack_dir + + def get_spack_installed_version(self): + """Returns the spack installed version""" + spack_version = run_command("bash", "-c", f'{self.spack_setup_script} && spack --version', + capture_output=True, text=True, check=True, + logger=self.logger, + info_msg=f"Getting spack version", + exception_msg=f"Error retrieving Spack version") + if spack_version: + return spack_version.stdout.strip().split()[0] + return None + + def install_spack(self, spack_version=f'{SPACK_VERSION}', spack_repo='https://github.com/spack/spack', + bashrc_path=os.path.expanduser("~/.bashrc")): + """Install spack. + Args: + spack_version (str): spack version + spack_repo (str): Git path to the Spack repository. + bashrc_path (str): Path to the .bashrc file. + """ + spack_version = f'v{spack_version}' + try: + user = os.getlogin() + except OSError: + user = None + + self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}") + if not self.spack_dir.exists(): + run_command( + "git", "clone", "--depth", "1", + "-c", "advice.detachedHead=false", + "-c", "feature.manyFiles=true", + "--branch", spack_version, spack_repo, self.spack_dir + , check=True, logger=self.logger) + self.logger.debug("Cloned spack") + else: + self.logger.debug("Spack already cloned.") + + if bashrc_path: + # ensure the file exists before opening it + if not os.path.exists(bashrc_path): + open(bashrc_path, "w").close() + # add spack setup commands to .bashrc + with open(bashrc_path, "a") as bashrc: + bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n') + spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}" + bashrc.write(f"{spack_setup_script}\n") + self.logger.info("Added Spack PATH to .bashrc") + if user: + run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger, + info_msg='Adding permissions to the logged in user') + self.logger.info("Spack install completed") + if self.spack_config.use_spack_global is True and bashrc_path is not None: + # Restart the bash only of the spack is used globally + self.logger.info('Restarting bash') + run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash') + os.system("exec bash") + # Configure upstream Spack instance if specified + if self.spack_config.upstream_instance: + search_path = os.path.join(self.spack_config.upstream_instance, 'spack', 'opt', 'spack', '**', '.spack-db') + spack_db_dirs = glob.glob(search_path, recursive=True) + upstream_prefix = [os.path.dirname(dir) for dir in spack_db_dirs] + spack_env_operation = SpackEnvOperation(spack_config=self.spack_config, + spack_setup_script=self.spack_setup_script) + for prefix in upstream_prefix: + # todo fix + spack_env_operation.config(SpackConfigCommand.ADD, f':upstream-spack-instance:install_tree:{prefix}') + self.logger.info("Added upstream spack instance")