diff --git a/dedal/docs/resources/dedal_UML_facade.png b/dedal/docs/resources/dedal_UML_facade.png
new file mode 100644
index 0000000000000000000000000000000000000000..d16e40ba7b2cee0c2ad78cae626d28bcdc704010
Binary files /dev/null and b/dedal/docs/resources/dedal_UML_facade.png differ
diff --git a/dedal/error_handling/exceptions.py b/dedal/error_handling/exceptions.py
index 76844ac6fbc5e071f444da881f4fdb3caeee03ef..d0a383dd26d1f57e7d17596e18e29250230389b6 100644
--- a/dedal/error_handling/exceptions.py
+++ b/dedal/error_handling/exceptions.py
@@ -87,4 +87,9 @@ class SpackConfigException(BashCommandException):
class SpackFindException(BashCommandException):
"""
To be thrown when the spack find command fails
+ """
+
+class MissingAttributeException(BashCommandException):
+ """
+ To be thrown when a missing attribute for a class is missing
"""
\ No newline at end of file
diff --git a/dedal/spack_factory/SpackCacheOperation.py b/dedal/spack_factory/SpackCacheOperation.py
new file mode 100644
index 0000000000000000000000000000000000000000..91a69c1e7fba63584c4ad175079b7ba5a70c2bf3
--- /dev/null
+++ b/dedal/spack_factory/SpackCacheOperation.py
@@ -0,0 +1,174 @@
+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+# (c) Copyright 2025 Dedal developers
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+from pathlib import Path
+
+from dedal.utils.utils import run_command, get_first_word
+
+from dedal.wrapper.spack_wrapper import check_spack_env
+
+from dedal.error_handling.exceptions import MissingAttributeException, SpackGpgException, SpackMirrorException, \
+ SpackReindexException
+from dedal.logger.logger_builder import get_logger
+
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackCacheOperation:
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__), spack_setup_script=None,
+ spack_dir=Path('./').resolve(), spack_command_on_env=None):
+ self.spack_config = spack_config
+ self.logger = logger
+ if spack_setup_script and spack_command_on_env:
+ self.spack_setup_script = spack_setup_script
+ self.spack_command_on_env = spack_command_on_env
+ else:
+ raise MissingAttributeException(f'Missing attribute for class {__name__}')
+ self.spack_dir = spack_dir
+
+ def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
+ """Adds a Spack mirror.
+ Adds a new mirror to the Spack configuration, either globally or to a specific environment.
+ Args:
+ mirror_name (str): The name of the mirror.
+ mirror_path (str): The path or URL of the mirror.
+ signed (bool): Whether to require signed packages from the mirror.
+ autopush (bool): Whether to enable autopush for the mirror.
+ global_mirror (bool): Whether to add the mirror globally (True) or to the current environment (False).
+ Raises:
+ ValueError: If mirror_name or mirror_path are empty.
+ NoSpackEnvironmentException: If global_mirror is False and no environment is defined.
+ """
+ autopush = '--autopush' if autopush else ''
+ signed = '--signed' if signed else ''
+ spack_add_mirror = f'spack mirror add {autopush} {signed} {mirror_name} {mirror_path}'
+ if global_mirror:
+ run_command("bash", "-c",
+ f'{self.spack_setup_script} && {spack_add_mirror}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException)
+ else:
+ check_spack_env(
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && {spack_add_mirror}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException))
+
+ def trust_gpg_key(self, public_key_path: str):
+ """Adds a GPG public key to the trusted keyring.
+ This method attempts to add the provided GPG public key to the
+ Spack trusted keyring.
+ Args:
+ public_key_path (str): Path to the GPG public key file.
+ Returns:
+ bool: True if the key was added successfully, False otherwise.
+ Raises:
+ ValueError: If public_key_path is empty.
+ NoSpackEnvironmentException: If the spack environment is not set up.
+ """
+ if not public_key_path:
+ raise ValueError("public_key_path is required")
+
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack gpg trust {public_key_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Trusted GPG key for {self.spack_config.env.name}',
+ exception_msg=f'Failed to trust GPG key for {self.spack_config.env.name}',
+ exception=SpackGpgException)
+
+ def mirror_list(self):
+ """Returns of available mirrors. When an environment is activated it will return the mirrors associated with it,
+ otherwise the mirrors set globally"""
+ mirrors = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack mirror list',
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f'Listing mirrors',
+ exception_msg=f'Failed list mirrors',
+ exception=SpackMirrorException).stdout
+ return list(map(get_first_word, list(mirrors.strip().splitlines())))
+
+ def remove_mirror(self, mirror_name: str):
+ """Removes a mirror from an environment (if it is activated), otherwise removes the mirror globally."""
+ if not mirror_name:
+ raise ValueError("mirror_name is required")
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack mirror rm {mirror_name}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Removing mirror {mirror_name}',
+ exception_msg=f'Failed to remove mirror {mirror_name}',
+ exception=SpackMirrorException)
+
+ def update_buildcache_index(self, mirror_path: str):
+ """Updates buildcache index"""
+ if not mirror_path:
+ raise ValueError("mirror_path is required")
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack buildcache update-index {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Updating build cache index for mirror {mirror_path}',
+ exception_msg=f'Failed to update build cache index for mirror {mirror_path}',
+ exception=SpackMirrorException)
+
+ def install_gpg_keys(self):
+ """Install gpg keys"""
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack buildcache keys --install --trust',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Installing gpg keys from mirror',
+ exception_msg=f'Failed to install gpg keys from mirror',
+ exception=SpackGpgException)
+
+ def create_gpg_keys(self):
+ """Creates GPG keys (which can be used when creating binary cashes) and adds it to the trusted keyring."""
+ if self.spack_config.gpg:
+ run_command("bash", "-c",
+ f'{self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Created pgp keys for {self.spack_config.env.name}',
+ exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.name}',
+ exception=SpackGpgException)
+ else:
+ raise SpackGpgException('No GPG configuration was defined is spack configuration')
+
+ def reindex(self):
+ """Reindex step for a spack environment
+ Raises:
+ SpackReindexException: If the spack reindex command fails.
+ """
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack reindex',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Reindex step.',
+ exception_msg=f'Failed the reindex.',
+ exception=SpackReindexException)
diff --git a/dedal/spack_factory/SpackEnvOperation.py b/dedal/spack_factory/SpackEnvOperation.py
new file mode 100644
index 0000000000000000000000000000000000000000..01955aff1a901b9b35cb191a1c083f0a7f7a3c8d
--- /dev/null
+++ b/dedal/spack_factory/SpackEnvOperation.py
@@ -0,0 +1,189 @@
+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+# (c) Copyright 2025 Dedal developers
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import os
+import subprocess
+from pathlib import Path
+from dedal.enum.SpackConfigCommand import SpackConfigCommand
+from dedal.wrapper.spack_wrapper import check_spack_env
+from dedal.error_handling.exceptions import BashCommandException, MissingAttributeException, \
+ NoSpackEnvironmentException, SpackRepoException, SpackConfigException, SpackFindException, SpackSpecException
+from dedal.utils.utils import git_clone_repo, run_command
+from dedal.logger.logger_builder import get_logger
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackEnvOperation:
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__), spack_setup_script=None,
+ env_path=None, spack_command_on_env=None):
+ self.spack_config = spack_config
+ self.logger = logger
+ self.env_path = env_path
+ if spack_setup_script and spack_command_on_env:
+ self.spack_setup_script = spack_setup_script
+ self.spack_command_on_env = spack_command_on_env
+ else:
+ raise MissingAttributeException(f'Missing attribute for class {__name__}')
+
+ def create_fetch_spack_environment(self):
+ """Fetches a spack environment if the git path is defined, otherwise creates it."""
+ if self.spack_config.env and self.spack_config.env.git_path:
+ git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name,
+ self.spack_config.env.git_path,
+ logger=self.logger)
+ else:
+ os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True)
+ if self.env_path:
+ run_command("bash", "-c",
+ f'{self.spack_setup_script} && spack env create -d {self.env_path}',
+ check=True, logger=self.logger,
+ info_msg=f"Created {self.spack_config.env.name} spack environment",
+ exception_msg=f"Failed to create {self.spack_config.env.name} spack environment",
+ exception=BashCommandException)
+ else:
+ raise MissingAttributeException(f'Missing env_path attribute class {__name__}')
+
+ def spack_repo_exists(self, repo_name: str) -> bool | None:
+ """Check if the given Spack repository exists.
+ Returns:
+ True if spack repository exists, False otherwise.
+ """
+ if self.spack_config.env is None:
+ result = run_command("bash", "-c",
+ f'{self.spack_setup_script} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Checking if {repo_name} exists')
+ if result is None:
+ return False
+ else:
+ if self.spack_env_exists():
+ result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Checking if repository {repo_name} was added').stdout
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+ if result is None:
+ return False
+ return any(line.strip().endswith(repo_name) for line in result.splitlines())
+
+ def spack_env_exists(self):
+ """Checks if a spack environments exists.
+ Returns:
+ True if spack environments exists, False otherwise.
+ """
+ result = run_command("bash", "-c",
+ self.spack_command_on_env,
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Checking if environment {self.spack_config.env.name} exists')
+ return result is not None
+
+ def add_spack_repo(self, repo_path: Path, repo_name: str):
+ """Add the Spack repository if it does not exist."""
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.name}",
+ exception=SpackRepoException)
+
+ def get_compiler_version(self):
+ """Returns the compiler version
+ Raises:
+ NoSpackEnvironmentException: If the spack environment is not set up.
+ """
+ result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack compiler list',
+ check=True, logger=self.logger,
+ capture_output=True, text=True,
+ info_msg=f"Checking spack environment compiler version for {self.spack_config.env.name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.name}",
+ exception=BashCommandException)
+
+ if result.stdout is None:
+ self.logger.debug(f'No gcc found for {self.spack_config.env.name}')
+ return None
+
+ # Find the first occurrence of a GCC compiler using regex
+ match = re.search(r"gcc@([\d\.]+)", result.stdout)
+ gcc_version = match.group(1)
+ self.logger.debug(f'Found gcc for {self.spack_config.env.name}: {gcc_version}')
+ return gcc_version
+
+ def config(self, config_type: SpackConfigCommand, config_parameter):
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack config {config_type.value} \"{config_parameter}\"',
+ check=True,
+ logger=self.logger,
+ info_msg='Spack config command',
+ exception_msg='Spack config command failed',
+ exception=SpackConfigException)
+
+ def find_packages(self):
+ """Returns a dictionary of installed Spack packages in the current environment.
+ Each key is the name of a Spack package, and the corresponding value is a list of
+ installed versions for that package.
+ Raises:
+ NoSpackEnvironmentException: If the spack environment is not set up.
+ """
+ packages = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack find -c',
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f'Listing installed packages.',
+ exception_msg=f'Failed to list installed packages',
+ exception=SpackFindException).stdout
+ dict_packages = {}
+ for package in packages.strip().splitlines():
+ if package.startswith('[+]'):
+ package = package.replace('@', ' ').split()
+ if len(package) == 3:
+ _, name, version = package
+ dict_packages.setdefault(name, []).append(version)
+ return dict_packages
+
+ def spec_pacakge(self, package_name: str):
+ """Reindex step for a spack environment
+ Raises:
+ SpackSpecException: If the spack spec command fails.
+ """
+ try:
+ spec_output = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack spec {package_name}',
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f'Spack spec {package_name}.',
+ exception_msg=f'Failed to spack spec {package_name}.',
+ exception=SpackSpecException).stdout
+ pattern = r'^\s*-\s*([\w.-]+@[\d.]+)'
+ match = re.search(pattern, spec_output)
+ if match:
+ return match.group(1)
+ return None
+ except SpackSpecException:
+ return None
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
index 52d1a200089ded717c38a3f3a3f9963f1b6646c2..3b3645bb44a1088311f92b39b304f6e72b816158 100644
--- a/dedal/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -16,19 +16,19 @@
# limitations under the License.
import os
-import re
import subprocess
from pathlib import Path
from dedal.configuration.SpackConfig import SpackConfig
from dedal.enum.SpackConfigCommand import SpackConfigCommand
-from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException, \
- SpackRepoException, SpackReindexException, SpackSpecException, SpackConfigException, SpackFindException
+from dedal.error_handling.exceptions import SpackInstallPackagesException, SpackConcertizeException, SpackGpgException, \
+ SpackReindexException
from dedal.logger.logger_builder import get_logger
+from dedal.spack_factory.SpackCacheOperation import SpackCacheOperation
+from dedal.spack_factory.SpackEnvOperation import SpackEnvOperation
+from dedal.spack_factory.SpackToolOperation import SpackToolOperation
from dedal.tests.testing_variables import SPACK_VERSION
-from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable, get_first_word
+from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
from dedal.wrapper.spack_wrapper import check_spack_env
-import glob
class SpackOperation:
@@ -47,12 +47,12 @@ class SpackOperation:
def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
self.spack_config = spack_config
+ self.logger = logger
self.spack_config.install_dir = spack_config.install_dir
os.makedirs(self.spack_config.install_dir, exist_ok=True)
self.spack_dir = self.spack_config.install_dir / 'spack'
-
+ self.env_path = None
self.spack_setup_script = "" if self.spack_config.use_spack_global else f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
- self.logger = logger
self.spack_config.concretization_dir = spack_config.concretization_dir
if self.spack_config.concretization_dir:
os.makedirs(self.spack_config.concretization_dir, exist_ok=True)
@@ -69,21 +69,17 @@ class SpackOperation:
self.spack_command_on_env = self.spack_setup_script
if self.spack_config.env and spack_config.env.path:
self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
-
- def create_fetch_spack_environment(self):
- """Fetches a spack environment if the git path is defined, otherwise creates it."""
- if self.spack_config.env and self.spack_config.env.git_path:
- git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name,
- self.spack_config.env.git_path,
- logger=self.logger)
- else:
- os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True)
- run_command("bash", "-c",
- f'{self.spack_setup_script} && spack env create -d {self.env_path}',
- check=True, logger=self.logger,
- info_msg=f"Created {self.spack_config.env.name} spack environment",
- exception_msg=f"Failed to create {self.spack_config.env.name} spack environment",
- exception=BashCommandException)
+ self.spack_tool_operation = SpackToolOperation(spack_config=spack_config,
+ spack_setup_script=self.spack_setup_script,
+ spack_dir=self.spack_dir)
+ self.spack_env_operation = SpackEnvOperation(spack_config=spack_config,
+ spack_setup_script=self.spack_setup_script,
+ env_path=self.env_path,
+ spack_command_on_env=self.spack_command_on_env)
+ self.spack_cache_operation = SpackCacheOperation(spack_config=spack_config,
+ spack_setup_script=self.spack_setup_script,
+ spack_command_on_env=self.spack_command_on_env,
+ spack_dir=self.spack_dir)
def setup_spack_env(self):
"""
@@ -103,7 +99,7 @@ class SpackOperation:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
# Restart the bash after adding environment variables
if self.spack_config.env:
- self.create_fetch_spack_environment()
+ self.spack_env_operation.create_fetch_spack_environment()
if self.spack_config.install_dir.exists():
for repo in self.spack_config.repos:
repo_dir = self.spack_config.install_dir / repo.path / repo.name
@@ -114,89 +110,6 @@ class SpackOperation:
else:
self.logger.debug(f'Spack repository {repo.name} already added')
- def spack_repo_exists(self, repo_name: str) -> bool | None:
- """Check if the given Spack repository exists.
- Returns:
- True if spack repository exists, False otherwise.
- """
- if self.spack_config.env is None:
- result = run_command("bash", "-c",
- f'{self.spack_setup_script} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Checking if {repo_name} exists')
- if result is None:
- return False
- else:
- if self.spack_env_exists():
- result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Checking if repository {repo_name} was added').stdout
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
- if result is None:
- return False
- return any(line.strip().endswith(repo_name) for line in result.splitlines())
-
- def spack_env_exists(self):
- """Checks if a spack environments exists.
- Returns:
- True if spack environments exists, False otherwise.
- """
- result = run_command("bash", "-c",
- self.spack_command_on_env,
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Checking if environment {self.spack_config.env.name} exists')
- return result is not None
-
- def add_spack_repo(self, repo_path: Path, repo_name: str):
- """Add the Spack repository if it does not exist."""
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger,
- info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.name}",
- exception=SpackRepoException)
-
- @check_spack_env
- def get_compiler_version(self):
- """Returns the compiler version
- Raises:
- NoSpackEnvironmentException: If the spack environment is not set up.
- """
- result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack compiler list',
- check=True, logger=self.logger,
- capture_output=True, text=True,
- info_msg=f"Checking spack environment compiler version for {self.spack_config.env.name}",
- exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.name}",
- exception=BashCommandException)
-
- if result.stdout is None:
- self.logger.debug(f'No gcc found for {self.spack_config.env.name}')
- return None
-
- # Find the first occurrence of a GCC compiler using regex
- match = re.search(r"gcc@([\d\.]+)", result.stdout)
- gcc_version = match.group(1)
- self.logger.debug(f'Found gcc for {self.spack_config.env.name}: {gcc_version}')
- return gcc_version
-
- def get_spack_installed_version(self):
- """Returns the spack installed version"""
- spack_version = run_command("bash", "-c", f'{self.spack_setup_script} && spack --version',
- capture_output=True, text=True, check=True,
- logger=self.logger,
- info_msg=f"Getting spack version",
- exception_msg=f"Error retrieving Spack version")
- if spack_version:
- return spack_version.stdout.strip().split()[0]
- return None
-
@check_spack_env
def concretize_spack_env(self, force=True, test=None):
"""Concretization step for a spack environment
@@ -216,172 +129,6 @@ class SpackOperation:
exception_msg=f'Failed the concertization step for {self.spack_config.env.name}',
exception=SpackConcertizeException)
- def reindex(self):
- """Reindex step for a spack environment
- Raises:
- SpackReindexException: If the spack reindex command fails.
- """
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack reindex',
- check=True,
- logger=self.logger,
- info_msg=f'Reindex step.',
- exception_msg=f'Failed the reindex.',
- exception=SpackReindexException)
-
- def spec_pacakge(self, package_name: str):
- """Reindex step for a spack environment
- Raises:
- SpackSpecException: If the spack spec command fails.
- """
- try:
- spec_output = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack spec {package_name}',
- check=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- logger=self.logger,
- info_msg=f'Spack spec {package_name}.',
- exception_msg=f'Failed to spack spec {package_name}.',
- exception=SpackSpecException).stdout
- pattern = r'^\s*-\s*([\w.-]+@[\d.]+)'
- match = re.search(pattern, spec_output)
- if match:
- return match.group(1)
- return None
- except SpackSpecException:
- return None
-
- def create_gpg_keys(self):
- """Creates GPG keys (which can be used when creating binary cashes) and adds it to the trusted keyring."""
- if self.spack_config.gpg:
- run_command("bash", "-c",
- f'{self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
- check=True,
- logger=self.logger,
- info_msg=f'Created pgp keys for {self.spack_config.env.name}',
- exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.name}',
- exception=SpackGpgException)
- else:
- raise SpackGpgException('No GPG configuration was defined is spack configuration')
-
- def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
- """Adds a Spack mirror.
- Adds a new mirror to the Spack configuration, either globally or to a specific environment.
- Args:
- mirror_name (str): The name of the mirror.
- mirror_path (str): The path or URL of the mirror.
- signed (bool): Whether to require signed packages from the mirror.
- autopush (bool): Whether to enable autopush for the mirror.
- global_mirror (bool): Whether to add the mirror globally (True) or to the current environment (False).
- Raises:
- ValueError: If mirror_name or mirror_path are empty.
- NoSpackEnvironmentException: If global_mirror is False and no environment is defined.
- """
- autopush = '--autopush' if autopush else ''
- signed = '--signed' if signed else ''
- spack_add_mirror = f'spack mirror add {autopush} {signed} {mirror_name} {mirror_path}'
- if global_mirror:
- run_command("bash", "-c",
- f'{self.spack_setup_script} && {spack_add_mirror}',
- check=True,
- logger=self.logger,
- info_msg=f'Added mirror {mirror_name}',
- exception_msg=f'Failed to add mirror {mirror_name}',
- exception=SpackMirrorException)
- else:
- check_spack_env(
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && {spack_add_mirror}',
- check=True,
- logger=self.logger,
- info_msg=f'Added mirror {mirror_name}',
- exception_msg=f'Failed to add mirror {mirror_name}',
- exception=SpackMirrorException))
-
- @check_spack_env
- def trust_gpg_key(self, public_key_path: str):
- """Adds a GPG public key to the trusted keyring.
- This method attempts to add the provided GPG public key to the
- Spack trusted keyring.
- Args:
- public_key_path (str): Path to the GPG public key file.
- Returns:
- bool: True if the key was added successfully, False otherwise.
- Raises:
- ValueError: If public_key_path is empty.
- NoSpackEnvironmentException: If the spack environment is not set up.
- """
- if not public_key_path:
- raise ValueError("public_key_path is required")
-
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack gpg trust {public_key_path}',
- check=True,
- logger=self.logger,
- info_msg=f'Trusted GPG key for {self.spack_config.env.name}',
- exception_msg=f'Failed to trust GPG key for {self.spack_config.env.name}',
- exception=SpackGpgException)
-
- def config(self, config_type: SpackConfigCommand, config_parameter):
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack config {config_type.value} \"{config_parameter}\"',
- check=True,
- logger=self.logger,
- info_msg='Spack config command',
- exception_msg='Spack config command failed',
- exception=SpackConfigException)
-
- def mirror_list(self):
- """Returns of available mirrors. When an environment is activated it will return the mirrors associated with it,
- otherwise the mirrors set globally"""
- mirrors = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack mirror list',
- check=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- logger=self.logger,
- info_msg=f'Listing mirrors',
- exception_msg=f'Failed list mirrors',
- exception=SpackMirrorException).stdout
- return list(map(get_first_word, list(mirrors.strip().splitlines())))
-
- def remove_mirror(self, mirror_name: str):
- """Removes a mirror from an environment (if it is activated), otherwise removes the mirror globally."""
- if not mirror_name:
- raise ValueError("mirror_name is required")
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack mirror rm {mirror_name}',
- check=True,
- logger=self.logger,
- info_msg=f'Removing mirror {mirror_name}',
- exception_msg=f'Failed to remove mirror {mirror_name}',
- exception=SpackMirrorException)
-
- def update_buildcache_index(self, mirror_path: str):
- """Updates buildcache index"""
- if not mirror_path:
- raise ValueError("mirror_path is required")
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack buildcache update-index {mirror_path}',
- check=True,
- logger=self.logger,
- info_msg=f'Updating build cache index for mirror {mirror_path}',
- exception_msg=f'Failed to update build cache index for mirror {mirror_path}',
- exception=SpackMirrorException)
-
- def install_gpg_keys(self):
- """Install gpg keys"""
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack buildcache keys --install --trust',
- check=True,
- logger=self.logger,
- info_msg=f'Installing gpg keys from mirror',
- exception_msg=f'Failed to install gpg keys from mirror',
- exception=SpackGpgException)
-
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False, debug=False, test=None):
"""Installs all spack packages.
@@ -408,83 +155,60 @@ class SpackOperation:
self.logger.error(f'Something went wrong during installation. Please check the logs.')
return install_result
+ def create_fetch_spack_environment(self):
+ self.spack_env_operation.create_fetch_spack_environment()
+
+ def add_spack_repo(self, repo_path: Path, repo_name: str):
+ self.spack_env_operation.add_spack_repo(repo_path, repo_name)
+
+ def spack_repo_exists(self, repo_name: str) -> bool | None:
+ return self.spack_env_operation.spack_repo_exists(repo_name)
+
+ def spack_env_exists(self):
+ return self.spack_env_operation.spack_env_exists()
+
+ @check_spack_env
+ def get_compiler_version(self):
+ return self.spack_env_operation.get_compiler_version()
+
+ def get_spack_installed_version(self):
+ return self.spack_tool_operation.get_spack_installed_version()
+
+ def reindex(self):
+ self.spack_cache_operation.reindex()
+
+ def spec_pacakge(self, package_name: str):
+ return self.spack_env_operation.spec_pacakge(package_name)
+
+ def create_gpg_keys(self):
+ self.spack_cache_operation.create_gpg_keys()
+
+ def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
+ return self.spack_cache_operation.add_mirror(mirror_name, mirror_path, signed, autopush, global_mirror)
+
+ @check_spack_env
+ def trust_gpg_key(self, public_key_path: str):
+ self.spack_cache_operation.trust_gpg_key(public_key_path)
+
+ def config(self, config_type: SpackConfigCommand, config_parameter):
+ self.spack_env_operation.config(config_type, config_parameter)
+
+ def mirror_list(self):
+ return self.spack_cache_operation.mirror_list()
+
+ def remove_mirror(self, mirror_name: str):
+ self.spack_cache_operation.remove_mirror(mirror_name)
+
+ def update_buildcache_index(self, mirror_path: str):
+ self.spack_cache_operation.update_buildcache_index(mirror_path)
+
+ def install_gpg_keys(self):
+ self.spack_cache_operation.install_gpg_keys()
+
@check_spack_env
def find_packages(self):
- """Returns a dictionary of installed Spack packages in the current environment.
- Each key is the name of a Spack package, and the corresponding value is a list of
- installed versions for that package.
- Raises:
- NoSpackEnvironmentException: If the spack environment is not set up.
- """
- packages = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack find -c',
- check=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- logger=self.logger,
- info_msg=f'Listing installed packages.',
- exception_msg=f'Failed to list installed packages',
- exception=SpackFindException).stdout
- dict_packages = {}
- for package in packages.strip().splitlines():
- if package.startswith('[+]'):
- package = package.replace('@', ' ').split()
- if len(package) == 3:
- _, name, version = package
- dict_packages.setdefault(name, []).append(version)
- return dict_packages
+ return self.spack_env_operation.find_packages()
def install_spack(self, spack_version=f'{SPACK_VERSION}', spack_repo='https://github.com/spack/spack',
bashrc_path=os.path.expanduser("~/.bashrc")):
- """Install spack.
- Args:
- spack_version (str): spack version
- spack_repo (str): Git path to the Spack repository.
- bashrc_path (str): Path to the .bashrc file.
- """
- spack_version = f'v{spack_version}'
- try:
- user = os.getlogin()
- except OSError:
- user = None
-
- self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
- if not self.spack_dir.exists():
- run_command(
- "git", "clone", "--depth", "1",
- "-c", "advice.detachedHead=false",
- "-c", "feature.manyFiles=true",
- "--branch", spack_version, spack_repo, self.spack_dir
- , check=True, logger=self.logger)
- self.logger.debug("Cloned spack")
- else:
- self.logger.debug("Spack already cloned.")
-
- if bashrc_path:
- # ensure the file exists before opening it
- if not os.path.exists(bashrc_path):
- open(bashrc_path, "w").close()
- # add spack setup commands to .bashrc
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
- spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
- bashrc.write(f"{spack_setup_script}\n")
- self.logger.info("Added Spack PATH to .bashrc")
- if user:
- run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
- info_msg='Adding permissions to the logged in user')
- self.logger.info("Spack install completed")
- if self.spack_config.use_spack_global is True and bashrc_path is not None:
- # Restart the bash only of the spack is used globally
- self.logger.info('Restarting bash')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
- os.system("exec bash")
- # Configure upstream Spack instance if specified
- if self.spack_config.upstream_instance:
- search_path = os.path.join(self.spack_config.upstream_instance, 'spack', 'opt', 'spack', '**', '.spack-db')
- spack_db_dirs = glob.glob(search_path, recursive=True)
- upstream_prefix = [os.path.dirname(dir) for dir in spack_db_dirs]
- for prefix in upstream_prefix:
- self.config(SpackConfigCommand.ADD, f':upstream-spack-instance:install_tree:{prefix}')
- self.logger.info("Added upstream spack instance")
+ self.spack_tool_operation.install_spack(spack_version, spack_repo, bashrc_path)
diff --git a/dedal/spack_factory/SpackToolOperation.py b/dedal/spack_factory/SpackToolOperation.py
new file mode 100644
index 0000000000000000000000000000000000000000..996824800d3da06d588a064a478e83058b47e748
--- /dev/null
+++ b/dedal/spack_factory/SpackToolOperation.py
@@ -0,0 +1,107 @@
+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+# (c) Copyright 2025 Dedal developers
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+from pathlib import Path
+from dedal.enum.SpackConfigCommand import SpackConfigCommand
+from dedal.error_handling.exceptions import MissingAttributeException
+from dedal.spack_factory.SpackEnvOperation import SpackEnvOperation
+from dedal.tests.testing_variables import SPACK_VERSION
+from dedal.utils.utils import run_command
+from dedal.logger.logger_builder import get_logger
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackToolOperation:
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__), spack_setup_script=None,
+ spack_dir=Path('./').resolve()):
+ self.spack_config = spack_config
+ self.logger = logger
+ if spack_setup_script:
+ self.spack_setup_script = spack_setup_script
+ else:
+ raise MissingAttributeException(f'Missing attribute for class {__name__}')
+ self.spack_dir = spack_dir
+
+ def get_spack_installed_version(self):
+ """Returns the spack installed version"""
+ spack_version = run_command("bash", "-c", f'{self.spack_setup_script} && spack --version',
+ capture_output=True, text=True, check=True,
+ logger=self.logger,
+ info_msg=f"Getting spack version",
+ exception_msg=f"Error retrieving Spack version")
+ if spack_version:
+ return spack_version.stdout.strip().split()[0]
+ return None
+
+ def install_spack(self, spack_version=f'{SPACK_VERSION}', spack_repo='https://github.com/spack/spack',
+ bashrc_path=os.path.expanduser("~/.bashrc")):
+ """Install spack.
+ Args:
+ spack_version (str): spack version
+ spack_repo (str): Git path to the Spack repository.
+ bashrc_path (str): Path to the .bashrc file.
+ """
+ spack_version = f'v{spack_version}'
+ try:
+ user = os.getlogin()
+ except OSError:
+ user = None
+
+ self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
+ if not self.spack_dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ "--branch", spack_version, spack_repo, self.spack_dir
+ , check=True, logger=self.logger)
+ self.logger.debug("Cloned spack")
+ else:
+ self.logger.debug("Spack already cloned.")
+
+ if bashrc_path:
+ # ensure the file exists before opening it
+ if not os.path.exists(bashrc_path):
+ open(bashrc_path, "w").close()
+ # add spack setup commands to .bashrc
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
+ spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
+ bashrc.write(f"{spack_setup_script}\n")
+ self.logger.info("Added Spack PATH to .bashrc")
+ if user:
+ run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
+ info_msg='Adding permissions to the logged in user')
+ self.logger.info("Spack install completed")
+ if self.spack_config.use_spack_global is True and bashrc_path is not None:
+ # Restart the bash only of the spack is used globally
+ self.logger.info('Restarting bash')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
+ os.system("exec bash")
+ # Configure upstream Spack instance if specified
+ if self.spack_config.upstream_instance:
+ search_path = os.path.join(self.spack_config.upstream_instance, 'spack', 'opt', 'spack', '**', '.spack-db')
+ spack_db_dirs = glob.glob(search_path, recursive=True)
+ upstream_prefix = [os.path.dirname(dir) for dir in spack_db_dirs]
+ spack_env_operation = SpackEnvOperation(spack_config=self.spack_config,
+ spack_setup_script=self.spack_setup_script)
+ for prefix in upstream_prefix:
+ # todo fix
+ spack_env_operation.config(SpackConfigCommand.ADD, f':upstream-spack-instance:install_tree:{prefix}')
+ self.logger.info("Added upstream spack instance")