diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 98a371df8b3604b478d5bf783571dc22e0fe1424..b248ae041b36e3fac5d7ef73ea268774ba923790 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,18 +28,16 @@ testing-pytest:
- docker-runner
image: ubuntu:22.04
script:
- - chmod +x esd/utils/bootstrap.sh
- - ./esd/utils/bootstrap.sh
- - echo "$SPACK_ENV_ACCESS_TOKEN"
- - pip install -e .[tests,dev]
- - pytest ./esd/tests/ -s --junitxml=test-results.xml
+ - chmod +x dedal/utils/bootstrap.sh
+ - ./dedal/utils/bootstrap.sh
+ - pip install e .[tests,dev]
+ - pytest ./dedal/tests/ -s --junitxml=test-results.xml
artifacts:
when: always
reports:
junit: test-results.xml
paths:
- test-results.xml
- - .esd.log
- - .generate_cache.log
+ - .dedal.log
expire_in: 1 week
diff --git a/esd/__init__.py b/dedal/__init__.py
similarity index 100%
rename from esd/__init__.py
rename to dedal/__init__.py
diff --git a/esd/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
similarity index 96%
rename from esd/build_cache/BuildCacheManager.py
rename to dedal/build_cache/BuildCacheManager.py
index 8641a68e98c916d85321d8853c81abf55ac4a560..eed6a45d41db7d677a5e347a4699caef90fce09e 100644
--- a/esd/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -3,10 +3,9 @@ import os
from os.path import join
from pathlib import Path
-import oras.client
-
-from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
-from esd.logger.logger_builder import get_logger
+from dedal.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from dedal.logger.logger_builder import get_logger
+from dedal.utils.utils import clean_up
class BuildCacheManager(BuildCacheManagerInterface):
@@ -113,7 +112,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
if tags is not None:
try:
self._client.delete_tags(self._oci_registry_path, tags)
- self._logger.info("Successfully deleted all artifacts form OCI registry.")
+ self._logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
self._logger.error(
f"Failed to delete artifacts: {e}")
diff --git a/esd/build_cache/BuildCacheManagerInterface.py b/dedal/build_cache/BuildCacheManagerInterface.py
similarity index 100%
rename from esd/build_cache/BuildCacheManagerInterface.py
rename to dedal/build_cache/BuildCacheManagerInterface.py
diff --git a/esd/build_cache/__init__.py b/dedal/build_cache/__init__.py
similarity index 100%
rename from esd/build_cache/__init__.py
rename to dedal/build_cache/__init__.py
diff --git a/esd/cli/SpackManager.py b/dedal/cli/SpackManager.py
similarity index 100%
rename from esd/cli/SpackManager.py
rename to dedal/cli/SpackManager.py
diff --git a/esd/cli/__init__.py b/dedal/cli/__init__.py
similarity index 100%
rename from esd/cli/__init__.py
rename to dedal/cli/__init__.py
diff --git a/esd/cli/fetch_cached_buildresults.py b/dedal/cli/fetch_cached_buildresults.py
similarity index 100%
rename from esd/cli/fetch_cached_buildresults.py
rename to dedal/cli/fetch_cached_buildresults.py
diff --git a/dedal/configuration/GpgConfig.py b/dedal/configuration/GpgConfig.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8f0c2d3bc0f39db8c5b251d9ffc6f6fa3a577ec
--- /dev/null
+++ b/dedal/configuration/GpgConfig.py
@@ -0,0 +1,7 @@
+class GpgConfig:
+ """
+ Configuration for gpg key used by spack
+ """
+ def __init__(self, gpg_name='example', gpg_mail='example@example.com'):
+ self.name = gpg_name
+ self.mail = gpg_mail
diff --git a/dedal/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d4706796ec13d6bf733c3d68459e455a6da53b7
--- /dev/null
+++ b/dedal/configuration/SpackConfig.py
@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.model import SpackDescriptor
+
+
+class SpackConfig:
+ def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
+ install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
+ concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None):
+ self.env = env
+ if repos is None:
+ self.repos = []
+ else:
+ self.repos = repos
+ self.install_dir = install_dir
+ if self.install_dir:
+ os.makedirs(self.install_dir, exist_ok=True)
+ self.upstream_instance = upstream_instance
+ self.system_name = system_name
+ self.concretization_dir = concretization_dir
+ if self.concretization_dir:
+ os.makedirs(self.concretization_dir, exist_ok=True)
+ self.buildcache_dir = buildcache_dir
+ if self.buildcache_dir:
+ os.makedirs(self.buildcache_dir, exist_ok=True)
+ self.gpg = gpg
+
+ def add_repo(self, repo: SpackDescriptor):
+ if self.repos is None:
+ self.repos = []
+ else:
+ self.repos.append(repo)
diff --git a/esd/logger/__init__.py b/dedal/configuration/__init__.py
similarity index 100%
rename from esd/logger/__init__.py
rename to dedal/configuration/__init__.py
diff --git a/esd/tests/__init__.py b/dedal/error_handling/__init__.py
similarity index 100%
rename from esd/tests/__init__.py
rename to dedal/error_handling/__init__.py
diff --git a/dedal/error_handling/exceptions.py b/dedal/error_handling/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..0256f886ab0cf4b958ac12d59d6fcea2d5f568ec
--- /dev/null
+++ b/dedal/error_handling/exceptions.py
@@ -0,0 +1,41 @@
+class SpackException(Exception):
+
+ def __init__(self, message):
+ super().__init__(message)
+ self.message = str(message)
+
+ def __str__(self):
+ return self.message
+
+
+class BashCommandException(SpackException):
+ """
+ To be thrown when a bash command has failed
+ """
+
+
+class NoSpackEnvironmentException(BashCommandException):
+ """
+ To be thrown when an operation on a spack environment is executed without the environment being activated or existent
+ """
+
+
+class SpackConcertizeException(BashCommandException):
+ """
+ To be thrown when the spack concretization step fails
+ """
+
+class SpackInstallPackagesException(BashCommandException):
+ """
+ To be thrown when the spack fails to install spack packages
+ """
+
+class SpackMirrorException(BashCommandException):
+ """
+ To be thrown when the spack add mirror command fails
+ """
+
+class SpackGpgException(BashCommandException):
+ """
+ To be thrown when the spack fails to create gpg keys
+ """
diff --git a/esd/fetch_cached_sources.py b/dedal/fetch_cached_sources.py
similarity index 100%
rename from esd/fetch_cached_sources.py
rename to dedal/fetch_cached_sources.py
diff --git a/esd/utils/__init__.py b/dedal/logger/__init__.py
similarity index 100%
rename from esd/utils/__init__.py
rename to dedal/logger/__init__.py
diff --git a/esd/logger/logger_builder.py b/dedal/logger/logger_builder.py
similarity index 100%
rename from esd/logger/logger_builder.py
rename to dedal/logger/logger_builder.py
diff --git a/dedal/logger/logger_config.py b/dedal/logger/logger_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ca3b000fd171dde8ceafdc6731dfb02009e845c
--- /dev/null
+++ b/dedal/logger/logger_config.py
@@ -0,0 +1,33 @@
+import logging
+
+
+class LoggerConfig:
+ """
+ This class sets up logging with a file handler
+ and a stream handler, ensuring consistent
+ and formatted log messages.
+ """
+ def __init__(self, log_file):
+ self.log_file = log_file
+ self._configure_logger()
+
+ def _configure_logger(self):
+ formatter = logging.Formatter(
+ fmt='%(asctime)s - %(levelname)s - %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S'
+ )
+
+ file_handler = logging.FileHandler(self.log_file)
+ file_handler.setFormatter(formatter)
+
+ stream_handler = logging.StreamHandler()
+ stream_handler.setFormatter(formatter)
+
+ self.logger = logging.getLogger(__name__)
+ self.logger.setLevel(logging.DEBUG)
+
+ self.logger.addHandler(file_handler)
+ self.logger.addHandler(stream_handler)
+
+ def get_logger(self):
+ return self.logger
diff --git a/esd/logger/logging.conf b/dedal/logger/logging.conf
similarity index 85%
rename from esd/logger/logging.conf
rename to dedal/logger/logging.conf
index d95ba87c9d23a99f1cf98c24f961ddb9ca0528c8..fd24c026b5563ee757fca411727b0e31adac050c 100644
--- a/esd/logger/logging.conf
+++ b/dedal/logger/logging.conf
@@ -1,8 +1,8 @@
############################################
-## ESD - logging configuration. ##
+## Dedal - logging configuration. ##
############################################
[loggers]
-keys=root, esd, oras
+keys=root, dedal, oras
[handlers]
keys=consoleHandler, fileHandler
@@ -16,12 +16,12 @@ handlers=consoleHandler, fileHandler
propagate=0
############################################
-## esd specific logging ##
+## dedal specific logging ##
############################################
-[logger_esd]
+[logger_dedal]
level=DEBUG
handlers=consoleHandler, fileHandler
-qualname=esd
+qualname=dedal
propagate=0
[logger_oras]
@@ -44,7 +44,7 @@ args=(sys.stdout,)
class=handlers.TimedRotatingFileHandler
level=INFO
formatter=simpleFormatter
-args=('.esd.log', 'midnight', 1, 30, None, False, False)
+args=('.dedal.log', 'midnight', 1, 30, None, False, False)
############################################
## Formatters ##
diff --git a/dedal/model/SpackDescriptor.py b/dedal/model/SpackDescriptor.py
new file mode 100644
index 0000000000000000000000000000000000000000..70e484fb3d39e4333389682d14a32ac46c08a912
--- /dev/null
+++ b/dedal/model/SpackDescriptor.py
@@ -0,0 +1,13 @@
+import os
+from pathlib import Path
+
+
+class SpackDescriptor:
+ """"
+ Provides details about the spack environment
+ """
+
+ def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
+ self.env_name = env_name
+ self.path = path
+ self.git_path = git_path
diff --git a/dedal/model/__init__.py b/dedal/model/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
new file mode 100644
index 0000000000000000000000000000000000000000..ecfbb8e5084fc1e49b32894aecd47190df36fd98
--- /dev/null
+++ b/dedal/spack_factory/SpackOperation.py
@@ -0,0 +1,269 @@
+import os
+import re
+import subprocess
+from pathlib import Path
+from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
+ SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
+from dedal.logger.logger_builder import get_logger
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.tests.testing_variables import SPACK_VERSION
+from dedal.wrapper.spack_wrapper import check_spack_env
+from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
+
+
+class SpackOperation:
+ """
+ This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
+ Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
+
+ Attributes:
+ -----------
+ env : SpackDescriptor
+ spack environment details
+ repos : list[SpackDescriptor]
+ upstream_instance : str
+ path to Spack instance to use as upstream (optional)
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
+ self.spack_config = spack_config
+ self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.spack_config.install_dir / 'spack'
+ self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
+ self.logger = logger
+ if self.spack_config.env and spack_config.env.path:
+ self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
+
+ def create_fetch_spack_environment(self):
+ if self.spack_config.env.git_path:
+ git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
+ self.spack_config.env.git_path,
+ logger=self.logger)
+ else:
+ os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+ check=True, logger=self.logger,
+ info_msg=f"Created {self.spack_config.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
+ exception=BashCommandException)
+
+ def setup_spack_env(self):
+ """
+ This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
+ """
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ if self.spack_config.system_name:
+ set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
+ os.environ['SYSTEMNAME'] = self.spack_config.system_name
+ if self.spack_dir.exists() and self.spack_dir.is_dir():
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
+ self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
+ else:
+ self.logger.error(f'Invalid installation path: {self.spack_dir}')
+ # Restart the bash after adding environment variables
+ self.create_fetch_spack_environment()
+ if self.spack_config.install_dir.exists():
+ for repo in self.spack_config.repos:
+ repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
+ git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
+ if not self.spack_repo_exists(repo.env_name):
+ self.add_spack_repo(repo.path, repo.env_name)
+ self.logger.debug(f'Added spack repository {repo.env_name}')
+ else:
+ self.logger.debug(f'Spack repository {repo.env_name} already added')
+
+ def spack_repo_exists(self, repo_name: str) -> bool | None:
+ """Check if the given Spack repository exists."""
+ if self.spack_config.env is None:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Checking if {repo_name} exists')
+ if result is None:
+ return False
+ else:
+ if self.spack_env_exists():
+ result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Checking if repository {repo_name} was added')
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+ if result is None:
+ return False
+ return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
+
+ def spack_env_exists(self):
+ result = run_command("bash", "-c",
+ self.spack_command_on_env,
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
+ if result is None:
+ return False
+ return True
+
+ @check_spack_env
+ def add_spack_repo(self, repo_path: Path, repo_name: str):
+ """Add the Spack repository if it does not exist."""
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
+ exception=BashCommandException)
+
+ @check_spack_env
+ def get_compiler_version(self):
+ result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack compiler list',
+ check=True, logger=self.logger,
+ capture_output=True, text=True,
+ info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
+ exception=BashCommandException)
+ # todo add error handling and tests
+ if result.stdout is None:
+ self.logger.debug('No gcc found for {self.env.env_name}')
+ return None
+
+ # Find the first occurrence of a GCC compiler using regex
+ match = re.search(r"gcc@([\d\.]+)", result.stdout)
+ gcc_version = match.group(1)
+ self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
+ return gcc_version
+
+ def get_spack_installed_version(self):
+ spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ capture_output=True, text=True, check=True,
+ logger=self.logger,
+ info_msg=f"Getting spack version",
+ exception_msg=f"Error retrieving Spack version")
+ if spack_version:
+ return spack_version.stdout.strip().split()[0]
+ return None
+
+ @check_spack_env
+ def concretize_spack_env(self, force=True):
+ force = '--force' if force else ''
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack concretize {force}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Concertization step for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
+ exception=SpackConcertizeException)
+
+ def create_gpg_keys(self):
+ if self.spack_config.gpg:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.env_name}',
+ exception=SpackGpgException)
+ else:
+ raise SpackGpgException('No GPG configuration was defined is spack configuration')
+
+ def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
+ autopush = '--autopush' if autopush else ''
+ signed = '--signed' if signed else ''
+ if global_mirror:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException)
+ else:
+ check_spack_env(
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException))
+
+ def remove_mirror(self, mirror_name: str):
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Removing mirror {mirror_name}',
+ exception_msg=f'Failed to remove mirror {mirror_name}',
+ exception=SpackMirrorException)
+
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
+ signed = '' if signed else '--no-check-signature'
+ fresh = '--fresh' if fresh else ''
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
+ exception=SpackInstallPackagesException)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
+ return install_result
+
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
+ try:
+ user = os.getlogin()
+ except OSError:
+ user = None
+
+ self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
+ if not self.spack_dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ "--branch", spack_version, spack_repo, self.spack_dir
+ , check=True, logger=self.logger)
+ self.logger.debug("Cloned spack")
+ else:
+ self.logger.debug("Spack already cloned.")
+
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ # ensure the file exists before opening it
+ if not os.path.exists(bashrc_path):
+ open(bashrc_path, "w").close()
+ # add spack setup commands to .bashrc
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
+ bashrc.write(f"source {self.spack_setup_script}\n")
+ self.logger.info("Added Spack PATH to .bashrc")
+ if user:
+ run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
+ info_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
+ self.logger.info("Spack install completed")
+ # Restart Bash after the installation ends
+ os.system("exec bash")
+
+ # Configure upstream Spack instance if specified
+ if self.spack_config.upstream_instance:
+ upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
+ with open(upstreams_yaml_path, "w") as file:
+ file.write(f"""upstreams:
+ upstream-spack-instance:
+ install_tree: {self.spack_config.upstream_instance}/spack/opt/spack
+ """)
+ self.logger.info("Added upstream spack instance")
diff --git a/dedal/spack_factory/SpackOperationCreator.py b/dedal/spack_factory/SpackOperationCreator.py
new file mode 100644
index 0000000000000000000000000000000000000000..54517a845bad14629c6019416f0e19581472991e
--- /dev/null
+++ b/dedal/spack_factory/SpackOperationCreator.py
@@ -0,0 +1,14 @@
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+
+
+class SpackOperationCreator:
+ @staticmethod
+ def get_spack_operator(spack_config: SpackConfig = None):
+ if spack_config is None:
+ return SpackOperation(SpackConfig())
+ elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
+ return SpackOperation(spack_config)
+ else:
+ return SpackOperationUseCache(spack_config)
diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
new file mode 100644
index 0000000000000000000000000000000000000000..efb9af763af58a0f9e44ab79cab13af56947502e
--- /dev/null
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -0,0 +1,32 @@
+import os
+from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.logger.logger_builder import get_logger
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationUseCache(SpackOperation):
+ """
+ This class uses caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
+ cache_version_build='cache'):
+ super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=cache_version_build)
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
diff --git a/dedal/spack_factory/__init__.py b/dedal/spack_factory/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/esd/specfile_dag_hash.py b/dedal/specfile_dag_hash.py
similarity index 100%
rename from esd/specfile_dag_hash.py
rename to dedal/specfile_dag_hash.py
diff --git a/esd/specfile_storage_path_build.py b/dedal/specfile_storage_path_build.py
similarity index 100%
rename from esd/specfile_storage_path_build.py
rename to dedal/specfile_storage_path_build.py
diff --git a/esd/specfile_storage_path_source.py b/dedal/specfile_storage_path_source.py
similarity index 100%
rename from esd/specfile_storage_path_source.py
rename to dedal/specfile_storage_path_source.py
diff --git a/dedal/tests/__init__.py b/dedal/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/tests/integration_tests/__init__.py b/dedal/tests/integration_tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..2fec80f743d72190d0b175191660250981f98255
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -0,0 +1,204 @@
+from pathlib import Path
+import pytest
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
+from dedal.utils.utils import file_exists_and_not_empty
+
+
+def test_spack_repo_exists_1():
+ spack_operation = SpackOperationCreator.get_spack_operator()
+ spack_operation.install_spack()
+ assert spack_operation.spack_repo_exists('ebrains-spack-builds') == False
+
+
+def test_spack_repo_exists_2(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ with pytest.raises(NoSpackEnvironmentException):
+ spack_operation.spack_repo_exists(env.env_name)
+
+
+def test_spack_repo_exists_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ print(spack_operation.get_spack_installed_version())
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == False
+
+
+def test_spack_from_scratch_setup_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == False
+
+
+def test_spack_from_scratch_setup_2(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ repo = env
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('new_env1', install_dir)
+ repo = env
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ with pytest.raises(BashCommandException):
+ spack_operation.setup_spack_env()
+
+
+def test_spack_from_scratch_setup_4(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('new_env2', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_env_exists() == True
+
+
+def test_spack_not_a_valid_repo():
+ env = SpackDescriptor('ebrains-spack-builds', Path(), None)
+ repo = env
+ config = SpackConfig(env=env, system_name='ebrainslab')
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ with pytest.raises(BashCommandException):
+ spack_operation.add_spack_repo(repo.path, repo.env_name)
+
+
+def test_spack_from_scratch_concretize_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ repo = env
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_2(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ repo = env
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ repo = env
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == False
+
+
+def test_spack_from_scratch_concretize_4(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_5(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_6(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_7(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_install(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+ install_result = spack_operation.install_packages(jobs=2, signed=False, fresh=True, debug=False)
+ assert install_result.returncode == 0
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..564d5c6aa2138e815cd7d092215a4f2eee8816f6
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -0,0 +1,12 @@
+import pytest
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.tests.testing_variables import SPACK_VERSION
+
+
+# run this test first so that spack is installed only once for all the tests
+@pytest.mark.run(order=1)
+def test_spack_install_scratch():
+ spack_operation = SpackOperation()
+ spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
+ installed_spack_version = spack_operation.get_spack_installed_version()
+ assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/testing_variables.py b/dedal/tests/testing_variables.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab95bfa1e02658b30cef40bd6654915a51d33ad3
--- /dev/null
+++ b/dedal/tests/testing_variables.py
@@ -0,0 +1,6 @@
+import os
+
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
+SPACK_VERSION = "0.22.0"
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
diff --git a/dedal/tests/unit_tests/__init__.py b/dedal/tests/unit_tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/tests/unit_tests/utils_test.py b/dedal/tests/unit_tests/utils_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd47860602b6624f59e8fb7c06c7c156608e3eb3
--- /dev/null
+++ b/dedal/tests/unit_tests/utils_test.py
@@ -0,0 +1,150 @@
+import subprocess
+
+import pytest
+from pathlib import Path
+from unittest.mock import mock_open, patch, MagicMock
+from dedal.utils.utils import clean_up, file_exists_and_not_empty, log_command, run_command
+
+
+@pytest.fixture
+def temp_directories(tmp_path):
+ """
+ Create temporary directories with files and subdirectories for testing.
+ """
+ test_dirs = []
+
+ for i in range(3):
+ dir_path = tmp_path / f"test_dir_{i}"
+ dir_path.mkdir()
+ test_dirs.append(str(dir_path))
+
+ # Add a file to the directory
+ file_path = dir_path / f"file_{i}.txt"
+ file_path.write_text(f"This is a test file in {dir_path}")
+
+ # Add a subdirectory with a file
+ sub_dir = dir_path / f"subdir_{i}"
+ sub_dir.mkdir()
+ sub_file = sub_dir / f"sub_file_{i}.txt"
+ sub_file.write_text(f"This is a sub file in {sub_dir}")
+
+ return test_dirs
+
+
+def test_clean_up(temp_directories, mocker):
+ """
+ Test the clean_up function to ensure directories and contents are removed.
+ """
+ # Mock the logger using pytest-mock's mocker fixture
+ mock_logger = mocker.MagicMock()
+
+ # Ensure directories exist before calling clean_up
+ for dir_path in temp_directories:
+ assert Path(dir_path).exists()
+
+ clean_up(temp_directories, mock_logger)
+
+ for dir_path in temp_directories:
+ assert not Path(dir_path).exists()
+
+ for dir_path in temp_directories:
+ mock_logger.info.assert_any_call(f"Removing {Path(dir_path).resolve()}")
+
+
+def test_clean_up_nonexistent_dirs(mocker):
+ """
+ Test the clean_up function with nonexistent directories.
+ """
+ # Mock the logger using pytest-mock's mocker fixture
+ mock_logger = mocker.MagicMock()
+ nonexistent_dirs = ["nonexistent_dir_1", "nonexistent_dir_2"]
+
+ clean_up(nonexistent_dirs, mock_logger)
+
+ for dir_path in nonexistent_dirs:
+ mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
+
+
+def test_file_does_not_exist(tmp_path: Path):
+ non_existent_file = tmp_path / "non_existent.txt"
+ assert not file_exists_and_not_empty(non_existent_file)
+
+
+def test_file_exists_but_empty(tmp_path: Path):
+ empty_file = tmp_path / "empty.txt"
+ # Create an empty file
+ empty_file.touch()
+ assert not file_exists_and_not_empty(empty_file)
+
+
+def test_file_exists_and_not_empty(tmp_path: Path):
+ non_empty_file = tmp_path / "non_empty.txt"
+ non_empty_file.write_text("Some content")
+ assert file_exists_and_not_empty(non_empty_file)
+
+
+def test_log_command():
+ results = MagicMock()
+ results.stdout = "Test output"
+ results.stderr = "Test error"
+ mock_file = mock_open()
+
+ with patch("builtins.open", mock_file):
+ log_command(results, "logfile.log")
+
+ mock_file.assert_called_once_with("logfile.log", "w")
+ handle = mock_file()
+ handle.write.assert_any_call("Test output")
+ handle.write.assert_any_call("\n--- STDERR ---\n")
+ handle.write.assert_any_call("Test error")
+
+
+def test_run_command_success(mocker):
+ mock_subprocess = mocker.patch("subprocess.run", return_value=MagicMock(returncode=0))
+ mock_logger = MagicMock()
+ result = run_command('bash', '-c', 'echo hello', logger=mock_logger, info_msg="Running echo")
+ mock_logger.info.assert_called_with("Running echo: args: ('bash', '-c', 'echo hello')")
+ mock_subprocess.assert_called_once_with(('bash', '-c', 'echo hello'))
+ assert result.returncode == 0
+
+
+def test_run_command_not_found(mocker):
+ mocker.patch("subprocess.run", side_effect=FileNotFoundError)
+ mock_logger = MagicMock()
+ run_command("invalid_command", logger=mock_logger)
+ mock_logger.error.assert_called_with("Command not found. Please check the command syntax.")
+
+
+def test_run_command_permission_error(mocker):
+ mocker.patch("subprocess.run", side_effect=PermissionError)
+ mock_logger = MagicMock()
+ run_command("restricted_command", logger=mock_logger)
+ mock_logger.error.assert_called_with("Permission denied. Try running with appropriate permissions.")
+
+
+def test_run_command_timeout(mocker):
+ mocker.patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="test", timeout=5))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger)
+ mock_logger.error.assert_called_with("Command timed out. Try increasing the timeout duration.")
+
+
+def test_run_command_os_error(mocker):
+ mocker.patch("subprocess.run", side_effect=OSError("OS Error"))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger)
+ mock_logger.error.assert_called_with("OS error occurred: OS Error")
+
+
+def test_run_command_unexpected_exception(mocker):
+ mocker.patch("subprocess.run", side_effect=Exception("Unexpected Error"))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger)
+ mock_logger.error.assert_called_with("An unexpected error occurred: Unexpected Error")
+
+
+def test_run_command_called_process_error(mocker):
+ mocker.patch("subprocess.run", side_effect=subprocess.CalledProcessError(1, "test"))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger, exception_msg="Process failed")
+ mock_logger.error.assert_called_with("Process failed: Command 'test' returned non-zero exit status 1.")
diff --git a/esd/update_cached_buildresults.py b/dedal/update_cached_buildresults.py
similarity index 100%
rename from esd/update_cached_buildresults.py
rename to dedal/update_cached_buildresults.py
diff --git a/esd/update_cached_sources.py b/dedal/update_cached_sources.py
similarity index 100%
rename from esd/update_cached_sources.py
rename to dedal/update_cached_sources.py
diff --git a/dedal/utils/__init__.py b/dedal/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/utils/bootstrap.sh b/dedal/utils/bootstrap.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9b7d0131e95a3be9b0f2cfc4dc82492517fb22dc
--- /dev/null
+++ b/dedal/utils/bootstrap.sh
@@ -0,0 +1,6 @@
+# Minimal prerequisites for installing the esd_library
+# pip must be installed on the OS
+echo "Bootstrapping..."
+apt update
+apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+python3 -m pip install --upgrade pip setuptools wheel
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9fc82ad520b4a22715c819d3ca11042b88b87c7f
--- /dev/null
+++ b/dedal/utils/utils.py
@@ -0,0 +1,100 @@
+import logging
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from dedal.error_handling.exceptions import BashCommandException
+import re
+
+
+def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
+ """
+ All the folders from the list dirs are removed with all the content in them
+ """
+ for cleanup_dir in dirs:
+ cleanup_dir = Path(cleanup_dir).resolve()
+ if cleanup_dir.exists():
+ logger.info(f"Removing {cleanup_dir}")
+ try:
+ shutil.rmtree(Path(cleanup_dir))
+ except OSError as e:
+ logger.error(f"Failed to remove {cleanup_dir}: {e}")
+ if not ignore_errors:
+ raise e
+ else:
+ logger.info(f"{cleanup_dir} does not exist")
+
+
+def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
+ exception=None, **kwargs):
+ try:
+ logger.info(f'{info_msg}: args: {args}')
+ return subprocess.run(args, **kwargs)
+ except subprocess.CalledProcessError as e:
+ if exception_msg is not None:
+ logger.error(f"{exception_msg}: {e}")
+ if exception is not None:
+ raise exception(f'{exception_msg} : {e}')
+ else:
+ return None
+ except FileNotFoundError:
+ logger.error(f"Command not found. Please check the command syntax.")
+ except PermissionError:
+ logger.error(f"Permission denied. Try running with appropriate permissions.")
+ except subprocess.TimeoutExpired:
+ logger.error(f"Command timed out. Try increasing the timeout duration.")
+ except ValueError:
+ logger.error(f"Invalid argument passed to subprocess. Check function parameters.")
+ except OSError as e:
+ logger.error(f"OS error occurred: {e}")
+ except Exception as e:
+ logger.error(f"An unexpected error occurred: {e}")
+
+
+def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
+ if not dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ git_path, dir
+ , check=True, logger=logger,
+ info_msg=f'Cloned repository {repo_name}',
+ exception_msg=f'Failed to clone repository: {repo_name}',
+ exception=BashCommandException)
+ else:
+ logger.info(f'Repository {repo_name} already cloned.')
+
+
+def file_exists_and_not_empty(file: Path) -> bool:
+ return file.is_file() and file.stat().st_size > 0
+
+
+def log_command(results, log_file: str):
+ with open(log_file, "w") as log_file:
+ log_file.write(results.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(results.stderr)
+
+
+def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
+ logger: logging = logging.getLogger(__name__)):
+ """Update or add an environment variable in ~/.bashrc."""
+ with open(bashrc_path, "r") as file:
+ lines = file.readlines()
+ pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
+ updated = False
+ # Modify the existing variable if found
+ for i, line in enumerate(lines):
+ if pattern.match(line):
+ lines[i] = f'export {var_name}={value}\n'
+ updated = True
+ break
+ if not updated:
+ lines.append(f'\nexport {var_name}={value}\n')
+ logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
+ else:
+ logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
+ with open(bashrc_path, "w") as file:
+ file.writelines(lines)
diff --git a/dedal/wrapper/__init__.py b/dedal/wrapper/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/wrapper/spack_wrapper.py b/dedal/wrapper/spack_wrapper.py
new file mode 100644
index 0000000000000000000000000000000000000000..018cad482083beda9ee0a278f4bbb2d107476849
--- /dev/null
+++ b/dedal/wrapper/spack_wrapper.py
@@ -0,0 +1,15 @@
+import functools
+
+from dedal.error_handling.exceptions import NoSpackEnvironmentException
+
+
+def check_spack_env(method):
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.spack_env_exists():
+ return method(self, *args, **kwargs) # Call the method with 'self'
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+
+ return wrapper
diff --git a/esd/tests/utils_test.py b/esd/tests/utils_test.py
index 7f930e2f91288aab49ad7eea1875c8fc8368fbf8..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/esd/tests/utils_test.py
+++ b/esd/tests/utils_test.py
@@ -1,81 +0,0 @@
-import pytest
-from pathlib import Path
-
-from esd.utils.utils import clean_up, file_exists_and_not_empty
-
-
-@pytest.fixture
-def temp_directories(tmp_path):
- """
- Create temporary directories with files and subdirectories for testing.
- """
- test_dirs = []
-
- for i in range(3):
- dir_path = tmp_path / f"test_dir_{i}"
- dir_path.mkdir()
- test_dirs.append(str(dir_path))
-
- # Add a file to the directory
- file_path = dir_path / f"file_{i}.txt"
- file_path.write_text(f"This is a test file in {dir_path}")
-
- # Add a subdirectory with a file
- sub_dir = dir_path / f"subdir_{i}"
- sub_dir.mkdir()
- sub_file = sub_dir / f"sub_file_{i}.txt"
- sub_file.write_text(f"This is a sub file in {sub_dir}")
-
- return test_dirs
-
-
-def test_clean_up(temp_directories, mocker):
- """
- Test the clean_up function to ensure directories and contents are removed.
- """
- # Mock the logger using pytest-mock's mocker fixture
- mock_logger = mocker.MagicMock()
-
- # Ensure directories exist before calling clean_up
- for dir_path in temp_directories:
- assert Path(dir_path).exists()
-
- clean_up(temp_directories, mock_logger)
-
- for dir_path in temp_directories:
- assert not Path(dir_path).exists()
-
- for dir_path in temp_directories:
- mock_logger.info.assert_any_call(f"Removing {Path(dir_path).resolve()}")
-
-
-def test_clean_up_nonexistent_dirs(mocker):
- """
- Test the clean_up function with nonexistent directories.
- """
- # Mock the logger using pytest-mock's mocker fixture
- mock_logger = mocker.MagicMock()
- nonexistent_dirs = ["nonexistent_dir_1", "nonexistent_dir_2"]
-
- clean_up(nonexistent_dirs, mock_logger)
-
- for dir_path in nonexistent_dirs:
- mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
-
-
-def test_file_does_not_exist(tmp_path: Path):
- non_existent_file = tmp_path / "non_existent.txt"
- assert not file_exists_and_not_empty(non_existent_file)
-
-
-def test_file_exists_but_empty(tmp_path: Path):
- empty_file = tmp_path / "empty.txt"
- # Create an empty file
- empty_file.touch()
- assert not file_exists_and_not_empty(empty_file)
-
-
-def test_file_exists_and_not_empty(tmp_path: Path):
- non_empty_file = tmp_path / "non_empty.txt"
- non_empty_file.write_text("Some content")
- assert file_exists_and_not_empty(non_empty_file)
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 033cbc54bbcd33a5f6c59399227d71b620e7544f..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -1,88 +0,0 @@
-import logging
-import os
-import shutil
-import subprocess
-from pathlib import Path
-
-from esd.error_handling.exceptions import BashCommandException
-import re
-
-
-def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
- """
- All the folders from the list dirs are removed with all the content in them
- """
- for cleanup_dir in dirs:
- cleanup_dir = Path(cleanup_dir).resolve()
- if cleanup_dir.exists():
- logger.info(f"Removing {cleanup_dir}")
- try:
- shutil.rmtree(Path(cleanup_dir))
- except OSError as e:
- logger.error(f"Failed to remove {cleanup_dir}: {e}")
- if not ignore_errors:
- raise e
- else:
- logger.info(f"{cleanup_dir} does not exist")
-
-
-def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
- exception=None, **kwargs):
- try:
- logger.info(f'{info_msg}: args: {args}')
- return subprocess.run(args, **kwargs)
- except subprocess.CalledProcessError as e:
- if exception_msg is not None:
- logger.error(f"{exception_msg}: {e}")
- if exception is not None:
- raise exception(f'{exception_msg} : {e}')
- else:
- return None
-
-
-def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
- if not dir.exists():
- run_command(
- "git", "clone", "--depth", "1",
- "-c", "advice.detachedHead=false",
- "-c", "feature.manyFiles=true",
- git_path, dir
- , check=True, logger=logger,
- info_msg=f'Cloned repository {repo_name}',
- exception_msg=f'Failed to clone repository: {repo_name}',
- exception=BashCommandException)
- else:
- logger.info(f'Repository {repo_name} already cloned.')
-
-
-def file_exists_and_not_empty(file: Path) -> bool:
- return file.is_file() and file.stat().st_size > 0
-
-
-def log_command(results, log_file: str):
- with open(log_file, "w") as log_file:
- log_file.write(results.stdout)
- log_file.write("\n--- STDERR ---\n")
- log_file.write(results.stderr)
-
-
-def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
- logger: logging = logging.getLogger(__name__)):
- """Update or add an environment variable in ~/.bashrc."""
- with open(bashrc_path, "r") as file:
- lines = file.readlines()
- pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
- updated = False
- # Modify the existing variable if found
- for i, line in enumerate(lines):
- if pattern.match(line):
- lines[i] = f'export {var_name}={value}\n'
- updated = True
- break
- if not updated:
- lines.append(f'\nexport {var_name}={value}\n')
- logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
- else:
- logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
- with open(bashrc_path, "w") as file:
- file.writelines(lines)
diff --git a/pyproject.toml b/pyproject.toml
index c8b8f7b69e8c777453bbe3153265ef8d268a333e..62d0146c3c48b233ed29040756d2bf76731fb694 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
-name = "esd-tools"
+name = "dedal"
version = "0.1.0"
authors = [
{ name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de" },
@@ -20,7 +20,7 @@ dependencies = [
]
[tool.setuptools.data-files]
-"esd-tools" = ["esd/logger/logging.conf"]
+"dedal" = ["dedal/logger/logging.conf"]
[options.extras_require]
test = [