From 0fbcce75a2b3a08d8790cbd12c0b50a20f41f4c1 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 11:25:05 +0200
Subject: [PATCH 01/21] esd-spack-installation: setup and tests for a spack env
---
.gitlab-ci.yml | 9 +-
dedal/build_cache/BuildCacheManager.py | 68 ++++---
dedal/logger/logger_config.py | 33 ----
.../spack_from_scratch_test.py | 14 +-
.../integration_tests/spack_install_test.py | 23 ++-
esd/error_handling/exceptions.py | 13 ++
esd/model/SpackModel.py | 12 ++
esd/model/__init__.py | 0
esd/spack_manager/SpackManager.py | 180 ++++++++++++++++++
esd/spack_manager/__init__.py | 0
esd/spack_manager/enums/SpackManagerEnum.py | 6 +
esd/spack_manager/enums/__init__.py | 0
.../factory/SpackManagerBuildCache.py | 19 ++
.../factory/SpackManagerCreator.py | 14 ++
.../factory/SpackManagerScratch.py | 15 ++
esd/spack_manager/factory/__init__.py | 0
esd/utils/bootstrap.sh | 6 +
esd/utils/utils.py | 42 ++++
pyproject.toml | 4 +-
19 files changed, 367 insertions(+), 91 deletions(-)
delete mode 100644 dedal/logger/logger_config.py
create mode 100644 esd/error_handling/exceptions.py
create mode 100644 esd/model/SpackModel.py
create mode 100644 esd/model/__init__.py
create mode 100644 esd/spack_manager/SpackManager.py
create mode 100644 esd/spack_manager/__init__.py
create mode 100644 esd/spack_manager/enums/SpackManagerEnum.py
create mode 100644 esd/spack_manager/enums/__init__.py
create mode 100644 esd/spack_manager/factory/SpackManagerBuildCache.py
create mode 100644 esd/spack_manager/factory/SpackManagerCreator.py
create mode 100644 esd/spack_manager/factory/SpackManagerScratch.py
create mode 100644 esd/spack_manager/factory/__init__.py
create mode 100644 esd/utils/bootstrap.sh
create mode 100644 esd/utils/utils.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2b497048..0ce02907 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,6 @@ stages:
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
-
build-wheel:
stage: build
tags:
@@ -28,16 +27,16 @@ testing-pytest:
- docker-runner
image: ubuntu:22.04
script:
- - chmod +x dedal/utils/bootstrap.sh
- - ./dedal/utils/bootstrap.sh
+ - chmod +x esd/utils/bootstrap.sh
+ - ./esd/utils/bootstrap.sh
- pip install .
- - pytest ./dedal/tests/ -s --junitxml=test-results.xml
+ - pytest ./esd/tests/ -s --junitxml=test-results.xml
artifacts:
when: always
reports:
junit: test-results.xml
paths:
- test-results.xml
- - .dedal.log
+ - .esd.log
expire_in: 1 week
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index 55fa10cb..e1bd6824 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -2,9 +2,9 @@ import os
import oras.client
from pathlib import Path
-from dedal.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
-from dedal.logger.logger_builder import get_logger
-from dedal.utils.utils import clean_up
+from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from esd.logger.logger_builder import get_logger
+from esd.utils.utils import clean_up
class BuildCacheManager(BuildCacheManagerInterface):
@@ -12,51 +12,48 @@ class BuildCacheManager(BuildCacheManagerInterface):
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
- auth_backend='basic',
- insecure=False):
- self._logger = get_logger(__name__, BuildCacheManager.__name__)
- self._registry_project = registry_project
+ def __init__(self, auth_backend='basic', insecure=False):
+ self.logger = get_logger(__name__, BuildCacheManager.__name__)
+ self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
+ self.registry_project = os.environ.get("REGISTRY_PROJECT")
- self._registry_username = registry_username
- self._registry_password = registry_password
+ self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
+ self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
- self._registry_host = registry_host
+ self.registry_host = str(os.environ.get("REGISTRY_HOST"))
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend,
- insecure=insecure)
- self._client.login(username=self._registry_username, password=self._registry_password)
- self.cache_version = cache_version
- self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/{self.cache_version}'
+ self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
+ self.client.login(username=self._registry_username, password=self._registry_password)
+ self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = out_dir.resolve()
+ build_cache_path = self.home_path / out_dir
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- self._logger.error(f"Path {build_cache_path} not found.")
+ self.logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
- target = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{str(sub_path.name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
+ target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
try:
- self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
- self._client.push(
+ self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self.client.push(
files=[str(sub_path)],
target=target,
# save in manifest the relative path for reconstruction
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self._logger.info(f"Successfully pushed {sub_path.name}")
+ self.logger.info(f"Successfully pushed {sub_path.env_name}")
except Exception as e:
- self._logger.error(
+ self.logger.error(
f"An error occurred while pushing: {e}")
# todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
# clean_up([str(build_cache_path)], self.logger)
@@ -66,38 +63,37 @@ class BuildCacheManager(BuildCacheManagerInterface):
This method retrieves all tags from an OCI Registry
"""
try:
- return self._client.get_tags(self._oci_registry_path)
+ return self.client.get_tags(self.oci_registry_path)
except Exception as e:
- self._logger.error(f"Failed to list tags: {e}")
+ self.logger.error(f"Failed to list tags: {e}")
return None
def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = in_dir.resolve()
+ build_cache_path = self.home_path / in_dir
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}"
+ ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self._client.get_manifest(
- f'{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}')[
+ self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
'annotations'][
'path']
try:
- self._client.pull(
+ self.client.pull(
ref,
# missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- self._logger.info(f"Successfully pulled artifact {tag}.")
+ self.logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- self._logger.error(
+ self.logger.error(
f"Failed to pull artifact {tag} : {e}")
def delete(self):
@@ -110,8 +106,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
tags = self.list_tags()
if tags is not None:
try:
- self._client.delete_tags(self._oci_registry_path, tags)
- self._logger.info(f"Successfully deleted all artifacts form OCI registry.")
+ self.client.delete_tags(self.oci_registry_path, tags)
+ self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- self._logger.error(
+ self.logger.error(
f"Failed to delete artifacts: {e}")
diff --git a/dedal/logger/logger_config.py b/dedal/logger/logger_config.py
deleted file mode 100644
index 3ca3b000..00000000
--- a/dedal/logger/logger_config.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import logging
-
-
-class LoggerConfig:
- """
- This class sets up logging with a file handler
- and a stream handler, ensuring consistent
- and formatted log messages.
- """
- def __init__(self, log_file):
- self.log_file = log_file
- self._configure_logger()
-
- def _configure_logger(self):
- formatter = logging.Formatter(
- fmt='%(asctime)s - %(levelname)s - %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S'
- )
-
- file_handler = logging.FileHandler(self.log_file)
- file_handler.setFormatter(formatter)
-
- stream_handler = logging.StreamHandler()
- stream_handler.setFormatter(formatter)
-
- self.logger = logging.getLogger(__name__)
- self.logger.setLevel(logging.DEBUG)
-
- self.logger.addHandler(file_handler)
- self.logger.addHandler(stream_handler)
-
- def get_logger(self):
- return self.logger
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
index d080bc7b..fa862301 100644
--- a/dedal/tests/integration_tests/spack_from_scratch_test.py
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -65,14 +65,12 @@ def test_spack_from_scratch_setup_3(tmp_path):
spack_operation.setup_spack_env()
-def test_spack_from_scratch_setup_4(tmp_path):
- install_dir = tmp_path
- env = SpackDescriptor('new_env2', install_dir)
- config = SpackConfig(env=env, install_dir=install_dir)
- spack_operation = SpackOperationCreator.get_spack_operator(config)
- spack_operation.install_spack()
- spack_operation.setup_spack_env()
- assert spack_operation.spack_env_exists() == True
+def test_spack_from_scratch_setup_4():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('new_environment', install_dir, )
+ spack_manager = SpackManagerScratch(env, system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == True
def test_spack_not_a_valid_repo():
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 564d5c6a..34f68323 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -1,12 +1,21 @@
import pytest
-from dedal.spack_factory.SpackOperation import SpackOperation
-from dedal.tests.testing_variables import SPACK_VERSION
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-# run this test first so that spack is installed only once for all the tests
+
+# we need this test to run first so that spack is installed only once
@pytest.mark.run(order=1)
def test_spack_install_scratch():
- spack_operation = SpackOperation()
- spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
- installed_spack_version = spack_operation.get_spack_installed_version()
- assert SPACK_VERSION == installed_spack_version
+ spack_manager = SpackManagerScratch()
+ spack_manager.install_spack(spack_version="v0.21.1")
+ installed_spack_version = spack_manager.get_spack_installed_version()
+ required_version = "0.21.1"
+ assert required_version == installed_spack_version
+
+
+def test_spack_install_buildcache():
+ spack_manager = SpackManagerBuildCache()
+ installed_spack_version = spack_manager.get_spack_installed_version()
+ required_version = "0.21.1"
+ assert required_version == installed_spack_version
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
new file mode 100644
index 00000000..2acca54e
--- /dev/null
+++ b/esd/error_handling/exceptions.py
@@ -0,0 +1,13 @@
+class SpackException(Exception):
+
+ def __init__(self, message):
+ super().__init__(message)
+ self.message = str(message)
+
+ def __str__(self):
+ return self.message
+
+class BashCommandException(SpackException):
+ """
+ To be thrown when an invalid input is received.
+ """
diff --git a/esd/model/SpackModel.py b/esd/model/SpackModel.py
new file mode 100644
index 00000000..4b065dba
--- /dev/null
+++ b/esd/model/SpackModel.py
@@ -0,0 +1,12 @@
+from pathlib import Path
+
+
+class SpackModel:
+ """"
+ Provides details about the spack environment
+ """
+
+ def __init__(self, env_name: str, path: Path, git_path: str = None):
+ self.env_name = env_name
+ self.path = path
+ self.git_path = git_path
diff --git a/esd/model/__init__.py b/esd/model/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
new file mode 100644
index 00000000..cdf4a1d3
--- /dev/null
+++ b/esd/spack_manager/SpackManager.py
@@ -0,0 +1,180 @@
+import os
+import subprocess
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from esd.error_handling.exceptions import BashCommandException
+from esd.logger.logger_builder import get_logger
+from esd.model.SpackModel import SpackModel
+from esd.utils.utils import run_command, git_clone_repo
+
+
+class SpackManager(ABC):
+ """
+ This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
+ Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
+
+ Attributes:
+ -----------
+ env : SpackModel
+ spack environment details
+ repos : list[SpackModel]
+ upstream_instance : str
+ path to Spack instance to use as upstream (optional)
+ """
+
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
+ if repos is None:
+ self.repos = list()
+ self.env = env
+ self.upstream_instance = upstream_instance
+ self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd())
+ self.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.install_dir / "spack"
+ self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ self.logger = logger
+ self.system_name = system_name
+
+ @abstractmethod
+ def concretize_spack_env(self, force=True):
+ pass
+
+ @abstractmethod
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
+
+ def create_fetch_spack_environment(self):
+ env_dir = self.install_dir / self.env.path / self.env.env_name
+ if self.env.git_path:
+ try:
+ git_clone_repo(self.env.env_name, env_dir, self.env.git_path, logger=self.logger)
+ except subprocess.CalledProcessError as e:
+ self.logger.exception(f'Failed to clone repository: {self.env.env_name}: {e}')
+ raise BashCommandException(f'Failed to clone repository: {self.env.env_name}: {e}')
+ else:
+ try:
+ os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env create -d {self.env.path}/{self.env.env_name}',
+ check=True, logger=self.logger)
+ self.logger.debug(f"Created {self.env.env_name} spack environment")
+ except subprocess.CalledProcessError as e:
+ self.logger.error(f"Failed to create {self.env.env_name} spack environment")
+ raise BashCommandException(f"Failed to create {self.env.env_name} spack environment")
+
+ def setup_spack_env(self):
+ """
+ This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
+ """
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ if self.system_name:
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
+ os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_dir.exists() and self.spack_dir.is_dir():
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
+ bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
+ else:
+ self.logger.error(f'Invalid installation path: {self.spack_dir}')
+ # Restart the bash after adding environment variables
+ self.create_fetch_spack_environment()
+ if self.install_dir.exists():
+ for repo in self.repos:
+ repo_dir = self.install_dir / repo.path / repo.env_name
+ git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
+ if not self.spack_repo_exists(repo.env_name):
+ self.add_spack_repo(repo.path, repo.env_name)
+ self.logger.debug(f'Added spack repository {repo.env_name}')
+ else:
+ self.logger.debug(f'Spack repository {repo.env_name} already added')
+
+ def spack_repo_exists(self, repo_name: str) -> bool:
+ """Check if the given Spack repository exists."""
+ if self.env is None:
+ try:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger)
+ except subprocess.CalledProcessError:
+ return False
+ else:
+ try:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger)
+ except subprocess.CalledProcessError:
+ return False
+ return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
+
+ def add_spack_repo(self, repo_path: Path, repo_name: str):
+ """Add the Spack repository if it does not exist."""
+ repo_path = repo_path.resolve().as_posix()
+ try:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger)
+ self.logger.debug(f"Added {repo_name} to spack environment {self.env.env_name}")
+ except subprocess.CalledProcessError as e:
+ self.logger.error(f"Failed to add {repo_name} to spack environment {self.env.env_name}")
+ raise BashCommandException(f"Failed to add {repo_name} to spack environment {self.env.env_name}: {e}")
+
+ def get_spack_installed_version(self):
+ try:
+ spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ capture_output=True, text=True, check=True,
+ logger=self.logger)
+ spack_version = spack_version.stdout.strip().split()[0]
+ self.logger.debug(f"Getting spack version: {spack_version}")
+ return spack_version
+ except subprocess.SubprocessError as e:
+ self.logger.error(f"Error retrieving Spack version: {e}")
+ return None
+
+ def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+ try:
+ user = os.getlogin()
+ except OSError:
+ user = None
+
+ self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
+ if not self.spack_dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ "--branch", spack_version, spack_repo, self.spack_dir
+ , check=True, logger=self.logger)
+ self.logger.debug("Cloned spack")
+ else:
+ self.logger.debug("Spack already cloned.")
+
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ # ensure the file exists before opening it
+ if not os.path.exists(bashrc_path):
+ open(bashrc_path, "w").close()
+ # add spack setup commands to .bashrc
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
+ bashrc.write(f"source {self.spack_setup_script}\n")
+ self.logger.info("Added Spack PATH to .bashrc")
+ if user:
+ run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger)
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger)
+ self.logger.info("Spack install completed")
+ # Restart Bash after the installation ends
+ os.system("exec bash")
+
+ # Configure upstream Spack instance if specified
+ if self.upstream_instance:
+ upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
+ with open(upstreams_yaml_path, "w") as file:
+ file.write(f"""upstreams:
+ upstream-spack-instance:
+ install_tree: {self.upstream_instance}/spack/opt/spack
+ """)
+ self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_manager/__init__.py b/esd/spack_manager/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
new file mode 100644
index 00000000..a2435839
--- /dev/null
+++ b/esd/spack_manager/enums/SpackManagerEnum.py
@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class SpackManagerEnum(Enum):
+ FROM_SCRATCH = "from_scratch",
+ FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/enums/__init__.py b/esd/spack_manager/enums/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
new file mode 100644
index 00000000..38151c6d
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerBuildCache.py
@@ -0,0 +1,19 @@
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerBuildCache(SpackManager):
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
+ super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
+
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
new file mode 100644
index 00000000..9728467f
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -0,0 +1,14 @@
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+class SpackManagerCreator:
+ @staticmethod
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
+ upstream_instance: str):
+ if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
+ return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
+ return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
+
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
new file mode 100644
index 00000000..5a79797c
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -0,0 +1,15 @@
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerScratch(SpackManager):
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
+ super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+ def concretize_spack_env(self, force=True):
+ pass
+
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_manager/factory/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/utils/bootstrap.sh b/esd/utils/bootstrap.sh
new file mode 100644
index 00000000..9b7d0131
--- /dev/null
+++ b/esd/utils/bootstrap.sh
@@ -0,0 +1,6 @@
+# Minimal prerequisites for installing the esd_library
+# pip must be installed on the OS
+echo "Bootstrapping..."
+apt update
+apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+python3 -m pip install --upgrade pip setuptools wheel
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
new file mode 100644
index 00000000..d229e345
--- /dev/null
+++ b/esd/utils/utils.py
@@ -0,0 +1,42 @@
+import logging
+import shutil
+import subprocess
+from pathlib import Path
+
+
+def clean_up(dirs: list[str], logging, ignore_errors=True):
+ """
+ All the folders from the list dirs are removed with all the content in them
+ """
+ for cleanup_dir in dirs:
+ cleanup_dir = Path(cleanup_dir).resolve()
+ if cleanup_dir.exists():
+ logging.info(f"Removing {cleanup_dir}")
+ try:
+ shutil.rmtree(Path(cleanup_dir))
+ except OSError as e:
+ logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ if not ignore_errors:
+ raise e
+ else:
+ logging.info(f"{cleanup_dir} does not exist")
+
+
+def run_command(*args, logger: None, **kwargs):
+ if logger is None:
+ logger = logging.getLogger(__name__)
+ logger.debug(f'{args}')
+ return subprocess.run(args, **kwargs)
+
+
+def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging):
+ if not dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ git_path, dir
+ , check=True, logger=logger)
+ logger.debug(f'Cloned repository {repo_name}')
+ else:
+ logger.debug(f'Repository {repo_name} already cloned.')
diff --git a/pyproject.toml b/pyproject.toml
index b6679ca1..abcbe05d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
-name = "dedal"
+name = "esd-tools"
version = "0.1.0"
authors = [
{name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
@@ -22,4 +22,4 @@ dependencies = [
]
[tool.setuptools.data-files]
-"dedal" = ["dedal/logger/logging.conf"]
\ No newline at end of file
+"esd-tools" = ["esd/logger/logging.conf"]
\ No newline at end of file
--
GitLab
From 49f82dfb268ffbc608e4a209221ccb386ab06ff7 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 11:33:28 +0200
Subject: [PATCH 02/21] esd-spack-installation: fixing tests
---
esd/spack_manager/SpackManager.py | 109 ++++++++++++++++--------------
esd/utils/utils.py | 27 +++++---
2 files changed, 76 insertions(+), 60 deletions(-)
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index cdf4a1d3..f7bb00b4 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -1,5 +1,4 @@
import os
-import subprocess
from abc import ABC, abstractmethod
from pathlib import Path
@@ -26,11 +25,17 @@ class SpackManager(ABC):
def __init__(self, env: SpackModel = None, repos=None,
upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
if repos is None:
- self.repos = list()
+ repos = []
+ self.repos = repos
self.env = env
- self.upstream_instance = upstream_instance
- self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd())
+ self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
self.install_dir.mkdir(parents=True, exist_ok=True)
+ self.env_path = None
+ if self.env and self.env.path:
+ self.env.path = self.env.path.resolve()
+ self.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = self.env.path / self.env.env_name
+ self.upstream_instance = upstream_instance
self.spack_dir = self.install_dir / "spack"
self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
self.logger = logger
@@ -45,23 +50,17 @@ class SpackManager(ABC):
pass
def create_fetch_spack_environment(self):
- env_dir = self.install_dir / self.env.path / self.env.env_name
+
if self.env.git_path:
- try:
- git_clone_repo(self.env.env_name, env_dir, self.env.git_path, logger=self.logger)
- except subprocess.CalledProcessError as e:
- self.logger.exception(f'Failed to clone repository: {self.env.env_name}: {e}')
- raise BashCommandException(f'Failed to clone repository: {self.env.env_name}: {e}')
+ git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
else:
- try:
- os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env create -d {self.env.path}/{self.env.env_name}',
- check=True, logger=self.logger)
- self.logger.debug(f"Created {self.env.env_name} spack environment")
- except subprocess.CalledProcessError as e:
- self.logger.error(f"Failed to create {self.env.env_name} spack environment")
- raise BashCommandException(f"Failed to create {self.env.env_name} spack environment")
+ os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+ check=True, logger=self.logger,
+ debug_msg=f"Created {self.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.env.env_name} spack environment",
+ exception=BashCommandException)
def setup_spack_env(self):
"""
@@ -94,46 +93,51 @@ class SpackManager(ABC):
def spack_repo_exists(self, repo_name: str) -> bool:
"""Check if the given Spack repository exists."""
if self.env is None:
- try:
- result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger)
- except subprocess.CalledProcessError:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if {repo_name} exists')
+ if result is None:
return False
else:
- try:
- result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger)
- except subprocess.CalledProcessError:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if repository {repo_name} was added')
+ if result is None:
return False
return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
+ def spack_env_exists(self):
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if environment {self.env.env_name} exists')
+ if result is None:
+ return False
+ return True
+
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
- repo_path = repo_path.resolve().as_posix()
- try:
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger)
- self.logger.debug(f"Added {repo_name} to spack environment {self.env.env_name}")
- except subprocess.CalledProcessError as e:
- self.logger.error(f"Failed to add {repo_name} to spack environment {self.env.env_name}")
- raise BashCommandException(f"Failed to add {repo_name} to spack environment {self.env.env_name}: {e}")
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ exception=BashCommandException)
def get_spack_installed_version(self):
- try:
- spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
- capture_output=True, text=True, check=True,
- logger=self.logger)
- spack_version = spack_version.stdout.strip().split()[0]
- self.logger.debug(f"Getting spack version: {spack_version}")
- return spack_version
- except subprocess.SubprocessError as e:
- self.logger.error(f"Error retrieving Spack version: {e}")
- return None
+ spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ capture_output=True, text=True, check=True,
+ logger=self.logger,
+ debug_msg=f"Getting spack version",
+ exception_msg=f"Error retrieving Spack version")
+ if spack_version:
+ return spack_version.stdout.strip().split()[0]
+ return None
def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
try:
@@ -163,8 +167,9 @@ class SpackManager(ABC):
bashrc.write(f"source {self.spack_setup_script}\n")
self.logger.info("Added Spack PATH to .bashrc")
if user:
- run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger)
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger)
+ run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
+ debug_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
self.logger.info("Spack install completed")
# Restart Bash after the installation ends
os.system("exec bash")
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index d229e345..48c500c3 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -3,6 +3,8 @@ import shutil
import subprocess
from pathlib import Path
+from esd.error_handling.exceptions import BashCommandException
+
def clean_up(dirs: list[str], logging, ignore_errors=True):
"""
@@ -22,21 +24,30 @@ def clean_up(dirs: list[str], logging, ignore_errors=True):
logging.info(f"{cleanup_dir} does not exist")
-def run_command(*args, logger: None, **kwargs):
- if logger is None:
- logger = logging.getLogger(__name__)
- logger.debug(f'{args}')
- return subprocess.run(args, **kwargs)
+def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+ exception=None, **kwargs):
+ try:
+ logger.debug(f'{debug_msg}: args: {args}')
+ return subprocess.run(args, **kwargs)
+ except subprocess.CalledProcessError as e:
+ if exception_msg is not None:
+ logger.error(f"{exception_msg}: {e}")
+ if exception is not None:
+ raise exception(f'{exception_msg} : {e}')
+ else:
+ return None
-def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging):
+def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
if not dir.exists():
run_command(
"git", "clone", "--depth", "1",
"-c", "advice.detachedHead=false",
"-c", "feature.manyFiles=true",
git_path, dir
- , check=True, logger=logger)
- logger.debug(f'Cloned repository {repo_name}')
+ , check=True, logger=logger,
+ debug_msg=f'Cloned repository {repo_name}',
+ exception_msg=f'Failed to clone repository: {repo_name}',
+ exception=BashCommandException)
else:
logger.debug(f'Repository {repo_name} already cloned.')
--
GitLab
From 53ddcc4184eb5929b8115a97315aa4c111feb9fc Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 15:18:03 +0200
Subject: [PATCH 03/21] esd-spack-installation: added spack env exceptions
---
esd/error_handling/exceptions.py | 5 +++++
esd/spack_manager/SpackManager.py | 26 +++++++++++++++++---------
2 files changed, 22 insertions(+), 9 deletions(-)
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 2acca54e..79f8051f 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -11,3 +11,8 @@ class BashCommandException(SpackException):
"""
To be thrown when an invalid input is received.
"""
+
+class NoSpackEnvironmentException(SpackException):
+ """
+ To be thrown when an invalid input is received.
+ """
\ No newline at end of file
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index f7bb00b4..340b1b95 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -2,7 +2,7 @@ import os
from abc import ABC, abstractmethod
from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
from esd.utils.utils import run_command, git_clone_repo
@@ -90,7 +90,7 @@ class SpackManager(ABC):
else:
self.logger.debug(f'Spack repository {repo.env_name} already added')
- def spack_repo_exists(self, repo_name: str) -> bool:
+ def spack_repo_exists(self, repo_name: str) -> bool | None:
"""Check if the given Spack repository exists."""
if self.env is None:
result = run_command("bash", "-c",
@@ -101,11 +101,15 @@ class SpackManager(ABC):
if result is None:
return False
else:
- result = run_command("bash", "-c",
+ if self.spack_env_exists():
+ result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
debug_msg=f'Checking if repository {repo_name} was added')
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
if result is None:
return False
return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
@@ -122,12 +126,16 @@ class SpackManager(ABC):
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
- exception=BashCommandException)
+ if self.spack_env_exists():
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ exception=BashCommandException)
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
--
GitLab
From dfa83ee626956f80711961f0bb4ffb3787caf65a Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 21:32:38 +0200
Subject: [PATCH 04/21] esd-spack-installation: added clean up after each test
---
dedal/tests/integration_tests/spack_install_test.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 34f68323..50e94044 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -4,18 +4,18 @@ from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCa
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+SPACK_VERSION = "0.22.0"
+
# we need this test to run first so that spack is installed only once
@pytest.mark.run(order=1)
def test_spack_install_scratch():
spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version="v0.21.1")
+ spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
def test_spack_install_buildcache():
spack_manager = SpackManagerBuildCache()
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
--
GitLab
From 4bb169ac555ec575b338efefec2826a41c47387a Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 23:09:24 +0200
Subject: [PATCH 05/21] esd-spack-installation: added concretization step and
tests
---
.../tests/integration_tests/spack_install_test.py | 2 +-
esd/error_handling/exceptions.py | 14 +++++++++++---
esd/spack_manager/factory/SpackManagerCreator.py | 10 +++++-----
esd/spack_manager/factory/SpackManagerScratch.py | 15 ++++++++++++++-
esd/utils/utils.py | 4 ++++
5 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 50e94044..9a32d5c7 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -6,7 +6,7 @@ from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
SPACK_VERSION = "0.22.0"
-# we need this test to run first so that spack is installed only once
+# we need this test to run first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
spack_manager = SpackManagerScratch()
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 79f8051f..d6de666b 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -7,12 +7,20 @@ class SpackException(Exception):
def __str__(self):
return self.message
+
class BashCommandException(SpackException):
"""
- To be thrown when an invalid input is received.
+ To be thrown when a bash command has failed
"""
+
class NoSpackEnvironmentException(SpackException):
"""
- To be thrown when an invalid input is received.
- """
\ No newline at end of file
+ To be thrown when an operation on a spack environment is executed without the environment being activated or existent
+ """
+
+
+class SpackConcertizeException(SpackException):
+ """
+ To be thrown when the spack concretization step fails
+ """
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
index 9728467f..6eb26a04 100644
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -1,3 +1,4 @@
+from esd.model.SpackModel import SpackModel
from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
@@ -5,10 +6,9 @@ from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
class SpackManagerCreator:
@staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
- upstream_instance: str):
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ return SpackManagerScratch(env, repos, upstream_instance, system_name)
elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
-
+ return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 5a79797c..6380d123 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -1,6 +1,8 @@
+from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
from esd.spack_manager.SpackManager import SpackManager
from esd.logger.logger_builder import get_logger
+from esd.utils.utils import run_command
class SpackManagerScratch(SpackManager):
@@ -9,7 +11,18 @@ class SpackManagerScratch(SpackManager):
super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
def concretize_spack_env(self, force=True):
- pass
+ force = '--force' if force else ''
+ if self.spack_env_exists():
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} $$ spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Concertization step for {self.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.env.env_name}',
+ exception=SpackConcertizeException)
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
pass
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 48c500c3..173347d0 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -51,3 +51,7 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
exception=BashCommandException)
else:
logger.debug(f'Repository {repo_name} already cloned.')
+
+
+def file_exists_and_not_empty(file: Path) -> bool:
+ return file.is_file() and file.stat().st_size > 0
--
GitLab
From 36a42d69240ddbd038f935bc072c844c062f0be8 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 6 Feb 2025 09:56:09 +0200
Subject: [PATCH 06/21] esd-spack-installation: fixed concretization step
---
esd/spack_manager/factory/SpackManagerScratch.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 6380d123..2ec22705 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -14,7 +14,7 @@ class SpackManagerScratch(SpackManager):
force = '--force' if force else ''
if self.spack_env_exists():
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} $$ spack concretize {force}',
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
check=True,
capture_output=True, text=True, logger=self.logger,
debug_msg=f'Concertization step for {self.env.env_name}',
--
GitLab
From 9abbf5f4ed004b1246cc23251611b43385bc0bcc Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 6 Feb 2025 10:10:33 +0200
Subject: [PATCH 07/21] esd-spack-installation: additional methods
---
.gitlab-ci.yml | 1 +
esd/error_handling/exceptions.py | 9 +++-
esd/spack_manager/SpackManager.py | 51 +++++++++++++++-------
esd/spack_manager/wrapper/__init__.py | 0
esd/spack_manager/wrapper/spack_wrapper.py | 15 +++++++
5 files changed, 59 insertions(+), 17 deletions(-)
create mode 100644 esd/spack_manager/wrapper/__init__.py
create mode 100644 esd/spack_manager/wrapper/spack_wrapper.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ce02907..7af0dd30 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,6 +5,7 @@ stages:
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
+
build-wheel:
stage: build
tags:
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index d6de666b..9d11b5fa 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -14,13 +14,18 @@ class BashCommandException(SpackException):
"""
-class NoSpackEnvironmentException(SpackException):
+class NoSpackEnvironmentException(BashCommandException):
"""
To be thrown when an operation on a spack environment is executed without the environment being activated or existent
"""
-class SpackConcertizeException(SpackException):
+class SpackConcertizeException(BashCommandException):
"""
To be thrown when the spack concretization step fails
"""
+
+class SpackInstallPackagesException(BashCommandException):
+ """
+ To be thrown when the spack fails to install spack packages
+ """
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index 340b1b95..a7f46c27 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -1,10 +1,13 @@
import os
+import re
from abc import ABC, abstractmethod
from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
+ SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
+from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
from esd.utils.utils import run_command, git_clone_repo
@@ -103,10 +106,10 @@ class SpackManager(ABC):
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -124,18 +127,35 @@ class SpackManager(ABC):
return False
return True
+ @no_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
- exception=BashCommandException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ exception=BashCommandException)
+
+ @no_spack_env
+ def get_compiler_version(self):
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ check=True, logger=self.logger,
+ capture_output=True, text=True,
+ debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ exception=BashCommandException)
+ # todo add error handling and tests
+ if result.stdout is None:
+ self.logger.debug('No gcc found for {self.env.env_name}')
+ return None
+
+ # Find the first occurrence of a GCC compiler using regex
+ match = re.search(r"gcc@([\d\.]+)", result.stdout)
+ gcc_version = match.group(1)
+ self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
@@ -147,6 +167,7 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
+
def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/spack_manager/wrapper/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/spack_manager/wrapper/spack_wrapper.py
new file mode 100644
index 00000000..d075a317
--- /dev/null
+++ b/esd/spack_manager/wrapper/spack_wrapper.py
@@ -0,0 +1,15 @@
+import functools
+
+from esd.error_handling.exceptions import NoSpackEnvironmentException
+
+
+def no_spack_env(method):
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.spack_env_exists():
+ return method(self, *args, **kwargs) # Call the method with 'self'
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+
+ return wrapper
--
GitLab
From c708d57a58db5333e07542ee773bae48f2e6431e Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Thu, 6 Feb 2025 18:14:49 +0200
Subject: [PATCH 08/21] esd-spack-installation: fixed passing access token to
tests; added log file for spack install step
---
.gitlab-ci.yml | 1 +
esd/spack_manager/SpackManager.py | 25 ++++++++++++++++------
esd/spack_manager/wrapper/spack_wrapper.py | 2 +-
3 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7af0dd30..d1b09e6a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,5 +39,6 @@ testing-pytest:
paths:
- test-results.xml
- .esd.log
+ - .generate_cache.log
expire_in: 1 week
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index a7f46c27..d534b30a 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -2,12 +2,13 @@ import os
import re
from abc import ABC, abstractmethod
from pathlib import Path
+from tabnanny import check
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
+from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
from esd.utils.utils import run_command, git_clone_repo
@@ -53,7 +54,6 @@ class SpackManager(ABC):
pass
def create_fetch_spack_environment(self):
-
if self.env.git_path:
git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
else:
@@ -127,7 +127,7 @@ class SpackManager(ABC):
return False
return True
- @no_spack_env
+ @check_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
@@ -137,7 +137,7 @@ class SpackManager(ABC):
exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
exception=BashCommandException)
- @no_spack_env
+ @check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
@@ -167,8 +167,21 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
-
- def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, fresh=False):
+ signed = '' if signed else '--no-check-signature'
+ fresh = '--fresh' if fresh else ''
+ with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack install --env {self.env.env_name} -v {signed} --j {jobs} {fresh}',
+ stdout=log_file,
+ capture_output=True, text=True, check=True,
+ logger=self.logger,
+ debug_msg=f"Installing spack packages for {self.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ exception=SpackInstallPackagesException)
+
+ def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/spack_manager/wrapper/spack_wrapper.py
index d075a317..c2f9c116 100644
--- a/esd/spack_manager/wrapper/spack_wrapper.py
+++ b/esd/spack_manager/wrapper/spack_wrapper.py
@@ -3,7 +3,7 @@ import functools
from esd.error_handling.exceptions import NoSpackEnvironmentException
-def no_spack_env(method):
+def check_spack_env(method):
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
if self.spack_env_exists():
--
GitLab
From a9253a23dc392c8786a5a23cb8e1eaa5b6bbf3de Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 7 Feb 2025 18:50:52 +0200
Subject: [PATCH 09/21] esd-spack-installation: spack install method and
additional tests
---
esd/spack_manager/SpackManager.py | 29 ++++++++++---------
.../factory/SpackManagerBuildCache.py | 3 --
.../factory/SpackManagerScratch.py | 3 --
3 files changed, 16 insertions(+), 19 deletions(-)
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index d534b30a..65a21d6e 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -1,5 +1,6 @@
import os
import re
+import subprocess
from abc import ABC, abstractmethod
from pathlib import Path
from tabnanny import check
@@ -49,10 +50,6 @@ class SpackManager(ABC):
def concretize_spack_env(self, force=True):
pass
- @abstractmethod
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
-
def create_fetch_spack_environment(self):
if self.env.git_path:
git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
@@ -168,18 +165,24 @@ class SpackManager(ABC):
return None
@check_spack_env
- def install_packages(self, jobs: int, signed=True, fresh=False):
+ def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
fresh = '--fresh' if fresh else ''
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ debug_msg=f"Installing spack packages for {self.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ exception=SpackInstallPackagesException)
with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack install --env {self.env.env_name} -v {signed} --j {jobs} {fresh}',
- stdout=log_file,
- capture_output=True, text=True, check=True,
- logger=self.logger,
- debug_msg=f"Installing spack packages for {self.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.env.env_name}",
- exception=SpackInstallPackagesException)
+ log_file.write(install_result.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(install_result.stderr)
+ return install_result
def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
try:
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
index 38151c6d..5b66f5c5 100644
--- a/esd/spack_manager/factory/SpackManagerBuildCache.py
+++ b/esd/spack_manager/factory/SpackManagerBuildCache.py
@@ -14,6 +14,3 @@ class SpackManagerBuildCache(SpackManager):
def concretize_spack_env(self, force=True):
pass
-
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 2ec22705..3dbc25f6 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -23,6 +23,3 @@ class SpackManagerScratch(SpackManager):
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
-
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
--
GitLab
From a55bf1aecbe962d0ba6f65615f7eb690a9688c62 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Mon, 10 Feb 2025 18:21:51 +0200
Subject: [PATCH 10/21] esd-spack-installation: refactoring
---
dedal/build_cache/BuildCacheManager.py | 50 +++++++++++++-------------
esd/spack_manager/SpackManager.py | 7 ++--
esd/utils/utils.py | 8 +++++
3 files changed, 35 insertions(+), 30 deletions(-)
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index e1bd6824..839ac524 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -13,47 +13,47 @@ class BuildCacheManager(BuildCacheManagerInterface):
"""
def __init__(self, auth_backend='basic', insecure=False):
- self.logger = get_logger(__name__, BuildCacheManager.__name__)
- self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.registry_project = os.environ.get("REGISTRY_PROJECT")
+ self._logger = get_logger(__name__, BuildCacheManager.__name__)
+ self._home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
+ self._registry_project = os.environ.get("REGISTRY_PROJECT")
self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
- self.registry_host = str(os.environ.get("REGISTRY_HOST"))
+ self._registry_host = str(os.environ.get("REGISTRY_HOST"))
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
- self.client.login(username=self._registry_username, password=self._registry_password)
- self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
+ self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend, insecure=insecure)
+ self._client.login(username=self._registry_username, password=self._registry_password)
+ self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/cache'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / out_dir
+ build_cache_path = self._home_path / out_dir
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- self.logger.error(f"Path {build_cache_path} not found.")
+ self._logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
- target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
+ target = f"{self._registry_host}/{self._registry_project}/cache:{str(sub_path.env_name)}"
try:
- self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
- self.client.push(
+ self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self._client.push(
files=[str(sub_path)],
target=target,
# save in manifest the relative path for reconstruction
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self.logger.info(f"Successfully pushed {sub_path.env_name}")
+ self._logger.info(f"Successfully pushed {sub_path.env_name}")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"An error occurred while pushing: {e}")
# todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
# clean_up([str(build_cache_path)], self.logger)
@@ -63,37 +63,37 @@ class BuildCacheManager(BuildCacheManagerInterface):
This method retrieves all tags from an OCI Registry
"""
try:
- return self.client.get_tags(self.oci_registry_path)
+ return self._client.get_tags(self._oci_registry_path)
except Exception as e:
- self.logger.error(f"Failed to list tags: {e}")
+ self._logger.error(f"Failed to list tags: {e}")
return None
def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / in_dir
+ build_cache_path = self._home_path / in_dir
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
+ ref = f"{self._registry_host}/{self._registry_project}/cache:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
+ self._client.get_manifest(f'{self._registry_host}/{self._registry_project}/cache:{tag}')[
'annotations'][
'path']
try:
- self.client.pull(
+ self._client.pull(
ref,
# missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- self.logger.info(f"Successfully pulled artifact {tag}.")
+ self._logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"Failed to pull artifact {tag} : {e}")
def delete(self):
@@ -106,8 +106,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
tags = self.list_tags()
if tags is not None:
try:
- self.client.delete_tags(self.oci_registry_path, tags)
- self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
+ self._client.delete_tags(self._oci_registry_path, tags)
+ self._logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- self.logger.error(
+ self._logger.error(
f"Failed to delete artifacts: {e}")
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index 65a21d6e..bf381c39 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -10,7 +10,7 @@ from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironme
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo
+from esd.utils.utils import run_command, git_clone_repo, log_command
class SpackManager(ABC):
@@ -178,10 +178,7 @@ class SpackManager(ABC):
debug_msg=f"Installing spack packages for {self.env.env_name}",
exception_msg=f"Error installing spack packages for {self.env.env_name}",
exception=SpackInstallPackagesException)
- with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
- log_file.write(install_result.stdout)
- log_file.write("\n--- STDERR ---\n")
- log_file.write(install_result.stderr)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 173347d0..3ce70f25 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -1,4 +1,5 @@
import logging
+import os
import shutil
import subprocess
from pathlib import Path
@@ -55,3 +56,10 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
def file_exists_and_not_empty(file: Path) -> bool:
return file.is_file() and file.stat().st_size > 0
+
+
+def log_command(results, log_file: str):
+ with open(log_file, "w") as log_file:
+ log_file.write(results.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(results.stderr)
--
GitLab
From 4374fe5c81e16712da831b1a5ede941120ce245e Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 11 Feb 2025 19:23:33 +0200
Subject: [PATCH 11/21] esd-spack-installation: major refactoring; fixed bug on
updating env vars in .bashrc
---
dedal/build_cache/BuildCacheManager.py | 34 +++--
.../integration_tests/spack_install_test.py | 21 +--
esd/configuration/SpackConfig.py | 25 ++++
.../__init__.py | 0
.../enums => error_handling}/__init__.py | 0
.../{SpackModel.py => SpackDescriptor.py} | 5 +-
.../SpackOperation.py} | 129 +++++++++---------
esd/spack_factory/SpackOperationCreator.py | 14 ++
esd/spack_factory/SpackOperationUseCache.py | 19 +++
.../factory => spack_factory}/__init__.py | 0
esd/spack_manager/enums/SpackManagerEnum.py | 6 -
.../factory/SpackManagerBuildCache.py | 16 ---
.../factory/SpackManagerCreator.py | 14 --
.../factory/SpackManagerScratch.py | 25 ----
esd/utils/utils.py | 39 ++++--
esd/{spack_manager => }/wrapper/__init__.py | 0
.../wrapper/spack_wrapper.py | 0
17 files changed, 183 insertions(+), 164 deletions(-)
create mode 100644 esd/configuration/SpackConfig.py
rename esd/{spack_manager => configuration}/__init__.py (100%)
rename esd/{spack_manager/enums => error_handling}/__init__.py (100%)
rename esd/model/{SpackModel.py => SpackDescriptor.py} (57%)
rename esd/{spack_manager/SpackManager.py => spack_factory/SpackOperation.py} (62%)
create mode 100644 esd/spack_factory/SpackOperationCreator.py
create mode 100644 esd/spack_factory/SpackOperationUseCache.py
rename esd/{spack_manager/factory => spack_factory}/__init__.py (100%)
delete mode 100644 esd/spack_manager/enums/SpackManagerEnum.py
delete mode 100644 esd/spack_manager/factory/SpackManagerBuildCache.py
delete mode 100644 esd/spack_manager/factory/SpackManagerCreator.py
delete mode 100644 esd/spack_manager/factory/SpackManagerScratch.py
rename esd/{spack_manager => }/wrapper/__init__.py (100%)
rename esd/{spack_manager => }/wrapper/spack_wrapper.py (100%)
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index 839ac524..cccb5846 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -12,36 +12,39 @@ class BuildCacheManager(BuildCacheManagerInterface):
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', insecure=False):
+ def __init__(self, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
+ auth_backend='basic',
+ insecure=False):
self._logger = get_logger(__name__, BuildCacheManager.__name__)
- self._home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self._registry_project = os.environ.get("REGISTRY_PROJECT")
+ self._registry_project = registry_project
- self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
- self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
+ self._registry_username = registry_username
+ self._registry_password = registry_password
- self._registry_host = str(os.environ.get("REGISTRY_HOST"))
+ self._registry_host = registry_host
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend, insecure=insecure)
+ self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend,
+ insecure=insecure)
self._client.login(username=self._registry_username, password=self._registry_password)
- self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/cache'
+ self.cache_version = cache_version
+ self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/{self.cache_version}'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self._home_path / out_dir
+ build_cache_path = out_dir.resolve()
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
self._logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
- target = f"{self._registry_host}/{self._registry_project}/cache:{str(sub_path.env_name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
+ target = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{str(sub_path.name)}"
try:
self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
self._client.push(
@@ -51,7 +54,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self._logger.info(f"Successfully pushed {sub_path.env_name}")
+ self._logger.info(f"Successfully pushed {sub_path.name}")
except Exception as e:
self._logger.error(
f"An error occurred while pushing: {e}")
@@ -72,16 +75,17 @@ class BuildCacheManager(BuildCacheManagerInterface):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self._home_path / in_dir
+ build_cache_path = in_dir.resolve()
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self._registry_host}/{self._registry_project}/cache:{tag}"
+ ref = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self._client.get_manifest(f'{self._registry_host}/{self._registry_project}/cache:{tag}')[
+ self._client.get_manifest(
+ f'{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}')[
'annotations'][
'path']
try:
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 9a32d5c7..28f8268e 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -1,21 +1,12 @@
import pytest
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.tests.testing_variables import SPACK_VERSION
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-SPACK_VERSION = "0.22.0"
-
-# we need this test to run first so that spack is installed only once for all the tests
+# run this test first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
- spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
- installed_spack_version = spack_manager.get_spack_installed_version()
- assert SPACK_VERSION == installed_spack_version
-
-
-def test_spack_install_buildcache():
- spack_manager = SpackManagerBuildCache()
- installed_spack_version = spack_manager.get_spack_installed_version()
+ spack_operation = SpackOperation()
+ spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
+ installed_spack_version = spack_operation.get_spack_installed_version()
assert SPACK_VERSION == installed_spack_version
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
new file mode 100644
index 00000000..93a2e874
--- /dev/null
+++ b/esd/configuration/SpackConfig.py
@@ -0,0 +1,25 @@
+import os
+from pathlib import Path
+from esd.model import SpackDescriptor
+
+
+class SpackConfig:
+ def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
+ install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
+ concretization_dir: Path = None, buildcache_dir: Path = None):
+ self.env = env
+ if repos is None:
+ self.repos = []
+ else:
+ self.repos = repos
+ self.install_dir = install_dir
+ self.upstream_instance = upstream_instance
+ self.system_name = system_name
+ self.concretization_dir = concretization_dir
+ self.buildcache_dir = buildcache_dir
+
+ def add_repo(self, repo: SpackDescriptor):
+ if self.repos is None:
+ self.repos = []
+ else:
+ self.repos.append(repo)
diff --git a/esd/spack_manager/__init__.py b/esd/configuration/__init__.py
similarity index 100%
rename from esd/spack_manager/__init__.py
rename to esd/configuration/__init__.py
diff --git a/esd/spack_manager/enums/__init__.py b/esd/error_handling/__init__.py
similarity index 100%
rename from esd/spack_manager/enums/__init__.py
rename to esd/error_handling/__init__.py
diff --git a/esd/model/SpackModel.py b/esd/model/SpackDescriptor.py
similarity index 57%
rename from esd/model/SpackModel.py
rename to esd/model/SpackDescriptor.py
index 4b065dba..70e484fb 100644
--- a/esd/model/SpackModel.py
+++ b/esd/model/SpackDescriptor.py
@@ -1,12 +1,13 @@
+import os
from pathlib import Path
-class SpackModel:
+class SpackDescriptor:
""""
Provides details about the spack environment
"""
- def __init__(self, env_name: str, path: Path, git_path: str = None):
+ def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
self.env_name = env_name
self.path = path
self.git_path = git_path
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_factory/SpackOperation.py
similarity index 62%
rename from esd/spack_manager/SpackManager.py
rename to esd/spack_factory/SpackOperation.py
index bf381c39..29f44f49 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -3,63 +3,57 @@ import re
import subprocess
from abc import ABC, abstractmethod
from pathlib import Path
-from tabnanny import check
-
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException
+ SpackInstallPackagesException, SpackConcertizeException
from esd.logger.logger_builder import get_logger
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo, log_command
+from esd.configuration.SpackConfig import SpackConfig
+from esd.tests.testing_variables import SPACK_VERSION
+from esd.wrapper.spack_wrapper import check_spack_env
+from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-class SpackManager(ABC):
+class SpackOperation(ABC):
"""
This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
Attributes:
-----------
- env : SpackModel
+ env : SpackDescriptor
spack environment details
- repos : list[SpackModel]
+ repos : list[SpackDescriptor]
upstream_instance : str
path to Spack instance to use as upstream (optional)
"""
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
- if repos is None:
- repos = []
- self.repos = repos
- self.env = env
- self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
- self.install_dir.mkdir(parents=True, exist_ok=True)
- self.env_path = None
- if self.env and self.env.path:
- self.env.path = self.env.path.resolve()
- self.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = self.env.path / self.env.env_name
- self.upstream_instance = upstream_instance
- self.spack_dir = self.install_dir / "spack"
- self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
+ self.spack_config = spack_config
+ self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.spack_config.install_dir / 'spack'
+ self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
self.logger = logger
- self.system_name = system_name
+ if self.spack_config.env and spack_config.env.path:
+ self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
@abstractmethod
def concretize_spack_env(self, force=True):
pass
def create_fetch_spack_environment(self):
- if self.env.git_path:
- git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
+ if self.spack_config.env.git_path:
+ git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
+ self.spack_config.env.git_path,
+ logger=self.logger)
else:
- os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
check=True, logger=self.logger,
- debug_msg=f"Created {self.env.env_name} spack environment",
- exception_msg=f"Failed to create {self.env.env_name} spack environment",
+ info_msg=f"Created {self.spack_config.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
exception=BashCommandException)
def setup_spack_env(self):
@@ -67,22 +61,20 @@ class SpackManager(ABC):
This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
"""
bashrc_path = os.path.expanduser("~/.bashrc")
- if self.system_name:
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
- os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_config.system_name:
+ set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
+ os.environ['SYSTEMNAME'] = self.spack_config.system_name
if self.spack_dir.exists() and self.spack_dir.is_dir():
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
- bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
else:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
# Restart the bash after adding environment variables
self.create_fetch_spack_environment()
- if self.install_dir.exists():
- for repo in self.repos:
- repo_dir = self.install_dir / repo.path / repo.env_name
+ if self.spack_config.install_dir.exists():
+ for repo in self.spack_config.repos:
+ repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
if not self.spack_repo_exists(repo.env_name):
self.add_spack_repo(repo.path, repo.env_name)
@@ -92,21 +84,21 @@ class SpackManager(ABC):
def spack_repo_exists(self, repo_name: str) -> bool | None:
"""Check if the given Spack repository exists."""
- if self.env is None:
+ if self.spack_config.env is None:
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if {repo_name} exists')
+ info_msg=f'Checking if {repo_name} exists')
if result is None:
return False
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ f'{self.spack_command_on_env} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ info_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -116,10 +108,10 @@ class SpackManager(ABC):
def spack_env_exists(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+ self.spack_command_on_env,
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if environment {self.env.env_name} exists')
+ info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
if result is None:
return False
return True
@@ -128,20 +120,20 @@ class SpackManager(ABC):
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
exception=BashCommandException)
@check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ f'{self.spack_command_on_env} && spack compiler list',
check=True, logger=self.logger,
capture_output=True, text=True,
- debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
- exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
exception=BashCommandException)
# todo add error handling and tests
if result.stdout is None:
@@ -151,37 +143,48 @@ class SpackManager(ABC):
# Find the first occurrence of a GCC compiler using regex
match = re.search(r"gcc@([\d\.]+)", result.stdout)
gcc_version = match.group(1)
- self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
capture_output=True, text=True, check=True,
logger=self.logger,
- debug_msg=f"Getting spack version",
+ info_msg=f"Getting spack version",
exception_msg=f"Error retrieving Spack version")
if spack_version:
return spack_version.stdout.strip().split()[0]
return None
+ @check_spack_env
+ def concretize_spack_env(self, force=True):
+ force = '--force' if force else ''
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Concertization step for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
+ exception=SpackConcertizeException)
+
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
fresh = '--fresh' if fresh else ''
debug = '--debug' if debug else ''
install_result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
logger=self.logger,
- debug_msg=f"Installing spack packages for {self.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
exception=SpackInstallPackagesException)
log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
- def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
@@ -210,18 +213,18 @@ class SpackManager(ABC):
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
- debug_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
+ info_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
self.logger.info("Spack install completed")
# Restart Bash after the installation ends
os.system("exec bash")
# Configure upstream Spack instance if specified
- if self.upstream_instance:
+ if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
with open(upstreams_yaml_path, "w") as file:
file.write(f"""upstreams:
upstream-spack-instance:
- install_tree: {self.upstream_instance}/spack/opt/spack
+ install_tree: {self.spack_config.upstream_instance}/spack/opt/spack
""")
self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_factory/SpackOperationCreator.py b/esd/spack_factory/SpackOperationCreator.py
new file mode 100644
index 00000000..8369c5ca
--- /dev/null
+++ b/esd/spack_factory/SpackOperationCreator.py
@@ -0,0 +1,14 @@
+from esd.configuration.SpackConfig import SpackConfig
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+
+
+class SpackOperationCreator:
+ @staticmethod
+ def get_spack_operator(spack_config: SpackConfig = None):
+ if spack_config is None:
+ return SpackOperation(SpackConfig())
+ elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
+ return SpackOperation(spack_config)
+ else:
+ return SpackOperationUseCache(spack_config)
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
new file mode 100644
index 00000000..15a3822f
--- /dev/null
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -0,0 +1,19 @@
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationUseCache(SpackOperation):
+ """
+ This class uses caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ super().__init__(spack_config, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_factory/__init__.py
similarity index 100%
rename from esd/spack_manager/factory/__init__.py
rename to esd/spack_factory/__init__.py
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
deleted file mode 100644
index a2435839..00000000
--- a/esd/spack_manager/enums/SpackManagerEnum.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from enum import Enum
-
-
-class SpackManagerEnum(Enum):
- FROM_SCRATCH = "from_scratch",
- FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
deleted file mode 100644
index 5b66f5c5..00000000
--- a/esd/spack_manager/factory/SpackManagerBuildCache.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-
-
-class SpackManagerBuildCache(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- def concretize_spack_env(self, force=True):
- pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
deleted file mode 100644
index 6eb26a04..00000000
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-
-class SpackManagerCreator:
- @staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env, repos, upstream_instance, system_name)
- elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
deleted file mode 100644
index 3dbc25f6..00000000
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-from esd.utils.utils import run_command
-
-
-class SpackManagerScratch(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def concretize_spack_env(self, force=True):
- force = '--force' if force else ''
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Concertization step for {self.env.env_name}',
- exception_msg=f'Failed the concertization step for {self.env.env_name}',
- exception=SpackConcertizeException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 3ce70f25..033cbc54 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -5,30 +5,31 @@ import subprocess
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException
+import re
-def clean_up(dirs: list[str], logging, ignore_errors=True):
+def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
for cleanup_dir in dirs:
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
- logging.info(f"Removing {cleanup_dir}")
+ logger.info(f"Removing {cleanup_dir}")
try:
shutil.rmtree(Path(cleanup_dir))
except OSError as e:
- logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ logger.error(f"Failed to remove {cleanup_dir}: {e}")
if not ignore_errors:
raise e
else:
- logging.info(f"{cleanup_dir} does not exist")
+ logger.info(f"{cleanup_dir} does not exist")
-def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
exception=None, **kwargs):
try:
- logger.debug(f'{debug_msg}: args: {args}')
+ logger.info(f'{info_msg}: args: {args}')
return subprocess.run(args, **kwargs)
except subprocess.CalledProcessError as e:
if exception_msg is not None:
@@ -47,11 +48,11 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
"-c", "feature.manyFiles=true",
git_path, dir
, check=True, logger=logger,
- debug_msg=f'Cloned repository {repo_name}',
+ info_msg=f'Cloned repository {repo_name}',
exception_msg=f'Failed to clone repository: {repo_name}',
exception=BashCommandException)
else:
- logger.debug(f'Repository {repo_name} already cloned.')
+ logger.info(f'Repository {repo_name} already cloned.')
def file_exists_and_not_empty(file: Path) -> bool:
@@ -63,3 +64,25 @@ def log_command(results, log_file: str):
log_file.write(results.stdout)
log_file.write("\n--- STDERR ---\n")
log_file.write(results.stderr)
+
+
+def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
+ logger: logging = logging.getLogger(__name__)):
+ """Update or add an environment variable in ~/.bashrc."""
+ with open(bashrc_path, "r") as file:
+ lines = file.readlines()
+ pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
+ updated = False
+ # Modify the existing variable if found
+ for i, line in enumerate(lines):
+ if pattern.match(line):
+ lines[i] = f'export {var_name}={value}\n'
+ updated = True
+ break
+ if not updated:
+ lines.append(f'\nexport {var_name}={value}\n')
+ logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
+ else:
+ logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
+ with open(bashrc_path, "w") as file:
+ file.writelines(lines)
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/wrapper/__init__.py
similarity index 100%
rename from esd/spack_manager/wrapper/__init__.py
rename to esd/wrapper/__init__.py
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/wrapper/spack_wrapper.py
similarity index 100%
rename from esd/spack_manager/wrapper/spack_wrapper.py
rename to esd/wrapper/spack_wrapper.py
--
GitLab
From fdadeab5abeefd42b028cfd780d46961c4255ee0 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 14 Feb 2025 11:28:46 +0200
Subject: [PATCH 12/21] esd-spack-installation: added additional spack commands
---
esd/spack_factory/SpackOperation.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/esd/spack_factory/SpackOperation.py b/esd/spack_factory/SpackOperation.py
index 29f44f49..cbf7b5be 100644
--- a/esd/spack_factory/SpackOperation.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -112,6 +112,7 @@ class SpackOperation(ABC):
check=True,
capture_output=True, text=True, logger=self.logger,
info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
+ print(result)
if result is None:
return False
return True
--
GitLab
From 2a6f1f8c16589fbde02d94b68774d90f16c70701 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 14 Feb 2025 11:28:46 +0200
Subject: [PATCH 13/21] esd-spack-installation: added additional spack commands
---
esd/spack_factory/SpackOperation.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/esd/spack_factory/SpackOperation.py b/esd/spack_factory/SpackOperation.py
index cbf7b5be..29f44f49 100644
--- a/esd/spack_factory/SpackOperation.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -112,7 +112,6 @@ class SpackOperation(ABC):
check=True,
capture_output=True, text=True, logger=self.logger,
info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
- print(result)
if result is None:
return False
return True
--
GitLab
From 5997975485703f8ebd7e6eb13ba5e412836d4069 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 11 Feb 2025 19:23:33 +0200
Subject: [PATCH 14/21] esd-spack-installation: major refactoring; fixed bug on
updating env vars in .bashrc
---
esd/spack_factory/SpackOperationUserCache.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
create mode 100644 esd/spack_factory/SpackOperationUserCache.py
diff --git a/esd/spack_factory/SpackOperationUserCache.py b/esd/spack_factory/SpackOperationUserCache.py
new file mode 100644
index 00000000..bf4e8465
--- /dev/null
+++ b/esd/spack_factory/SpackOperationUserCache.py
@@ -0,0 +1,15 @@
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationUseCache(SpackOperation):
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ super().__init__(spack_config, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
--
GitLab
From 17a4d51b23433501abbc41566f96d133d83d3185 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 11 Feb 2025 19:23:33 +0200
Subject: [PATCH 15/21] esd-spack-installation: major refactoring; fixed bug on
updating env vars in .bashrc
---
esd/spack_factory/SpackOperationBuildCache.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
create mode 100644 esd/spack_factory/SpackOperationBuildCache.py
diff --git a/esd/spack_factory/SpackOperationBuildCache.py b/esd/spack_factory/SpackOperationBuildCache.py
new file mode 100644
index 00000000..46ff64b5
--- /dev/null
+++ b/esd/spack_factory/SpackOperationBuildCache.py
@@ -0,0 +1,15 @@
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationBuildCache(SpackOperation):
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ super().__init__(spack_config, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
--
GitLab
From 71712f5aa27fddf1fce97335069d47efc4ce23a6 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 12 Feb 2025 13:45:22 +0200
Subject: [PATCH 16/21] esd-concretize-buildcache: implemented concretization
with cache and install methods for spack create cache
---
.../integration_tests/spack_install_test.py | 10 ++--
dedal/tests/spack_from_cache_test.py | 38 +++++++++++++
esd/spack_factory/SpackOperation.py | 2 +-
esd/spack_factory/SpackOperationBuildCache.py | 15 -----
.../SpackOperationCreateCache.py | 53 ++++++++++++++++++
esd/spack_factory/SpackOperationCreator.py | 3 +
esd/spack_factory/SpackOperationUseCache.py | 55 ++++++++++++++++++-
esd/spack_factory/SpackOperationUserCache.py | 15 -----
esd/utils/utils.py | 48 ++++++++++++++++
esd/utils/variables.py | 5 ++
10 files changed, 205 insertions(+), 39 deletions(-)
create mode 100644 dedal/tests/spack_from_cache_test.py
delete mode 100644 esd/spack_factory/SpackOperationBuildCache.py
create mode 100644 esd/spack_factory/SpackOperationCreateCache.py
delete mode 100644 esd/spack_factory/SpackOperationUserCache.py
create mode 100644 esd/utils/variables.py
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 28f8268e..2373c098 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -1,12 +1,12 @@
-import pytest
+from esd.configuration.SpackConfig import SpackConfig
from esd.spack_factory.SpackOperation import SpackOperation
from esd.tests.testing_variables import SPACK_VERSION
-# run this test first so that spack is installed only once for all the tests
-@pytest.mark.run(order=1)
-def test_spack_install_scratch():
- spack_operation = SpackOperation()
+def test_spack_install_scratch(tmp_path):
+ install_dir = tmp_path
+ spack_config = SpackConfig(install_dir=install_dir)
+ spack_operation = SpackOperation(spack_config)
spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
installed_spack_version = spack_operation.get_spack_installed_version()
assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/spack_from_cache_test.py b/dedal/tests/spack_from_cache_test.py
new file mode 100644
index 00000000..293a58d1
--- /dev/null
+++ b/dedal/tests/spack_from_cache_test.py
@@ -0,0 +1,38 @@
+from esd.configuration.SpackConfig import SpackConfig
+from esd.model.SpackDescriptor import SpackDescriptor
+from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
+from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from esd.utils.utils import file_exists_and_not_empty
+from esd.utils.variables import test_spack_env_git, ebrains_spack_builds_git
+
+
+def test_spack_from_cache_concretize(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.concretize_spack_env() == False
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_cache_install(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.concretize_spack_env() == False
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+ install_result = spack_operation.install_packages(jobs=2, signed=False, debug=False)
+ assert install_result.returncode == 0
diff --git a/esd/spack_factory/SpackOperation.py b/esd/spack_factory/SpackOperation.py
index 29f44f49..3a1a5d9c 100644
--- a/esd/spack_factory/SpackOperation.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -35,7 +35,7 @@ class SpackOperation(ABC):
if self.spack_config.env and spack_config.env.path:
self.spack_config.env.path = spack_config.env.path.resolve()
self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = spack_config.env.path / spack_config.env.env_name
+ self.env_path: Path = spack_config.env.path / spack_config.env.env_name
self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
@abstractmethod
diff --git a/esd/spack_factory/SpackOperationBuildCache.py b/esd/spack_factory/SpackOperationBuildCache.py
deleted file mode 100644
index 46ff64b5..00000000
--- a/esd/spack_factory/SpackOperationBuildCache.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from esd.logger.logger_builder import get_logger
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.configuration.SpackConfig import SpackConfig
-
-
-class SpackOperationBuildCache(SpackOperation):
- def __init__(self, spack_config: SpackConfig = SpackConfig()):
- super().__init__(spack_config, logger=get_logger(__name__))
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- def concretize_spack_env(self, force=True):
- pass
diff --git a/esd/spack_factory/SpackOperationCreateCache.py b/esd/spack_factory/SpackOperationCreateCache.py
new file mode 100644
index 00000000..02cef258
--- /dev/null
+++ b/esd/spack_factory/SpackOperationCreateCache.py
@@ -0,0 +1,53 @@
+import os
+
+from esd.error_handling.exceptions import NoSpackEnvironmentException
+
+from esd.utils.utils import copy_to_tmp, copy_file
+from esd.wrapper.spack_wrapper import check_spack_env
+from esd.build_cache.BuildCacheManager import BuildCacheManager
+from esd.configuration.SpackConfig import SpackConfig
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+
+
+class SpackOperationCreateCache(SpackOperation):
+ """
+ This class creates caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
+ cache_version_build='cache'):
+ super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=cache_version_build)
+
+ @check_spack_env
+ def concretize_spack_env(self):
+ super().concretize_spack_env(force=True)
+ dependency_path = self.spack_config.env.path / self.spack_config.env.env_name / 'spack.lock'
+ copy_file(dependency_path, self.spack_config.concretization_dir, logger=self.logger)
+ self.cache_dependency.upload(self.spack_config.concretization_dir)
+ self.logger.info(f'Created new spack concretization for create cache: {self.spack_config.env.env_name}')
+
+ @check_spack_env
+ def install_packages(self, jobs: int, debug=False):
+ signed = False
+ if self.spack_config.gpg:
+ signed = True
+ self.create_gpg_keys()
+ self.logger.info(f'Created GPG keys for {self.spack_config.env.name}')
+ self.add_mirror('local_cache', self.spack_config.buildcache_dir, signed=signed, autopush=True,
+ global_mirror=False)
+ self.logger.info(f'Added mirror for {self.spack_config.env.name}')
+ super().install_packages(jobs=jobs, signed=signed, debug=debug, fresh=True)
+ self.logger.info(f'Installed spack packages for {self.spack_config.env.name}')
+ self.build_cache.upload(self.spack_config.buildcache_dir)
+ self.logger.info(f'Pushed spack packages for {self.spack_config.env.name}')
diff --git a/esd/spack_factory/SpackOperationCreator.py b/esd/spack_factory/SpackOperationCreator.py
index 8369c5ca..1fceb7b9 100644
--- a/esd/spack_factory/SpackOperationCreator.py
+++ b/esd/spack_factory/SpackOperationCreator.py
@@ -1,5 +1,6 @@
from esd.configuration.SpackConfig import SpackConfig
from esd.spack_factory.SpackOperation import SpackOperation
+from esd.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
@@ -10,5 +11,7 @@ class SpackOperationCreator:
return SpackOperation(SpackConfig())
elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
return SpackOperation(spack_config)
+ elif spack_config.concretization_dir and spack_config.buildcache_dir:
+ return SpackOperationCreateCache(spack_config)
else:
return SpackOperationUseCache(spack_config)
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
index 15a3822f..3ec78fb0 100644
--- a/esd/spack_factory/SpackOperationUseCache.py
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -1,6 +1,14 @@
+import os
+import subprocess
+from pathlib import Path
+
+from esd.build_cache.BuildCacheManager import BuildCacheManager
+from esd.error_handling.exceptions import SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.spack_factory.SpackOperation import SpackOperation
from esd.configuration.SpackConfig import SpackConfig
+from esd.utils.utils import file_exists_and_not_empty, run_command, log_command, copy_to_tmp, copy_file
+from esd.wrapper.spack_wrapper import check_spack_env
class SpackOperationUseCache(SpackOperation):
@@ -8,12 +16,53 @@ class SpackOperationUseCache(SpackOperation):
This class uses caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
+ cache_version_build='cache'):
super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=cache_version_build)
def setup_spack_env(self):
super().setup_spack_env()
# todo add buildcache to the spack environment
- def concretize_spack_env(self, force=True):
- pass
+ @check_spack_env
+ def concretize_spack_env(self, upload=False):
+ concretization_redo = False
+ self.cache_dependency.download(self.spack_config.concretization_dir)
+ if file_exists_and_not_empty(self.spack_config.concretization_dir / 'spack.lock'):
+ concretization_file_path = self.env_path / 'spack.lock'
+ copy_file(self.spack_config.concretization_dir / 'spack.lock', self.env_path)
+ # redo the concretization step if spack.lock file was not downloaded from the cache
+ if not file_exists_and_not_empty(concretization_file_path):
+ super().concretize_spack_env(force=True)
+ concretization_redo = True
+ else:
+ # redo the concretization step if spack.lock file was not downloaded from the cache
+ super().concretize_spack_env(force=True)
+ concretization_redo = True
+ return concretization_redo
+
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, debug=False):
+ signed = '' if signed else '--no-check-signature'
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v --reuse {signed} --j {jobs}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ debug_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
+ exception=SpackInstallPackagesException)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
+ return install_result
diff --git a/esd/spack_factory/SpackOperationUserCache.py b/esd/spack_factory/SpackOperationUserCache.py
deleted file mode 100644
index bf4e8465..00000000
--- a/esd/spack_factory/SpackOperationUserCache.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from esd.logger.logger_builder import get_logger
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.configuration.SpackConfig import SpackConfig
-
-
-class SpackOperationUseCache(SpackOperation):
- def __init__(self, spack_config: SpackConfig = SpackConfig()):
- super().__init__(spack_config, logger=get_logger(__name__))
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- def concretize_spack_env(self, force=True):
- pass
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 033cbc54..f9a7d0e8 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -2,6 +2,7 @@ import logging
import os
import shutil
import subprocess
+import tempfile
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException
@@ -66,6 +67,21 @@ def log_command(results, log_file: str):
log_file.write(results.stderr)
+def copy_to_tmp(file_path: Path) -> Path:
+ """
+ Creates a temporary directory and copies the given file into it.
+
+ :param file_path: Path to the file that needs to be copied.
+ :return: Path to the copied file inside the temporary directory.
+ """
+ if not file_path.is_file():
+ raise FileNotFoundError(f"File not found: {file_path}")
+ tmp_dir = Path(tempfile.mkdtemp())
+ tmp_file_path = tmp_dir / file_path.name
+ shutil.copy(file_path, tmp_file_path)
+ return tmp_file_path
+
+
def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
logger: logging = logging.getLogger(__name__)):
"""Update or add an environment variable in ~/.bashrc."""
@@ -86,3 +102,35 @@ def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.ex
logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
with open(bashrc_path, "w") as file:
file.writelines(lines)
+
+
+def copy_file(src: Path, dst: Path, logger: logging = logging.getLogger(__name__)) -> None:
+ """
+ Copy a file from src to dest.
+ """
+ if not os.path.exists(src):
+ raise FileNotFoundError(f"Source file '{src}' does not exist.")
+ src.resolve().as_posix()
+ dst.resolve().as_posix()
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ shutil.copy2(src, dst)
+ logger.debug(f"File copied from '{src}' to '{dst}'")
+
+
+def delete_file(file_path: str, logger: logging = logging.getLogger(__name__)) -> bool:
+ """
+ Deletes a file at the given path. Returns True if successful, False if the file doesn't exist.
+ """
+ try:
+ os.remove(file_path)
+ logger.debug(f"File '{file_path}' deleted.")
+ return True
+ except FileNotFoundError:
+ logger.error(f"File not found: {file_path}")
+ return False
+ except PermissionError:
+ logger.error(f"Permission denied: {file_path}")
+ return False
+ except Exception as e:
+ logger.error(f"Error deleting file {file_path}: {e}")
+ return False
diff --git a/esd/utils/variables.py b/esd/utils/variables.py
new file mode 100644
index 00000000..553ccf97
--- /dev/null
+++ b/esd/utils/variables.py
@@ -0,0 +1,5 @@
+import os
+
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
--
GitLab
From 19f4b25ef95edf88c9e6ebb8613024752826aa98 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Mon, 17 Feb 2025 16:44:44 +0200
Subject: [PATCH 17/21] esd-concretize-buildcache: added tests for spack create
cashing
esd-concretize-buildcache
---
.gitlab-ci.yml | 2 +-
dedal/cli/SpackManagerAPI.py | 2 +
dedal/tests/spack_create_cache.py | 52 +++++++++++++++++++
esd/bll/SpackManager.py | 0
esd/bll/__init__.py | 0
.../SpackOperationCreateCache.py | 11 ++--
6 files changed, 60 insertions(+), 7 deletions(-)
create mode 100644 dedal/cli/SpackManagerAPI.py
create mode 100644 dedal/tests/spack_create_cache.py
create mode 100644 esd/bll/SpackManager.py
create mode 100644 esd/bll/__init__.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d1b09e6a..b3aaf6e6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,7 +31,7 @@ testing-pytest:
- chmod +x esd/utils/bootstrap.sh
- ./esd/utils/bootstrap.sh
- pip install .
- - pytest ./esd/tests/ -s --junitxml=test-results.xml
+ - pytest ./esd/tests/ -s --junitxml=test-results.xml --ignore=./esd/tests/spack_create_cache.py
artifacts:
when: always
reports:
diff --git a/dedal/cli/SpackManagerAPI.py b/dedal/cli/SpackManagerAPI.py
new file mode 100644
index 00000000..4f883a1b
--- /dev/null
+++ b/dedal/cli/SpackManagerAPI.py
@@ -0,0 +1,2 @@
+class SpackManagerAPI:
+ pass
diff --git a/dedal/tests/spack_create_cache.py b/dedal/tests/spack_create_cache.py
new file mode 100644
index 00000000..d22801c1
--- /dev/null
+++ b/dedal/tests/spack_create_cache.py
@@ -0,0 +1,52 @@
+from pathlib import Path
+
+from esd.configuration.GpgConfig import GpgConfig
+from esd.configuration.SpackConfig import SpackConfig
+
+from esd.model.SpackDescriptor import SpackDescriptor
+from esd.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
+from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
+from esd.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
+
+"""
+Before running those tests, the repositories where the caching is stored must be cleared after each run.
+Ebrains Harbour does not support deletion via API, so the clean up must be done manually
+"""
+
+
+def test_spack_create_cache_concretization(tmp_path):
+ install_dir = tmp_path
+ concretization_dir = install_dir / 'concretization'
+ buildcache_dir = install_dir / 'buildcache'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ gpg = GpgConfig(gpg_name='test-gpg', gpg_mail='test@test.com')
+ config = SpackConfig(env=env, install_dir=install_dir, concretization_dir=concretization_dir,
+ buildcache_dir=buildcache_dir, gpg=gpg)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperationCreateCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env()
+ assert len(spack_operation.cache_dependency.list_tags()) > 0
+
+
+def test_spack_create_cache_installation(tmp_path):
+ install_dir = tmp_path
+ concretization_dir = install_dir / 'concretization'
+ buildcache_dir = install_dir / 'buildcache'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ gpg = GpgConfig(gpg_name='test-gpg', gpg_mail='test@test.com')
+ config = SpackConfig(env=env, install_dir=install_dir, concretization_dir=concretization_dir,
+ buildcache_dir=buildcache_dir, gpg=gpg)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperationCreateCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env()
+ assert len(spack_operation.cache_dependency.list_tags()) > 0
+ spack_operation.install_packages()
+ assert len(spack_operation.build_cache.list_tags()) > 0
diff --git a/esd/bll/SpackManager.py b/esd/bll/SpackManager.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/bll/__init__.py b/esd/bll/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_factory/SpackOperationCreateCache.py b/esd/spack_factory/SpackOperationCreateCache.py
index 02cef258..34bed32f 100644
--- a/esd/spack_factory/SpackOperationCreateCache.py
+++ b/esd/spack_factory/SpackOperationCreateCache.py
@@ -38,16 +38,15 @@ class SpackOperationCreateCache(SpackOperation):
self.logger.info(f'Created new spack concretization for create cache: {self.spack_config.env.env_name}')
@check_spack_env
- def install_packages(self, jobs: int, debug=False):
+ def install_packages(self, jobs: int = 2, debug=False):
signed = False
if self.spack_config.gpg:
signed = True
self.create_gpg_keys()
- self.logger.info(f'Created GPG keys for {self.spack_config.env.name}')
- self.add_mirror('local_cache', self.spack_config.buildcache_dir, signed=signed, autopush=True,
+ self.add_mirror('local_cache', self.spack_config.buildcache_dir, signed=signed, autopush=signed,
global_mirror=False)
- self.logger.info(f'Added mirror for {self.spack_config.env.name}')
+ self.logger.info(f'Added mirror for {self.spack_config.env.env_name}')
super().install_packages(jobs=jobs, signed=signed, debug=debug, fresh=True)
- self.logger.info(f'Installed spack packages for {self.spack_config.env.name}')
+ self.logger.info(f'Installed spack packages for {self.spack_config.env.env_name}')
self.build_cache.upload(self.spack_config.buildcache_dir)
- self.logger.info(f'Pushed spack packages for {self.spack_config.env.name}')
+ self.logger.info(f'Pushed spack packages for {self.spack_config.env.env_name}')
--
GitLab
From e2bf31f6979a54e43c32de1e18d958094ece2d5c Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Mon, 17 Feb 2025 16:52:06 +0200
Subject: [PATCH 18/21] esd-concretize-buildcache: cli; package renaming to
dedal
---
.gitlab-ci.yml | 8 +-
dedal/bll/SpackManager.py | 29 +++
{esd => dedal}/bll/__init__.py | 0
dedal/bll/cli_utils.py | 23 ++
dedal/build_cache/BuildCacheManager.py | 6 +-
dedal/cli/SpackManager.py | 0
dedal/cli/SpackManagerAPI.py | 2 -
dedal/cli/spack_manager_api.py | 146 +++++++++++
dedal/configuration/SpackConfig.py | 9 +-
dedal/error_handling/exceptions.py | 5 +
dedal/spack_factory/SpackOperation.py | 30 +--
.../SpackOperationCreateCache.py | 14 +-
dedal/spack_factory/SpackOperationCreator.py | 3 +
dedal/spack_factory/SpackOperationUseCache.py | 40 ++-
.../integration_tests/spack_install_test.py | 6 +-
dedal/tests/spack_create_cache.py | 12 +-
dedal/tests/spack_from_cache_test.py | 14 +-
dedal/utils/utils.py | 56 +++++
{esd => dedal}/utils/variables.py | 0
esd/bll/SpackManager.py | 0
esd/configuration/SpackConfig.py | 25 --
esd/configuration/__init__.py | 0
esd/error_handling/__init__.py | 0
esd/error_handling/exceptions.py | 31 ---
esd/model/SpackDescriptor.py | 13 -
esd/model/__init__.py | 0
esd/spack_factory/SpackOperation.py | 230 ------------------
esd/spack_factory/SpackOperationCreator.py | 17 --
esd/spack_factory/SpackOperationUseCache.py | 68 ------
esd/spack_factory/__init__.py | 0
esd/utils/bootstrap.sh | 6 -
esd/utils/utils.py | 136 -----------
esd/wrapper/__init__.py | 0
esd/wrapper/spack_wrapper.py | 15 --
pyproject.toml | 9 +-
35 files changed, 359 insertions(+), 594 deletions(-)
create mode 100644 dedal/bll/SpackManager.py
rename {esd => dedal}/bll/__init__.py (100%)
create mode 100644 dedal/bll/cli_utils.py
delete mode 100644 dedal/cli/SpackManager.py
delete mode 100644 dedal/cli/SpackManagerAPI.py
create mode 100644 dedal/cli/spack_manager_api.py
rename {esd => dedal}/spack_factory/SpackOperationCreateCache.py (85%)
rename {esd => dedal}/utils/variables.py (100%)
delete mode 100644 esd/bll/SpackManager.py
delete mode 100644 esd/configuration/SpackConfig.py
delete mode 100644 esd/configuration/__init__.py
delete mode 100644 esd/error_handling/__init__.py
delete mode 100644 esd/error_handling/exceptions.py
delete mode 100644 esd/model/SpackDescriptor.py
delete mode 100644 esd/model/__init__.py
delete mode 100644 esd/spack_factory/SpackOperation.py
delete mode 100644 esd/spack_factory/SpackOperationCreator.py
delete mode 100644 esd/spack_factory/SpackOperationUseCache.py
delete mode 100644 esd/spack_factory/__init__.py
delete mode 100644 esd/utils/bootstrap.sh
delete mode 100644 esd/utils/utils.py
delete mode 100644 esd/wrapper/__init__.py
delete mode 100644 esd/wrapper/spack_wrapper.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b3aaf6e6..f3e20af8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,17 +28,17 @@ testing-pytest:
- docker-runner
image: ubuntu:22.04
script:
- - chmod +x esd/utils/bootstrap.sh
- - ./esd/utils/bootstrap.sh
+ - chmod +x dedal/utils/bootstrap.sh
+ - ./dedal/utils/bootstrap.sh
- pip install .
- - pytest ./esd/tests/ -s --junitxml=test-results.xml --ignore=./esd/tests/spack_create_cache.py
+ - pytest ./dedal/tests/ -s --junitxml=test-results.xml --ignore=./dedal/tests/spack_create_cache.py
artifacts:
when: always
reports:
junit: test-results.xml
paths:
- test-results.xml
- - .esd.log
+ - .dedal.log
- .generate_cache.log
expire_in: 1 week
diff --git a/dedal/bll/SpackManager.py b/dedal/bll/SpackManager.py
new file mode 100644
index 00000000..252cfe97
--- /dev/null
+++ b/dedal/bll/SpackManager.py
@@ -0,0 +1,29 @@
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackManager:
+ def __init__(self, spack_config: SpackConfig = None):
+ self._spack_config = spack_config
+
+ def _get_spack_operation(self):
+ return SpackOperationCreator.get_spack_operator(self._spack_config)
+
+ def install_spack(self, version: str):
+ self._get_spack_operation().install_spack(spack_version=f'v{version}')
+
+ def add_spack_repo(self, repo: SpackDescriptor):
+ """
+ After additional repo was added, setup_spack_env must be reinvoked
+ """
+ self._spack_config.add_repo(repo)
+
+ def setup_spack_env(self):
+ self._get_spack_operation().setup_spack_env()
+
+ def concretize_spack_env(self):
+ self._get_spack_operation().concretize_spack_env()
+
+ def install_packages(self, jobs: int):
+ self._get_spack_operation().install_packages(jobs=jobs)
diff --git a/esd/bll/__init__.py b/dedal/bll/__init__.py
similarity index 100%
rename from esd/bll/__init__.py
rename to dedal/bll/__init__.py
diff --git a/dedal/bll/cli_utils.py b/dedal/bll/cli_utils.py
new file mode 100644
index 00000000..bfc74ed0
--- /dev/null
+++ b/dedal/bll/cli_utils.py
@@ -0,0 +1,23 @@
+import jsonpickle
+import os
+
+
+def save_config(spack_config_data, config_path: str):
+ """Save config to JSON file."""
+ with open(config_path, "w") as data_file:
+ data_file.write(jsonpickle.encode(spack_config_data))
+
+
+def load_config(config_path: str):
+ """Load config from JSON file."""
+ if os.path.exists(config_path):
+ with open(config_path, "r") as data_file:
+ data = jsonpickle.decode(data_file.read())
+ return data
+ return {}
+
+
+def clear_config(config_path: str):
+ """Delete the JSON config file."""
+ if os.path.exists(config_path):
+ os.remove(config_path)
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index cccb5846..55fa10cb 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -2,9 +2,9 @@ import os
import oras.client
from pathlib import Path
-from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
-from esd.logger.logger_builder import get_logger
-from esd.utils.utils import clean_up
+from dedal.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from dedal.logger.logger_builder import get_logger
+from dedal.utils.utils import clean_up
class BuildCacheManager(BuildCacheManagerInterface):
diff --git a/dedal/cli/SpackManager.py b/dedal/cli/SpackManager.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/dedal/cli/SpackManagerAPI.py b/dedal/cli/SpackManagerAPI.py
deleted file mode 100644
index 4f883a1b..00000000
--- a/dedal/cli/SpackManagerAPI.py
+++ /dev/null
@@ -1,2 +0,0 @@
-class SpackManagerAPI:
- pass
diff --git a/dedal/cli/spack_manager_api.py b/dedal/cli/spack_manager_api.py
new file mode 100644
index 00000000..5021b1f5
--- /dev/null
+++ b/dedal/cli/spack_manager_api.py
@@ -0,0 +1,146 @@
+import os
+from pathlib import Path
+import click
+import jsonpickle
+
+from dedal.bll.SpackManager import SpackManager
+from dedal.bll.cli_utils import save_config, load_config
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.utils.utils import resolve_path
+
+SESSION_CONFIG_PATH = os.path.expanduser(f"~/tmp/dedal/dedal_session.json")
+os.makedirs(os.path.dirname(SESSION_CONFIG_PATH), exist_ok=True)
+
+
+@click.group()
+@click.pass_context
+def cli(ctx):
+ config = load_config(SESSION_CONFIG_PATH)
+ if ctx.invoked_subcommand not in ['set-config', 'install-spack'] and not config:
+ click.echo("No configuration set. Use `set-config` first.")
+ ctx.exit(1)
+ if config:
+ config['env_path'] = resolve_path(config['env_path'])
+ config['concretization_dir'] = resolve_path(config['concretization_dir'])
+ config['buildcache_dir'] = resolve_path(config['buildcache_dir'])
+ env = SpackDescriptor(config['env_name'], config['env_path'], config['env_git_path'])
+ gpg = GpgConfig(config['gpg_name'], config['gpg_mail']) if config['gpg_name'] and config['gpg_mail'] else None
+ spack_config = SpackConfig(env=env, repos=None, install_dir=config['install_dir'],
+ upstream_instance=config['upstream_instance'],
+ concretization_dir=config['concretization_dir'],
+ buildcache_dir=config['buildcache_dir'],
+ system_name=config['system_name'], gpg=gpg,
+ use_spack_global=config['use_spack_global'])
+ ctx.obj = SpackManager(spack_config)
+
+
+@cli.command()
+@click.option('--use_spack_global', is_flag=True, default=False, help="Uses spack installed globally on the os")
+@click.option("--env_name", type=str, default=None, help="Environment name")
+@click.option("--env_path", type=str, default=None, help="Environment path to download locally")
+@click.option("--env_git_path", type=str, default=None, help="Git path to download the environment")
+@click.option("--install_dir", type=str,
+ help="Install directory for installing spack; spack environments and repositories are stored here")
+@click.option("--upstream_instance", type=str, default=None, help="Upstream instance for spack environment")
+@click.option("--system_name", type=str, default=None, help="System name; it is used inside the spack environment")
+@click.option("--concretization_dir", type=str, default=None,
+ help="Directory where the concretization caching (spack.lock) will be downloaded")
+@click.option("--buildcache_dir", type=str, default=None,
+ help="Directory where the binary caching is downloaded for the spack packages")
+@click.option("--gpg_name", type=str, default=None, help="Gpg name")
+@click.option("--gpg_mail", type=str, default=None, help="Gpg mail contact address")
+def set_config(env_name, env_path, env_git_path, install_dir, upstream_instance, system_name, concretization_dir,
+ buildcache_dir, gpg_name, gpg_mail, use_spack_global):
+ """Set configuration parameters for the session."""
+ spack_config_data = {
+ "env_name": env_name,
+ "env_path": env_path,
+ "env_git_path": env_git_path,
+ "install_dir": install_dir,
+ "upstream_instance": upstream_instance,
+ "system_name": system_name,
+ "concretization_dir": Path(concretization_dir),
+ "buildcache_dir": Path(buildcache_dir),
+ "gpg_name": gpg_name,
+ "gpg_mail": gpg_mail,
+ "use_spack_global": use_spack_global,
+ "repos": []
+ }
+ save_config(spack_config_data, SESSION_CONFIG_PATH)
+ click.echo("Configuration saved.")
+
+
+@click.command()
+def show_config():
+ """Show the current configuration."""
+ config = load_config(SESSION_CONFIG_PATH)
+ if config:
+ click.echo(jsonpickle.encode(config, indent=2))
+ else:
+ click.echo("No configuration set. Use `set-config` first.")
+
+
+@cli.command()
+@click.option("--spack_version", type=str, default='0.23.0', help="Spack version")
+@click.pass_context
+def install_spack(ctx, spack_version: str):
+ """Install spack in the install_dir folder"""
+ if ctx.obj is None:
+ SpackManager().install_spack(spack_version)
+ else:
+ ctx.obj.install_spack(spack_version)
+
+
+@cli.command()
+@click.option("--repo_name", type=str, required=True, default=None, help="Repository name")
+@click.option("--path", type=str, required=True, default=None, help="Repository path to download locally")
+@click.option("--git_path", type=str, required=True, default=None, help="Git path to download the repository")
+def add_spack_repo(repo_name: str, path: str, git_path: str = None):
+ """Adds a spack repository to the spack environments. The setup command must be rerun."""
+ path = resolve_path(path)
+ repo = SpackDescriptor(repo_name, path, git_path)
+ config = load_config(SESSION_CONFIG_PATH)
+ config['repos'].append(repo)
+ save_config(config, SESSION_CONFIG_PATH)
+ click.echo("dedal setup_spack_env must be reran after each repo is added for the environment.")
+
+
+@cli.command()
+@click.pass_context
+def setup_spack_env(ctx):
+ """Setups a spack environment according to the given configuration."""
+ ctx.obj.setup_spack_env()
+
+
+@cli.command()
+@click.pass_context
+def concretize(ctx):
+ """Spack concretization step"""
+ ctx.obj.concretize_spack_env()
+
+
+@cli.command()
+@click.option("--jobs", type=int, default=2, help="Number of parallel jobs for spack installation")
+@click.pass_context
+def install_packages(ctx, jobs):
+ """Installs spack packages present in the spack environment defined in configuration"""
+ ctx.obj.install_packages(jobs=jobs)
+
+
+@click.command()
+def clear_config():
+ """Clear stored configuration"""
+ if os.path.exists(SESSION_CONFIG_PATH):
+ os.remove(SESSION_CONFIG_PATH)
+ click.echo("Configuration cleared!")
+ else:
+ click.echo("No configuration to clear.")
+
+
+cli.add_command(show_config)
+cli.add_command(clear_config)
+
+if __name__ == "__main__":
+ cli()
diff --git a/dedal/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py
index 0d470679..f0037702 100644
--- a/dedal/configuration/SpackConfig.py
+++ b/dedal/configuration/SpackConfig.py
@@ -8,15 +8,17 @@ from dedal.model import SpackDescriptor
class SpackConfig:
def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
- concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None):
+ concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None,
+ use_spack_global=False):
self.env = env
if repos is None:
self.repos = []
else:
self.repos = repos
self.install_dir = install_dir
- if self.install_dir:
- os.makedirs(self.install_dir, exist_ok=True)
+ if self.install_dir is None:
+ self.install_dir = Path(os.getcwd()).resolve()
+ os.makedirs(self.install_dir, exist_ok=True)
self.upstream_instance = upstream_instance
self.system_name = system_name
self.concretization_dir = concretization_dir
@@ -26,6 +28,7 @@ class SpackConfig:
if self.buildcache_dir:
os.makedirs(self.buildcache_dir, exist_ok=True)
self.gpg = gpg
+ self.use_spack_global = use_spack_global
def add_repo(self, repo: SpackDescriptor):
if self.repos is None:
diff --git a/dedal/error_handling/exceptions.py b/dedal/error_handling/exceptions.py
index 0256f886..4653e72f 100644
--- a/dedal/error_handling/exceptions.py
+++ b/dedal/error_handling/exceptions.py
@@ -39,3 +39,8 @@ class SpackGpgException(BashCommandException):
"""
To be thrown when the spack fails to create gpg keys
"""
+
+class SpackRepoException(BashCommandException):
+ """
+ To be thrown when the spack fails to add a repo
+ """
\ No newline at end of file
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
index ecfbb8e5..497bce12 100644
--- a/dedal/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -3,7 +3,7 @@ import re
import subprocess
from pathlib import Path
from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
+ SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException, SpackRepoException
from dedal.logger.logger_builder import get_logger
from dedal.configuration.SpackConfig import SpackConfig
from dedal.tests.testing_variables import SPACK_VERSION
@@ -29,13 +29,14 @@ class SpackOperation:
self.spack_config = spack_config
self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
self.spack_dir = self.spack_config.install_dir / 'spack'
- self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
+
+ self.spack_setup_script = "" if self.spack_config.use_spack_global else f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'} &&"
self.logger = logger
if self.spack_config.env and spack_config.env.path:
self.spack_config.env.path = spack_config.env.path.resolve()
self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = spack_config.env.path / spack_config.env.env_name
- self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
+ self.env_path: Path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'{self.spack_setup_script} spack env activate -p {self.env_path}'
def create_fetch_spack_environment(self):
if self.spack_config.env.git_path:
@@ -45,7 +46,7 @@ class SpackOperation:
else:
os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+ f'{self.spack_setup_script} spack env create -d {self.env_path}',
check=True, logger=self.logger,
info_msg=f"Created {self.spack_config.env.env_name} spack environment",
exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
@@ -83,7 +84,7 @@ class SpackOperation:
"""Check if the given Spack repository exists."""
if self.spack_config.env is None:
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack repo list',
+ f'{self.spack_setup_script} spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
info_msg=f'Checking if {repo_name} exists')
@@ -121,7 +122,7 @@ class SpackOperation:
check=True, logger=self.logger,
info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
- exception=BashCommandException)
+ exception=SpackRepoException)
@check_spack_env
def get_compiler_version(self):
@@ -144,7 +145,7 @@ class SpackOperation:
return gcc_version
def get_spack_installed_version(self):
- spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ spack_version = run_command("bash", "-c", f'{self.spack_setup_script} spack --version',
capture_output=True, text=True, check=True,
logger=self.logger,
info_msg=f"Getting spack version",
@@ -159,7 +160,7 @@ class SpackOperation:
run_command("bash", "-c",
f'{self.spack_command_on_env} && spack concretize {force}',
check=True,
- logger=self.logger,
+ logger=self.logger,
info_msg=f'Concertization step for {self.spack_config.env.env_name}',
exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
exception=SpackConcertizeException)
@@ -167,7 +168,7 @@ class SpackOperation:
def create_gpg_keys(self):
if self.spack_config.gpg:
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+ f'{self.spack_setup_script} spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
check=True,
logger=self.logger,
info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
@@ -181,7 +182,7 @@ class SpackOperation:
signed = '--signed' if signed else ''
if global_mirror:
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ f'{self.spack_setup_script} spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
check=True,
logger=self.logger,
info_msg=f'Added mirror {mirror_name}',
@@ -199,7 +200,7 @@ class SpackOperation:
def remove_mirror(self, mirror_name: str):
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+ f'{self.spack_setup_script} spack mirror rm {mirror_name}',
check=True,
logger=self.logger,
info_msg=f'Removing mirror {mirror_name}',
@@ -248,7 +249,9 @@ class SpackOperation:
# add spack setup commands to .bashrc
with open(bashrc_path, "a") as bashrc:
bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
- bashrc.write(f"source {self.spack_setup_script}\n")
+ spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
+ bashrc.write(f"{spack_setup_script}\n")
+ bashrc.write(f"{spack_setup_script}\n")
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
@@ -257,7 +260,6 @@ class SpackOperation:
self.logger.info("Spack install completed")
# Restart Bash after the installation ends
os.system("exec bash")
-
# Configure upstream Spack instance if specified
if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
diff --git a/esd/spack_factory/SpackOperationCreateCache.py b/dedal/spack_factory/SpackOperationCreateCache.py
similarity index 85%
rename from esd/spack_factory/SpackOperationCreateCache.py
rename to dedal/spack_factory/SpackOperationCreateCache.py
index 34bed32f..68dc9c56 100644
--- a/esd/spack_factory/SpackOperationCreateCache.py
+++ b/dedal/spack_factory/SpackOperationCreateCache.py
@@ -1,13 +1,13 @@
import os
-from esd.error_handling.exceptions import NoSpackEnvironmentException
+from dedal.error_handling.exceptions import NoSpackEnvironmentException
-from esd.utils.utils import copy_to_tmp, copy_file
-from esd.wrapper.spack_wrapper import check_spack_env
-from esd.build_cache.BuildCacheManager import BuildCacheManager
-from esd.configuration.SpackConfig import SpackConfig
-from esd.logger.logger_builder import get_logger
-from esd.spack_factory.SpackOperation import SpackOperation
+from dedal.utils.utils import copy_to_tmp, copy_file
+from dedal.wrapper.spack_wrapper import check_spack_env
+from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.logger.logger_builder import get_logger
+from dedal.spack_factory.SpackOperation import SpackOperation
class SpackOperationCreateCache(SpackOperation):
diff --git a/dedal/spack_factory/SpackOperationCreator.py b/dedal/spack_factory/SpackOperationCreator.py
index 54517a84..6ad30827 100644
--- a/dedal/spack_factory/SpackOperationCreator.py
+++ b/dedal/spack_factory/SpackOperationCreator.py
@@ -1,5 +1,6 @@
from dedal.configuration.SpackConfig import SpackConfig
from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
@@ -10,5 +11,7 @@ class SpackOperationCreator:
return SpackOperation(SpackConfig())
elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
return SpackOperation(spack_config)
+ elif spack_config.concretization_dir and spack_config.buildcache_dir:
+ return SpackOperationCreateCache(spack_config)
else:
return SpackOperationUseCache(spack_config)
diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
index 41a9094c..d3b24809 100644
--- a/dedal/spack_factory/SpackOperationUseCache.py
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -1,8 +1,14 @@
import os
+import subprocess
+from pathlib import Path
+
from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.error_handling.exceptions import SpackInstallPackagesException
from dedal.logger.logger_builder import get_logger
from dedal.spack_factory.SpackOperation import SpackOperation
from dedal.configuration.SpackConfig import SpackConfig
+from dedal.utils.utils import file_exists_and_not_empty, run_command, log_command, copy_to_tmp, copy_file
+from dedal.wrapper.spack_wrapper import check_spack_env
class SpackOperationUseCache(SpackOperation):
@@ -28,5 +34,35 @@ class SpackOperationUseCache(SpackOperation):
super().setup_spack_env()
# todo add buildcache to the spack environment
- def concretize_spack_env(self, force=True):
- pass
+ @check_spack_env
+ def concretize_spack_env(self):
+ concretization_redo = False
+ self.cache_dependency.download(self.spack_config.concretization_dir)
+ if file_exists_and_not_empty(self.spack_config.concretization_dir / 'spack.lock'):
+ concretization_file_path = self.env_path / 'spack.lock'
+ copy_file(self.spack_config.concretization_dir / 'spack.lock', self.env_path)
+ # redo the concretization step if spack.lock file was not downloaded from the cache
+ if not file_exists_and_not_empty(concretization_file_path):
+ super().concretize_spack_env(force=True)
+ concretization_redo = True
+ else:
+ # redo the concretization step if spack.lock file was not downloaded from the cache
+ super().concretize_spack_env(force=True)
+ concretization_redo = True
+ return concretization_redo
+
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, debug=False):
+ signed = '' if signed else '--no-check-signature'
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack {debug} install -v --reuse {signed} --j {jobs}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
+ exception=SpackInstallPackagesException)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
+ return install_result
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 2373c098..0c6cf127 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -1,6 +1,6 @@
-from esd.configuration.SpackConfig import SpackConfig
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.tests.testing_variables import SPACK_VERSION
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.tests.testing_variables import SPACK_VERSION
def test_spack_install_scratch(tmp_path):
diff --git a/dedal/tests/spack_create_cache.py b/dedal/tests/spack_create_cache.py
index d22801c1..92684e67 100644
--- a/dedal/tests/spack_create_cache.py
+++ b/dedal/tests/spack_create_cache.py
@@ -1,12 +1,12 @@
from pathlib import Path
-from esd.configuration.GpgConfig import GpgConfig
-from esd.configuration.SpackConfig import SpackConfig
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
-from esd.model.SpackDescriptor import SpackDescriptor
-from esd.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
-from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
-from esd.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
"""
Before running those tests, the repositories where the caching is stored must be cleared after each run.
diff --git a/dedal/tests/spack_from_cache_test.py b/dedal/tests/spack_from_cache_test.py
index 293a58d1..d54ced7a 100644
--- a/dedal/tests/spack_from_cache_test.py
+++ b/dedal/tests/spack_from_cache_test.py
@@ -1,9 +1,9 @@
-from esd.configuration.SpackConfig import SpackConfig
-from esd.model.SpackDescriptor import SpackDescriptor
-from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
-from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
-from esd.utils.utils import file_exists_and_not_empty
-from esd.utils.variables import test_spack_env_git, ebrains_spack_builds_git
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.utils.utils import file_exists_and_not_empty
+from dedal.utils.variables import test_spack_env_git, ebrains_spack_builds_git
def test_spack_from_cache_concretize(tmp_path):
@@ -34,5 +34,5 @@ def test_spack_from_cache_install(tmp_path):
assert spack_operation.concretize_spack_env() == False
concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
- install_result = spack_operation.install_packages(jobs=2, signed=False, debug=False)
+ install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False)
assert install_result.returncode == 0
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 9fc82ad5..2f7c4847 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -2,6 +2,7 @@ import logging
import os
import shutil
import subprocess
+import tempfile
from pathlib import Path
from dedal.error_handling.exceptions import BashCommandException
@@ -78,6 +79,21 @@ def log_command(results, log_file: str):
log_file.write(results.stderr)
+def copy_to_tmp(file_path: Path) -> Path:
+ """
+ Creates a temporary directory and copies the given file into it.
+
+ :param file_path: Path to the file that needs to be copied.
+ :return: Path to the copied file inside the temporary directory.
+ """
+ if not file_path.is_file():
+ raise FileNotFoundError(f"File not found: {file_path}")
+ tmp_dir = Path(tempfile.mkdtemp())
+ tmp_file_path = tmp_dir / file_path.name
+ shutil.copy(file_path, tmp_file_path)
+ return tmp_file_path
+
+
def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
logger: logging = logging.getLogger(__name__)):
"""Update or add an environment variable in ~/.bashrc."""
@@ -98,3 +114,43 @@ def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.ex
logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
with open(bashrc_path, "w") as file:
file.writelines(lines)
+
+
+def copy_file(src: Path, dst: Path, logger: logging = logging.getLogger(__name__)) -> None:
+ """
+ Copy a file from src to dest.
+ """
+ if not os.path.exists(src):
+ raise FileNotFoundError(f"Source file '{src}' does not exist.")
+ src.resolve().as_posix()
+ dst.resolve().as_posix()
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ shutil.copy2(src, dst)
+ logger.debug(f"File copied from '{src}' to '{dst}'")
+
+
+def delete_file(file_path: str, logger: logging = logging.getLogger(__name__)) -> bool:
+ """
+ Deletes a file at the given path. Returns True if successful, False if the file doesn't exist.
+ """
+ try:
+ os.remove(file_path)
+ logger.debug(f"File '{file_path}' deleted.")
+ return True
+ except FileNotFoundError:
+ logger.error(f"File not found: {file_path}")
+ return False
+ except PermissionError:
+ logger.error(f"Permission denied: {file_path}")
+ return False
+ except Exception as e:
+ logger.error(f"Error deleting file {file_path}: {e}")
+ return False
+
+
+def resolve_path(path: str):
+ if path is None:
+ path = Path(os.getcwd()).resolve()
+ else:
+ path = Path(path).resolve()
+ return path
diff --git a/esd/utils/variables.py b/dedal/utils/variables.py
similarity index 100%
rename from esd/utils/variables.py
rename to dedal/utils/variables.py
diff --git a/esd/bll/SpackManager.py b/esd/bll/SpackManager.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
deleted file mode 100644
index 93a2e874..00000000
--- a/esd/configuration/SpackConfig.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import os
-from pathlib import Path
-from esd.model import SpackDescriptor
-
-
-class SpackConfig:
- def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
- install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
- concretization_dir: Path = None, buildcache_dir: Path = None):
- self.env = env
- if repos is None:
- self.repos = []
- else:
- self.repos = repos
- self.install_dir = install_dir
- self.upstream_instance = upstream_instance
- self.system_name = system_name
- self.concretization_dir = concretization_dir
- self.buildcache_dir = buildcache_dir
-
- def add_repo(self, repo: SpackDescriptor):
- if self.repos is None:
- self.repos = []
- else:
- self.repos.append(repo)
diff --git a/esd/configuration/__init__.py b/esd/configuration/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/esd/error_handling/__init__.py b/esd/error_handling/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
deleted file mode 100644
index 9d11b5fa..00000000
--- a/esd/error_handling/exceptions.py
+++ /dev/null
@@ -1,31 +0,0 @@
-class SpackException(Exception):
-
- def __init__(self, message):
- super().__init__(message)
- self.message = str(message)
-
- def __str__(self):
- return self.message
-
-
-class BashCommandException(SpackException):
- """
- To be thrown when a bash command has failed
- """
-
-
-class NoSpackEnvironmentException(BashCommandException):
- """
- To be thrown when an operation on a spack environment is executed without the environment being activated or existent
- """
-
-
-class SpackConcertizeException(BashCommandException):
- """
- To be thrown when the spack concretization step fails
- """
-
-class SpackInstallPackagesException(BashCommandException):
- """
- To be thrown when the spack fails to install spack packages
- """
diff --git a/esd/model/SpackDescriptor.py b/esd/model/SpackDescriptor.py
deleted file mode 100644
index 70e484fb..00000000
--- a/esd/model/SpackDescriptor.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import os
-from pathlib import Path
-
-
-class SpackDescriptor:
- """"
- Provides details about the spack environment
- """
-
- def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
- self.env_name = env_name
- self.path = path
- self.git_path = git_path
diff --git a/esd/model/__init__.py b/esd/model/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/esd/spack_factory/SpackOperation.py b/esd/spack_factory/SpackOperation.py
deleted file mode 100644
index 3a1a5d9c..00000000
--- a/esd/spack_factory/SpackOperation.py
+++ /dev/null
@@ -1,230 +0,0 @@
-import os
-import re
-import subprocess
-from abc import ABC, abstractmethod
-from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException, SpackConcertizeException
-from esd.logger.logger_builder import get_logger
-from esd.configuration.SpackConfig import SpackConfig
-from esd.tests.testing_variables import SPACK_VERSION
-from esd.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-
-
-class SpackOperation(ABC):
- """
- This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
- Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
-
- Attributes:
- -----------
- env : SpackDescriptor
- spack environment details
- repos : list[SpackDescriptor]
- upstream_instance : str
- path to Spack instance to use as upstream (optional)
- """
-
- def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
- self.spack_config = spack_config
- self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
- self.spack_dir = self.spack_config.install_dir / 'spack'
- self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
- self.logger = logger
- if self.spack_config.env and spack_config.env.path:
- self.spack_config.env.path = spack_config.env.path.resolve()
- self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path: Path = spack_config.env.path / spack_config.env.env_name
- self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
-
- @abstractmethod
- def concretize_spack_env(self, force=True):
- pass
-
- def create_fetch_spack_environment(self):
- if self.spack_config.env.git_path:
- git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
- self.spack_config.env.git_path,
- logger=self.logger)
- else:
- os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
- check=True, logger=self.logger,
- info_msg=f"Created {self.spack_config.env.env_name} spack environment",
- exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
- exception=BashCommandException)
-
- def setup_spack_env(self):
- """
- This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
- """
- bashrc_path = os.path.expanduser("~/.bashrc")
- if self.spack_config.system_name:
- set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
- os.environ['SYSTEMNAME'] = self.spack_config.system_name
- if self.spack_dir.exists() and self.spack_dir.is_dir():
- set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
- set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
- self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
- else:
- self.logger.error(f'Invalid installation path: {self.spack_dir}')
- # Restart the bash after adding environment variables
- self.create_fetch_spack_environment()
- if self.spack_config.install_dir.exists():
- for repo in self.spack_config.repos:
- repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
- git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
- if not self.spack_repo_exists(repo.env_name):
- self.add_spack_repo(repo.path, repo.env_name)
- self.logger.debug(f'Added spack repository {repo.env_name}')
- else:
- self.logger.debug(f'Spack repository {repo.env_name} already added')
-
- def spack_repo_exists(self, repo_name: str) -> bool | None:
- """Check if the given Spack repository exists."""
- if self.spack_config.env is None:
- result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Checking if {repo_name} exists')
- if result is None:
- return False
- else:
- if self.spack_env_exists():
- result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Checking if repository {repo_name} was added')
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
- if result is None:
- return False
- return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
-
- def spack_env_exists(self):
- result = run_command("bash", "-c",
- self.spack_command_on_env,
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
- if result is None:
- return False
- return True
-
- @check_spack_env
- def add_spack_repo(self, repo_path: Path, repo_name: str):
- """Add the Spack repository if it does not exist."""
- run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger,
- info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
- exception=BashCommandException)
-
- @check_spack_env
- def get_compiler_version(self):
- result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack compiler list',
- check=True, logger=self.logger,
- capture_output=True, text=True,
- info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
- exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
- exception=BashCommandException)
- # todo add error handling and tests
- if result.stdout is None:
- self.logger.debug('No gcc found for {self.env.env_name}')
- return None
-
- # Find the first occurrence of a GCC compiler using regex
- match = re.search(r"gcc@([\d\.]+)", result.stdout)
- gcc_version = match.group(1)
- self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
- return gcc_version
-
- def get_spack_installed_version(self):
- spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
- capture_output=True, text=True, check=True,
- logger=self.logger,
- info_msg=f"Getting spack version",
- exception_msg=f"Error retrieving Spack version")
- if spack_version:
- return spack_version.stdout.strip().split()[0]
- return None
-
- @check_spack_env
- def concretize_spack_env(self, force=True):
- force = '--force' if force else ''
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- info_msg=f'Concertization step for {self.spack_config.env.env_name}',
- exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
- exception=SpackConcertizeException)
-
- @check_spack_env
- def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
- signed = '' if signed else '--no-check-signature'
- fresh = '--fresh' if fresh else ''
- debug = '--debug' if debug else ''
- install_result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- logger=self.logger,
- info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
- exception=SpackInstallPackagesException)
- log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
- return install_result
-
- def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
- try:
- user = os.getlogin()
- except OSError:
- user = None
-
- self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
- if not self.spack_dir.exists():
- run_command(
- "git", "clone", "--depth", "1",
- "-c", "advice.detachedHead=false",
- "-c", "feature.manyFiles=true",
- "--branch", spack_version, spack_repo, self.spack_dir
- , check=True, logger=self.logger)
- self.logger.debug("Cloned spack")
- else:
- self.logger.debug("Spack already cloned.")
-
- bashrc_path = os.path.expanduser("~/.bashrc")
- # ensure the file exists before opening it
- if not os.path.exists(bashrc_path):
- open(bashrc_path, "w").close()
- # add spack setup commands to .bashrc
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
- bashrc.write(f"source {self.spack_setup_script}\n")
- self.logger.info("Added Spack PATH to .bashrc")
- if user:
- run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
- info_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
- self.logger.info("Spack install completed")
- # Restart Bash after the installation ends
- os.system("exec bash")
-
- # Configure upstream Spack instance if specified
- if self.spack_config.upstream_instance:
- upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
- with open(upstreams_yaml_path, "w") as file:
- file.write(f"""upstreams:
- upstream-spack-instance:
- install_tree: {self.spack_config.upstream_instance}/spack/opt/spack
- """)
- self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_factory/SpackOperationCreator.py b/esd/spack_factory/SpackOperationCreator.py
deleted file mode 100644
index 1fceb7b9..00000000
--- a/esd/spack_factory/SpackOperationCreator.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from esd.configuration.SpackConfig import SpackConfig
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
-from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
-
-
-class SpackOperationCreator:
- @staticmethod
- def get_spack_operator(spack_config: SpackConfig = None):
- if spack_config is None:
- return SpackOperation(SpackConfig())
- elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
- return SpackOperation(spack_config)
- elif spack_config.concretization_dir and spack_config.buildcache_dir:
- return SpackOperationCreateCache(spack_config)
- else:
- return SpackOperationUseCache(spack_config)
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
deleted file mode 100644
index 3ec78fb0..00000000
--- a/esd/spack_factory/SpackOperationUseCache.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import os
-import subprocess
-from pathlib import Path
-
-from esd.build_cache.BuildCacheManager import BuildCacheManager
-from esd.error_handling.exceptions import SpackInstallPackagesException
-from esd.logger.logger_builder import get_logger
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.configuration.SpackConfig import SpackConfig
-from esd.utils.utils import file_exists_and_not_empty, run_command, log_command, copy_to_tmp, copy_file
-from esd.wrapper.spack_wrapper import check_spack_env
-
-
-class SpackOperationUseCache(SpackOperation):
- """
- This class uses caching for the concretization step and for the installation step.
- """
-
- def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
- cache_version_build='cache'):
- super().__init__(spack_config, logger=get_logger(__name__))
- self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
- os.environ.get('CONCRETIZE_OCI_PROJECT'),
- os.environ.get('CONCRETIZE_OCI_USERNAME'),
- os.environ.get('CONCRETIZE_OCI_PASSWORD'),
- cache_version=cache_version_concretize)
- self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
- os.environ.get('BUILDCACHE_OCI_PROJECT'),
- os.environ.get('BUILDCACHE_OCI_USERNAME'),
- os.environ.get('BUILDCACHE_OCI_PASSWORD'),
- cache_version=cache_version_build)
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- @check_spack_env
- def concretize_spack_env(self, upload=False):
- concretization_redo = False
- self.cache_dependency.download(self.spack_config.concretization_dir)
- if file_exists_and_not_empty(self.spack_config.concretization_dir / 'spack.lock'):
- concretization_file_path = self.env_path / 'spack.lock'
- copy_file(self.spack_config.concretization_dir / 'spack.lock', self.env_path)
- # redo the concretization step if spack.lock file was not downloaded from the cache
- if not file_exists_and_not_empty(concretization_file_path):
- super().concretize_spack_env(force=True)
- concretization_redo = True
- else:
- # redo the concretization step if spack.lock file was not downloaded from the cache
- super().concretize_spack_env(force=True)
- concretization_redo = True
- return concretization_redo
-
- @check_spack_env
- def install_packages(self, jobs: int, signed=True, debug=False):
- signed = '' if signed else '--no-check-signature'
- debug = '--debug' if debug else ''
- install_result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v --reuse {signed} --j {jobs}',
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- logger=self.logger,
- debug_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
- exception=SpackInstallPackagesException)
- log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
- return install_result
diff --git a/esd/spack_factory/__init__.py b/esd/spack_factory/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/esd/utils/bootstrap.sh b/esd/utils/bootstrap.sh
deleted file mode 100644
index 9b7d0131..00000000
--- a/esd/utils/bootstrap.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-# Minimal prerequisites for installing the esd_library
-# pip must be installed on the OS
-echo "Bootstrapping..."
-apt update
-apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
-python3 -m pip install --upgrade pip setuptools wheel
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
deleted file mode 100644
index f9a7d0e8..00000000
--- a/esd/utils/utils.py
+++ /dev/null
@@ -1,136 +0,0 @@
-import logging
-import os
-import shutil
-import subprocess
-import tempfile
-from pathlib import Path
-
-from esd.error_handling.exceptions import BashCommandException
-import re
-
-
-def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
- """
- All the folders from the list dirs are removed with all the content in them
- """
- for cleanup_dir in dirs:
- cleanup_dir = Path(cleanup_dir).resolve()
- if cleanup_dir.exists():
- logger.info(f"Removing {cleanup_dir}")
- try:
- shutil.rmtree(Path(cleanup_dir))
- except OSError as e:
- logger.error(f"Failed to remove {cleanup_dir}: {e}")
- if not ignore_errors:
- raise e
- else:
- logger.info(f"{cleanup_dir} does not exist")
-
-
-def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
- exception=None, **kwargs):
- try:
- logger.info(f'{info_msg}: args: {args}')
- return subprocess.run(args, **kwargs)
- except subprocess.CalledProcessError as e:
- if exception_msg is not None:
- logger.error(f"{exception_msg}: {e}")
- if exception is not None:
- raise exception(f'{exception_msg} : {e}')
- else:
- return None
-
-
-def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
- if not dir.exists():
- run_command(
- "git", "clone", "--depth", "1",
- "-c", "advice.detachedHead=false",
- "-c", "feature.manyFiles=true",
- git_path, dir
- , check=True, logger=logger,
- info_msg=f'Cloned repository {repo_name}',
- exception_msg=f'Failed to clone repository: {repo_name}',
- exception=BashCommandException)
- else:
- logger.info(f'Repository {repo_name} already cloned.')
-
-
-def file_exists_and_not_empty(file: Path) -> bool:
- return file.is_file() and file.stat().st_size > 0
-
-
-def log_command(results, log_file: str):
- with open(log_file, "w") as log_file:
- log_file.write(results.stdout)
- log_file.write("\n--- STDERR ---\n")
- log_file.write(results.stderr)
-
-
-def copy_to_tmp(file_path: Path) -> Path:
- """
- Creates a temporary directory and copies the given file into it.
-
- :param file_path: Path to the file that needs to be copied.
- :return: Path to the copied file inside the temporary directory.
- """
- if not file_path.is_file():
- raise FileNotFoundError(f"File not found: {file_path}")
- tmp_dir = Path(tempfile.mkdtemp())
- tmp_file_path = tmp_dir / file_path.name
- shutil.copy(file_path, tmp_file_path)
- return tmp_file_path
-
-
-def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
- logger: logging = logging.getLogger(__name__)):
- """Update or add an environment variable in ~/.bashrc."""
- with open(bashrc_path, "r") as file:
- lines = file.readlines()
- pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
- updated = False
- # Modify the existing variable if found
- for i, line in enumerate(lines):
- if pattern.match(line):
- lines[i] = f'export {var_name}={value}\n'
- updated = True
- break
- if not updated:
- lines.append(f'\nexport {var_name}={value}\n')
- logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
- else:
- logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
- with open(bashrc_path, "w") as file:
- file.writelines(lines)
-
-
-def copy_file(src: Path, dst: Path, logger: logging = logging.getLogger(__name__)) -> None:
- """
- Copy a file from src to dest.
- """
- if not os.path.exists(src):
- raise FileNotFoundError(f"Source file '{src}' does not exist.")
- src.resolve().as_posix()
- dst.resolve().as_posix()
- os.makedirs(os.path.dirname(dst), exist_ok=True)
- shutil.copy2(src, dst)
- logger.debug(f"File copied from '{src}' to '{dst}'")
-
-
-def delete_file(file_path: str, logger: logging = logging.getLogger(__name__)) -> bool:
- """
- Deletes a file at the given path. Returns True if successful, False if the file doesn't exist.
- """
- try:
- os.remove(file_path)
- logger.debug(f"File '{file_path}' deleted.")
- return True
- except FileNotFoundError:
- logger.error(f"File not found: {file_path}")
- return False
- except PermissionError:
- logger.error(f"Permission denied: {file_path}")
- return False
- except Exception as e:
- logger.error(f"Error deleting file {file_path}: {e}")
- return False
diff --git a/esd/wrapper/__init__.py b/esd/wrapper/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/esd/wrapper/spack_wrapper.py b/esd/wrapper/spack_wrapper.py
deleted file mode 100644
index c2f9c116..00000000
--- a/esd/wrapper/spack_wrapper.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import functools
-
-from esd.error_handling.exceptions import NoSpackEnvironmentException
-
-
-def check_spack_env(method):
- @functools.wraps(method)
- def wrapper(self, *args, **kwargs):
- if self.spack_env_exists():
- return method(self, *args, **kwargs) # Call the method with 'self'
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
-
- return wrapper
diff --git a/pyproject.toml b/pyproject.toml
index abcbe05d..aad18fa5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
-name = "esd-tools"
+name = "dedal"
version = "0.1.0"
authors = [
{name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
@@ -19,7 +19,12 @@ dependencies = [
"pytest",
"pytest-mock",
"pytest-ordering",
+ "click",
+ "jsonpickle",
]
+[project.scripts]
+dedal = "dedal.cli.spack_manager_api:cli"
+
[tool.setuptools.data-files]
-"esd-tools" = ["esd/logger/logging.conf"]
\ No newline at end of file
+"dedal" = ["dedal/logger/logging.conf"]
\ No newline at end of file
--
GitLab
From f3f6195f3f5949567fccb35b14d25e85191faf35 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 20 Feb 2025 13:01:51 +0200
Subject: [PATCH 19/21] esd-concretize-buildcache: test package restructure
---
dedal/tests/{ => integration_tests}/spack_create_cache.py | 0
dedal/tests/{ => integration_tests}/spack_from_cache_test.py | 0
2 files changed, 0 insertions(+), 0 deletions(-)
rename dedal/tests/{ => integration_tests}/spack_create_cache.py (100%)
rename dedal/tests/{ => integration_tests}/spack_from_cache_test.py (100%)
diff --git a/dedal/tests/spack_create_cache.py b/dedal/tests/integration_tests/spack_create_cache.py
similarity index 100%
rename from dedal/tests/spack_create_cache.py
rename to dedal/tests/integration_tests/spack_create_cache.py
diff --git a/dedal/tests/spack_from_cache_test.py b/dedal/tests/integration_tests/spack_from_cache_test.py
similarity index 100%
rename from dedal/tests/spack_from_cache_test.py
rename to dedal/tests/integration_tests/spack_from_cache_test.py
--
GitLab
From 887160db4792c8ffc47b392558e041d19034919d Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 20 Feb 2025 14:51:03 +0200
Subject: [PATCH 20/21] esd-concretize-buildcache: fixed factory for
SpackOperation; additional tests
---
.gitlab-ci.yml | 2 +-
dedal/spack_factory/SpackOperationCreator.py | 10 ++--
...te_cache.py => spack_create_cache_test.py} | 6 +++
.../spack_from_cache_test.py | 12 +++--
.../spack_operation_creator_test.py | 50 +++++++++++++++++++
5 files changed, 71 insertions(+), 9 deletions(-)
rename dedal/tests/integration_tests/{spack_create_cache.py => spack_create_cache_test.py} (88%)
create mode 100644 dedal/tests/integration_tests/spack_operation_creator_test.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f3e20af8..4f15b9ab 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,7 +31,7 @@ testing-pytest:
- chmod +x dedal/utils/bootstrap.sh
- ./dedal/utils/bootstrap.sh
- pip install .
- - pytest ./dedal/tests/ -s --junitxml=test-results.xml --ignore=./dedal/tests/spack_create_cache.py
+ - pytest ./dedal/tests/ -s --junitxml=test-results.xml
artifacts:
when: always
reports:
diff --git a/dedal/spack_factory/SpackOperationCreator.py b/dedal/spack_factory/SpackOperationCreator.py
index 6ad30827..fdc929d3 100644
--- a/dedal/spack_factory/SpackOperationCreator.py
+++ b/dedal/spack_factory/SpackOperationCreator.py
@@ -6,12 +6,14 @@ from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
class SpackOperationCreator:
@staticmethod
- def get_spack_operator(spack_config: SpackConfig = None):
+ def get_spack_operator(spack_config: SpackConfig = None, use_cache: bool = False) -> SpackOperation:
if spack_config is None:
- return SpackOperation(SpackConfig())
+ return SpackOperation()
elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
return SpackOperation(spack_config)
- elif spack_config.concretization_dir and spack_config.buildcache_dir:
+ elif (spack_config.concretization_dir and spack_config.buildcache_dir) and not use_cache:
return SpackOperationCreateCache(spack_config)
- else:
+ elif (spack_config.concretization_dir and spack_config.buildcache_dir) and use_cache:
return SpackOperationUseCache(spack_config)
+ else:
+ return SpackOperation(SpackConfig())
diff --git a/dedal/tests/integration_tests/spack_create_cache.py b/dedal/tests/integration_tests/spack_create_cache_test.py
similarity index 88%
rename from dedal/tests/integration_tests/spack_create_cache.py
rename to dedal/tests/integration_tests/spack_create_cache_test.py
index 92684e67..fcef47a8 100644
--- a/dedal/tests/integration_tests/spack_create_cache.py
+++ b/dedal/tests/integration_tests/spack_create_cache_test.py
@@ -1,5 +1,7 @@
from pathlib import Path
+import pytest
+
from dedal.configuration.GpgConfig import GpgConfig
from dedal.configuration.SpackConfig import SpackConfig
@@ -14,6 +16,8 @@ Ebrains Harbour does not support deletion via API, so the clean up must be done
"""
+@pytest.mark.skip(
+ reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.")
def test_spack_create_cache_concretization(tmp_path):
install_dir = tmp_path
concretization_dir = install_dir / 'concretization'
@@ -32,6 +36,8 @@ def test_spack_create_cache_concretization(tmp_path):
assert len(spack_operation.cache_dependency.list_tags()) > 0
+@pytest.mark.skip(
+ reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.")
def test_spack_create_cache_installation(tmp_path):
install_dir = tmp_path
concretization_dir = install_dir / 'concretization'
diff --git a/dedal/tests/integration_tests/spack_from_cache_test.py b/dedal/tests/integration_tests/spack_from_cache_test.py
index d54ced7a..33f44833 100644
--- a/dedal/tests/integration_tests/spack_from_cache_test.py
+++ b/dedal/tests/integration_tests/spack_from_cache_test.py
@@ -1,3 +1,4 @@
+import pytest
from dedal.configuration.SpackConfig import SpackConfig
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
@@ -10,9 +11,10 @@ def test_spack_from_cache_concretize(tmp_path):
install_dir = tmp_path
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir / 'concretize',
+ buildcache_dir=install_dir / 'buildcache')
spack_config.add_repo(repo)
- spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
assert isinstance(spack_operation, SpackOperationUseCache)
spack_operation.install_spack()
spack_operation.setup_spack_env()
@@ -21,13 +23,15 @@ def test_spack_from_cache_concretize(tmp_path):
assert file_exists_and_not_empty(concretization_file_path) == True
+@pytest.mark.skip(reason="Skipping test::test_spack_from_cache_install until all the functionalities in SpackOperationUseCache")
def test_spack_from_cache_install(tmp_path):
install_dir = tmp_path
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir / 'concretize',
+ buildcache_dir=install_dir / 'buildcache')
spack_config.add_repo(repo)
- spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
assert isinstance(spack_operation, SpackOperationUseCache)
spack_operation.install_spack()
spack_operation.setup_spack_env()
diff --git a/dedal/tests/integration_tests/spack_operation_creator_test.py b/dedal/tests/integration_tests/spack_operation_creator_test.py
new file mode 100644
index 00000000..226184b0
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_operation_creator_test.py
@@ -0,0 +1,50 @@
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
+
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.tests.testing_variables import ebrains_spack_builds_git, test_spack_env_git
+
+
+def test_spack_creator_scratch_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperation)
+
+
+def test_spack_creator_scratch_2(tmp_path):
+ spack_config = None
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperation)
+
+
+def test_spack_creator_scratch_3():
+ spack_config = SpackConfig()
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperation)
+
+
+def test_spack_creator_create_cache(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir, buildcache_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperationCreateCache)
+
+
+def test_spack_creator_use_cache(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir, buildcache_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+ assert isinstance(spack_operation, SpackOperationUseCache)
--
GitLab
From 688ec8edf7613db476ba620c1f933ad0a5f79564 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 20 Feb 2025 18:36:10 +0200
Subject: [PATCH 21/21] esd-concretize-buildcache: cli, tests and bug fixing
---
.env | 3 +-
README.md | 16 +-
dedal/bll/SpackManager.py | 16 +-
dedal/cli/spack_manager_api.py | 110 ++++++-----
dedal/configuration/SpackConfig.py | 20 +-
dedal/model/SpackDescriptor.py | 2 +-
dedal/spack_factory/SpackOperation.py | 26 ++-
.../SpackOperationCreateCache.py | 7 +-
dedal/spack_factory/SpackOperationUseCache.py | 9 +-
.../spack_from_scratch_test.py | 30 ++-
.../unit_tests/spack_manager_api_test.py | 181 ++++++++++++++++++
11 files changed, 315 insertions(+), 105 deletions(-)
create mode 100644 dedal/tests/unit_tests/spack_manager_api_test.py
diff --git a/.env b/.env
index 93e62653..786f93e2 100644
--- a/.env
+++ b/.env
@@ -2,8 +2,7 @@ BUILDCACHE_OCI_HOST=""
BUILDCACHE_OCI_PASSWORD=""
BUILDCACHE_OCI_PROJECT=""
BUILDCACHE_OCI_USERNAME=""
-
CONCRETIZE_OCI_HOST=""
CONCRETIZE_OCI_PASSWORD=""
CONCRETIZE_OCI_PROJECT=""
-CONCRETIZE_OCI_USERNAME""
+CONCRETIZE_OCI_USERNAME=""
diff --git a/README.md b/README.md
index dd68dcfa..5dc3fcc1 100644
--- a/README.md
+++ b/README.md
@@ -2,14 +2,6 @@
This repository provides functionalities to easily ```managed spack environments``` and ```helpers for the container image build flow```.
-This library runs only on different kinds of linux distribution operating systems.
-
-The lowest ```spack version``` compatible with this library is ```v0.23.0```.
-
-
- This repository also provied CLI interface. For more informations, after installing this library, call dedal --help.
-
-
**Setting up the needed environment variables**
The ````<checkout path>\dedal\.env```` file contains the environment variables required for OCI registry used for caching.
Ensure that you edit the ````<checkout path>\dedal\.env```` file to match your environment.
@@ -51,3 +43,11 @@ The lowest ```spack version``` compatible with this library is ```v0.23.0```.
For both concretization and binary caches, the cache version can be changed via the attributes ```cache_version_concretize``` and ```cache_version_build```.
The default values are ```v1```.
+
+Before using this library, the following tool must be installed on Linux distribution:
+````
+ apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+````
+````
+ python3 -m pip install --upgrade pip setuptools wheel
+````
diff --git a/dedal/bll/SpackManager.py b/dedal/bll/SpackManager.py
index 252cfe97..e5fae221 100644
--- a/dedal/bll/SpackManager.py
+++ b/dedal/bll/SpackManager.py
@@ -1,21 +1,27 @@
+import os
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.configuration.SpackConfig import SpackConfig
class SpackManager:
- def __init__(self, spack_config: SpackConfig = None):
+ """
+ This class defines the logic used by the CLI
+ """
+
+ def __init__(self, spack_config: SpackConfig = None, use_cache=False):
self._spack_config = spack_config
+ self._use_cache = use_cache
def _get_spack_operation(self):
- return SpackOperationCreator.get_spack_operator(self._spack_config)
+ return SpackOperationCreator.get_spack_operator(self._spack_config, self._use_cache)
- def install_spack(self, version: str):
- self._get_spack_operation().install_spack(spack_version=f'v{version}')
+ def install_spack(self, version: str, bashrc_path=os.path.expanduser("~/.bashrc")):
+ self._get_spack_operation().install_spack(spack_version=f'v{version}', bashrc_path=bashrc_path)
def add_spack_repo(self, repo: SpackDescriptor):
"""
- After additional repo was added, setup_spack_env must be reinvoked
+ After additional repo was added, setup_spack_env must be invoked
"""
self._spack_config.add_repo(repo)
diff --git a/dedal/cli/spack_manager_api.py b/dedal/cli/spack_manager_api.py
index 5021b1f5..43867028 100644
--- a/dedal/cli/spack_manager_api.py
+++ b/dedal/cli/spack_manager_api.py
@@ -10,21 +10,19 @@ from dedal.configuration.SpackConfig import SpackConfig
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.utils.utils import resolve_path
-SESSION_CONFIG_PATH = os.path.expanduser(f"~/tmp/dedal/dedal_session.json")
+SESSION_CONFIG_PATH = os.path.expanduser(f'~/tmp/dedal/dedal_session.json')
os.makedirs(os.path.dirname(SESSION_CONFIG_PATH), exist_ok=True)
@click.group()
@click.pass_context
-def cli(ctx):
+def cli(ctx: click.Context):
config = load_config(SESSION_CONFIG_PATH)
if ctx.invoked_subcommand not in ['set-config', 'install-spack'] and not config:
- click.echo("No configuration set. Use `set-config` first.")
+ click.echo('No configuration set. Use `set-config` first.')
ctx.exit(1)
if config:
config['env_path'] = resolve_path(config['env_path'])
- config['concretization_dir'] = resolve_path(config['concretization_dir'])
- config['buildcache_dir'] = resolve_path(config['buildcache_dir'])
env = SpackDescriptor(config['env_name'], config['env_path'], config['env_git_path'])
gpg = GpgConfig(config['gpg_name'], config['gpg_mail']) if config['gpg_name'] and config['gpg_mail'] else None
spack_config = SpackConfig(env=env, repos=None, install_dir=config['install_dir'],
@@ -33,43 +31,50 @@ def cli(ctx):
buildcache_dir=config['buildcache_dir'],
system_name=config['system_name'], gpg=gpg,
use_spack_global=config['use_spack_global'])
- ctx.obj = SpackManager(spack_config)
+ ctx.obj = SpackManager(spack_config, use_cache=config['use_cache'])
@cli.command()
-@click.option('--use_spack_global', is_flag=True, default=False, help="Uses spack installed globally on the os")
-@click.option("--env_name", type=str, default=None, help="Environment name")
-@click.option("--env_path", type=str, default=None, help="Environment path to download locally")
-@click.option("--env_git_path", type=str, default=None, help="Git path to download the environment")
-@click.option("--install_dir", type=str,
- help="Install directory for installing spack; spack environments and repositories are stored here")
-@click.option("--upstream_instance", type=str, default=None, help="Upstream instance for spack environment")
-@click.option("--system_name", type=str, default=None, help="System name; it is used inside the spack environment")
-@click.option("--concretization_dir", type=str, default=None,
- help="Directory where the concretization caching (spack.lock) will be downloaded")
-@click.option("--buildcache_dir", type=str, default=None,
- help="Directory where the binary caching is downloaded for the spack packages")
-@click.option("--gpg_name", type=str, default=None, help="Gpg name")
-@click.option("--gpg_mail", type=str, default=None, help="Gpg mail contact address")
-def set_config(env_name, env_path, env_git_path, install_dir, upstream_instance, system_name, concretization_dir,
- buildcache_dir, gpg_name, gpg_mail, use_spack_global):
- """Set configuration parameters for the session."""
+@click.option('--use_cache', is_flag=True, default=False, help='Enables cashing')
+@click.option('--use_spack_global', is_flag=True, default=False, help='Uses spack installed globally on the os')
+@click.option('--env_name', type=str, default=None, help='Environment name')
+@click.option('--env_path', type=str, default=None, help='Environment path to download locally')
+@click.option('--env_git_path', type=str, default=None, help='Git path to download the environment')
+@click.option('--install_dir', type=str,
+ help='Install directory for installing spack; spack environments and repositories are stored here')
+@click.option('--upstream_instance', type=str, default=None, help='Upstream instance for spack environment')
+@click.option('--system_name', type=str, default=None, help='System name; it is used inside the spack environment')
+@click.option('--concretization_dir', type=str, default=None,
+ help='Directory where the concretization caching (spack.lock) will be downloaded')
+@click.option('--buildcache_dir', type=str, default=None,
+ help='Directory where the binary caching is downloaded for the spack packages')
+@click.option('--gpg_name', type=str, default=None, help='Gpg name')
+@click.option('--gpg_mail', type=str, default=None, help='Gpg mail contact address')
+@click.option('--cache_version_concretize', type=str, default='v1', help='Cache version for concretizaion data')
+@click.option('--cache_version_build', type=str, default='v1', help='Cache version for binary caches data')
+def set_config(use_cache, env_name, env_path, env_git_path, install_dir, upstream_instance, system_name,
+ concretization_dir,
+ buildcache_dir, gpg_name, gpg_mail, use_spack_global, cache_version_concretize, cache_version_build):
+ """Set configuration parameters for tahe session."""
spack_config_data = {
- "env_name": env_name,
- "env_path": env_path,
- "env_git_path": env_git_path,
- "install_dir": install_dir,
- "upstream_instance": upstream_instance,
- "system_name": system_name,
- "concretization_dir": Path(concretization_dir),
- "buildcache_dir": Path(buildcache_dir),
- "gpg_name": gpg_name,
- "gpg_mail": gpg_mail,
- "use_spack_global": use_spack_global,
- "repos": []
+ 'use_cache': use_cache,
+ 'env_name': env_name,
+ 'env_path': env_path,
+ 'env_git_path': env_git_path,
+ 'install_dir': install_dir,
+ 'upstream_instance': upstream_instance,
+ 'system_name': system_name,
+ 'concretization_dir': Path(concretization_dir) if concretization_dir else None,
+ 'buildcache_dir': Path(buildcache_dir) if buildcache_dir else None,
+ 'gpg_name': gpg_name,
+ 'gpg_mail': gpg_mail,
+ 'use_spack_global': use_spack_global,
+ 'repos': [],
+ 'cache_version_concretize': cache_version_concretize,
+ 'cache_version_build': cache_version_build,
}
save_config(spack_config_data, SESSION_CONFIG_PATH)
- click.echo("Configuration saved.")
+ click.echo('Configuration saved.')
@click.command()
@@ -79,24 +84,25 @@ def show_config():
if config:
click.echo(jsonpickle.encode(config, indent=2))
else:
- click.echo("No configuration set. Use `set-config` first.")
+ click.echo('No configuration set. Use `set-config` first.')
@cli.command()
-@click.option("--spack_version", type=str, default='0.23.0', help="Spack version")
+@click.option('--spack_version', type=str, default='0.23.0', help='Spack version')
+@click.option('--bashrc_path', type=str, default='~/.bashrc', help='Path to .bashrc')
@click.pass_context
-def install_spack(ctx, spack_version: str):
+def install_spack(ctx: click.Context, spack_version: str, bashrc_path: str):
"""Install spack in the install_dir folder"""
if ctx.obj is None:
- SpackManager().install_spack(spack_version)
+ SpackManager().install_spack(spack_version, bashrc_path)
else:
- ctx.obj.install_spack(spack_version)
+ ctx.obj.install_spack(spack_version, bashrc_path)
@cli.command()
-@click.option("--repo_name", type=str, required=True, default=None, help="Repository name")
-@click.option("--path", type=str, required=True, default=None, help="Repository path to download locally")
-@click.option("--git_path", type=str, required=True, default=None, help="Git path to download the repository")
+@click.option('--repo_name', type=str, required=True, default=None, help='Repository name')
+@click.option('--path', type=str, required=True, default=None, help='Repository path to download locally')
+@click.option('--git_path', type=str, required=True, default=None, help='Git path to download the repository')
def add_spack_repo(repo_name: str, path: str, git_path: str = None):
"""Adds a spack repository to the spack environments. The setup command must be rerun."""
path = resolve_path(path)
@@ -104,27 +110,27 @@ def add_spack_repo(repo_name: str, path: str, git_path: str = None):
config = load_config(SESSION_CONFIG_PATH)
config['repos'].append(repo)
save_config(config, SESSION_CONFIG_PATH)
- click.echo("dedal setup_spack_env must be reran after each repo is added for the environment.")
+ click.echo('dedal setup_spack_env must be reran after each repo is added for the environment.')
@cli.command()
@click.pass_context
-def setup_spack_env(ctx):
+def setup_spack_env(ctx: click.Context):
"""Setups a spack environment according to the given configuration."""
ctx.obj.setup_spack_env()
@cli.command()
@click.pass_context
-def concretize(ctx):
+def concretize(ctx: click.Context):
"""Spack concretization step"""
ctx.obj.concretize_spack_env()
@cli.command()
-@click.option("--jobs", type=int, default=2, help="Number of parallel jobs for spack installation")
+@click.option('--jobs', type=int, default=2, help='Number of parallel jobs for spack installation')
@click.pass_context
-def install_packages(ctx, jobs):
+def install_packages(ctx: click.Context, jobs):
"""Installs spack packages present in the spack environment defined in configuration"""
ctx.obj.install_packages(jobs=jobs)
@@ -134,13 +140,13 @@ def clear_config():
"""Clear stored configuration"""
if os.path.exists(SESSION_CONFIG_PATH):
os.remove(SESSION_CONFIG_PATH)
- click.echo("Configuration cleared!")
+ click.echo('Configuration cleared!')
else:
- click.echo("No configuration to clear.")
+ click.echo('No configuration to clear.')
cli.add_command(show_config)
cli.add_command(clear_config)
-if __name__ == "__main__":
+if __name__ == '__main__':
cli()
diff --git a/dedal/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py
index f0037702..d76783ec 100644
--- a/dedal/configuration/SpackConfig.py
+++ b/dedal/configuration/SpackConfig.py
@@ -1,34 +1,30 @@
import os
from pathlib import Path
-
from dedal.configuration.GpgConfig import GpgConfig
from dedal.model import SpackDescriptor
+from dedal.utils.utils import resolve_path
class SpackConfig:
def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None,
- use_spack_global=False):
+ use_spack_global=False, cache_version_concretize='v1',
+ cache_version_build='v1'):
self.env = env
if repos is None:
self.repos = []
else:
self.repos = repos
- self.install_dir = install_dir
- if self.install_dir is None:
- self.install_dir = Path(os.getcwd()).resolve()
- os.makedirs(self.install_dir, exist_ok=True)
self.upstream_instance = upstream_instance
self.system_name = system_name
- self.concretization_dir = concretization_dir
- if self.concretization_dir:
- os.makedirs(self.concretization_dir, exist_ok=True)
- self.buildcache_dir = buildcache_dir
- if self.buildcache_dir:
- os.makedirs(self.buildcache_dir, exist_ok=True)
+ self.concretization_dir = concretization_dir if concretization_dir is None else resolve_path(concretization_dir)
+ self.buildcache_dir = buildcache_dir if buildcache_dir is None else resolve_path(buildcache_dir)
+ self.install_dir = resolve_path(install_dir)
self.gpg = gpg
self.use_spack_global = use_spack_global
+ self.cache_version_concretize = cache_version_concretize
+ self.cache_version_build = cache_version_build
def add_repo(self, repo: SpackDescriptor):
if self.repos is None:
diff --git a/dedal/model/SpackDescriptor.py b/dedal/model/SpackDescriptor.py
index 70e484fb..421c4824 100644
--- a/dedal/model/SpackDescriptor.py
+++ b/dedal/model/SpackDescriptor.py
@@ -9,5 +9,5 @@ class SpackDescriptor:
def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
self.env_name = env_name
- self.path = path
+ self.path = path if isinstance(path,Path) else Path(path)
self.git_path = git_path
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
index 497bce12..58dcad8b 100644
--- a/dedal/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -27,13 +27,20 @@ class SpackOperation:
def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
self.spack_config = spack_config
- self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_config.install_dir = spack_config.install_dir
+ os.makedirs(self.spack_config.install_dir, exist_ok=True)
self.spack_dir = self.spack_config.install_dir / 'spack'
self.spack_setup_script = "" if self.spack_config.use_spack_global else f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'} &&"
self.logger = logger
+ self.spack_config.concretization_dir = spack_config.concretization_dir
+ if self.spack_config.concretization_dir:
+ os.makedirs(self.spack_config.concretization_dir, exist_ok=True)
+ self.spack_config.buildcache_dir = spack_config.buildcache_dir
+ if self.spack_config.buildcache_dir:
+ os.makedirs(self.spack_config.buildcache_dir, exist_ok=True)
if self.spack_config.env and spack_config.env.path:
- self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path = spack_config.env.path
self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
self.env_path: Path = spack_config.env.path / spack_config.env.env_name
self.spack_command_on_env = f'{self.spack_setup_script} spack env activate -p {self.env_path}'
@@ -213,7 +220,7 @@ class SpackOperation:
fresh = '--fresh' if fresh else ''
debug = '--debug' if debug else ''
install_result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} -j {jobs} {fresh}',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
@@ -224,7 +231,8 @@ class SpackOperation:
log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
- def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack',
+ bashrc_path=os.path.expanduser("~/.bashrc")):
try:
user = os.getlogin()
except OSError:
@@ -242,7 +250,6 @@ class SpackOperation:
else:
self.logger.debug("Spack already cloned.")
- bashrc_path = os.path.expanduser("~/.bashrc")
# ensure the file exists before opening it
if not os.path.exists(bashrc_path):
open(bashrc_path, "w").close()
@@ -251,15 +258,16 @@ class SpackOperation:
bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
bashrc.write(f"{spack_setup_script}\n")
- bashrc.write(f"{spack_setup_script}\n")
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
info_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
self.logger.info("Spack install completed")
- # Restart Bash after the installation ends
- os.system("exec bash")
+ if self.spack_config.use_spack_global is True:
+ # Restart the bash only of the spack is used globally
+ self.logger.info('Restarting bash')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
+ os.system("exec bash")
# Configure upstream Spack instance if specified
if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
diff --git a/dedal/spack_factory/SpackOperationCreateCache.py b/dedal/spack_factory/SpackOperationCreateCache.py
index 68dc9c56..f04eae3a 100644
--- a/dedal/spack_factory/SpackOperationCreateCache.py
+++ b/dedal/spack_factory/SpackOperationCreateCache.py
@@ -15,19 +15,18 @@ class SpackOperationCreateCache(SpackOperation):
This class creates caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
- cache_version_build='cache'):
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
super().__init__(spack_config, logger=get_logger(__name__))
self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
os.environ.get('CONCRETIZE_OCI_PROJECT'),
os.environ.get('CONCRETIZE_OCI_USERNAME'),
os.environ.get('CONCRETIZE_OCI_PASSWORD'),
- cache_version=cache_version_concretize)
+ cache_version=spack_config.cache_version_concretize)
self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
os.environ.get('BUILDCACHE_OCI_PROJECT'),
os.environ.get('BUILDCACHE_OCI_USERNAME'),
os.environ.get('BUILDCACHE_OCI_PASSWORD'),
- cache_version=cache_version_build)
+ cache_version=spack_config.cache_version_build)
@check_spack_env
def concretize_spack_env(self):
diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
index d3b24809..cb2b3ac8 100644
--- a/dedal/spack_factory/SpackOperationUseCache.py
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -16,19 +16,18 @@ class SpackOperationUseCache(SpackOperation):
This class uses caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='v1',
- cache_version_build='v1'):
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
super().__init__(spack_config, logger=get_logger(__name__))
self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
os.environ.get('CONCRETIZE_OCI_PROJECT'),
os.environ.get('CONCRETIZE_OCI_USERNAME'),
os.environ.get('CONCRETIZE_OCI_PASSWORD'),
- cache_version=cache_version_concretize)
+ cache_version=spack_config.cache_version_concretize)
self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
os.environ.get('BUILDCACHE_OCI_PROJECT'),
os.environ.get('BUILDCACHE_OCI_USERNAME'),
os.environ.get('BUILDCACHE_OCI_PASSWORD'),
- cache_version=cache_version_build)
+ cache_version=spack_config.cache_version_build)
def setup_spack_env(self):
super().setup_spack_env()
@@ -56,7 +55,7 @@ class SpackOperationUseCache(SpackOperation):
signed = '' if signed else '--no-check-signature'
debug = '--debug' if debug else ''
install_result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack {debug} install -v --reuse {signed} --j {jobs}',
+ f'{self.spack_command_on_env} && spack {debug} install -v --reuse {signed} -j {jobs}',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
index fa862301..794caef1 100644
--- a/dedal/tests/integration_tests/spack_from_scratch_test.py
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -6,6 +6,7 @@ from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
from dedal.utils.utils import file_exists_and_not_empty
+from dedal.spack_factory.SpackOperation import SpackOperation
def test_spack_repo_exists_1(tmp_path):
@@ -23,8 +24,8 @@ def test_spack_repo_exists_2(tmp_path):
env = SpackDescriptor('ebrains-spack-builds', install_dir)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
- print(spack_operation.get_spack_installed_version())
spack_operation.setup_spack_env()
assert spack_operation.spack_repo_exists(env.env_name) == False
@@ -34,6 +35,7 @@ def test_spack_from_scratch_setup_1(tmp_path):
env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
assert spack_operation.spack_repo_exists(env.env_name) == False
@@ -47,6 +49,7 @@ def test_spack_from_scratch_setup_2(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
assert spack_operation.spack_repo_exists(env.env_name) == True
@@ -60,17 +63,21 @@ def test_spack_from_scratch_setup_3(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
with pytest.raises(BashCommandException):
spack_operation.setup_spack_env()
-def test_spack_from_scratch_setup_4():
- install_dir = Path('./install').resolve()
- env = SpackModel('new_environment', install_dir, )
- spack_manager = SpackManagerScratch(env, system_name='ebrainslab')
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == True
+def test_spack_from_scratch_setup_4(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('new_env2', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_env_exists() == True
def test_spack_not_a_valid_repo():
@@ -79,6 +86,7 @@ def test_spack_not_a_valid_repo():
config = SpackConfig(env=env, system_name='ebrainslab')
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
with pytest.raises(BashCommandException):
spack_operation.add_spack_repo(repo.path, repo.env_name)
@@ -93,6 +101,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.install_spack()
spack_operation.setup_spack_env()
@@ -111,6 +120,7 @@ def test_spack_from_scratch_concretize_2(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=False)
@@ -126,6 +136,7 @@ def test_spack_from_scratch_concretize_3(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
concretization_file_path = spack_operation.env_path / 'spack.lock'
@@ -137,6 +148,7 @@ def test_spack_from_scratch_concretize_4(tmp_path):
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=False)
@@ -149,6 +161,7 @@ def test_spack_from_scratch_concretize_5(tmp_path):
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=True)
@@ -163,6 +176,7 @@ def test_spack_from_scratch_concretize_6(tmp_path):
config = SpackConfig(env=env, install_dir=install_dir)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=False)
@@ -177,6 +191,7 @@ def test_spack_from_scratch_concretize_7(tmp_path):
config = SpackConfig(env=env)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=True)
@@ -191,6 +206,7 @@ def test_spack_from_scratch_install(tmp_path):
config = SpackConfig(env=env)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=True)
diff --git a/dedal/tests/unit_tests/spack_manager_api_test.py b/dedal/tests/unit_tests/spack_manager_api_test.py
new file mode 100644
index 00000000..4184a864
--- /dev/null
+++ b/dedal/tests/unit_tests/spack_manager_api_test.py
@@ -0,0 +1,181 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from click.testing import CliRunner
+from dedal.cli.spack_manager_api import show_config, clear_config, install_spack, add_spack_repo, install_packages, \
+ setup_spack_env, concretize, set_config
+from dedal.model.SpackDescriptor import SpackDescriptor
+
+
+@pytest.fixture
+def runner():
+ return CliRunner()
+
+
+@pytest.fixture
+def mocked_session_path():
+ return '/mocked/tmp/session.json'
+
+
+@pytest.fixture
+def mock_spack_manager():
+ mock_spack_manager = MagicMock()
+ mock_spack_manager.install_spack = MagicMock()
+ mock_spack_manager.add_spack_repo = MagicMock()
+ mock_spack_manager.setup_spack_env = MagicMock()
+ mock_spack_manager.concretize_spack_env = MagicMock()
+ mock_spack_manager.install_packages = MagicMock()
+ return mock_spack_manager
+
+
+@pytest.fixture
+def mock_load_config():
+ with patch('dedal.cli.spack_manager_api.load_config') as mock_load:
+ yield mock_load
+
+
+@pytest.fixture
+def mock_save_config():
+ with patch('dedal.cli.spack_manager_api.save_config') as mock_save:
+ yield mock_save
+
+
+@pytest.fixture
+def mock_clear_config():
+ with patch('dedal.cli.spack_manager_api.clear_config') as mock_clear:
+ yield mock_clear
+
+
+def test_show_config_no_config(runner, mock_load_config):
+ mock_load_config.return_value = None
+ result = runner.invoke(show_config)
+ assert 'No configuration set. Use `set-config` first.' in result.output
+
+
+def test_show_config_with_config(runner, mock_load_config):
+ """Test the show_config command when config is present."""
+ mock_load_config.return_value = {"key": "value"}
+ result = runner.invoke(show_config)
+ assert result.exit_code == 0
+ assert '"key": "value"' in result.output
+
+
+def test_clear_config(runner, mock_clear_config):
+ """Test the clear_config command."""
+ with patch('os.path.exists', return_value=True), patch('os.remove') as mock_remove:
+ result = runner.invoke(clear_config)
+ assert 'Configuration cleared!' in result.output
+ mock_remove.assert_called_once()
+
+
+def test_install_spack_no_context_1(runner, mock_spack_manager):
+ """Test install_spack with no context, using SpackManager."""
+ with patch('dedal.cli.spack_manager_api.SpackManager', return_value=mock_spack_manager):
+ result = runner.invoke(install_spack, ['--spack_version', '0.24.0'])
+ mock_spack_manager.install_spack.assert_called_once_with('0.24.0', '~/.bashrc')
+ assert result.exit_code == 0
+
+
+def test_install_spack_no_context_2(runner, mock_spack_manager):
+ """Test install_spack with no context, using SpackManager and the default value for spack_version."""
+ with patch('dedal.cli.spack_manager_api.SpackManager', return_value=mock_spack_manager):
+ result = runner.invoke(install_spack)
+ mock_spack_manager.install_spack.assert_called_once_with('0.23.0', '~/.bashrc')
+ assert result.exit_code == 0
+
+
+def test_install_spack_with_mocked_context_1(runner, mock_spack_manager):
+ """Test install_spack with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(install_spack, ['--spack_version', '0.24.0', '--bashrc_path', '/home/.bahsrc'], obj=mock_spack_manager)
+ mock_spack_manager.install_spack.assert_called_once_with('0.24.0', '/home/.bahsrc')
+ assert result.exit_code == 0
+
+
+def test_install_spack_with_mocked_context_2(runner, mock_spack_manager):
+ """Test install_spack with a mocked context, using ctx.obj as SpackManager and the default value for spack_version."""
+ result = runner.invoke(install_spack, obj=mock_spack_manager)
+ mock_spack_manager.install_spack.assert_called_once_with('0.23.0', '~/.bashrc')
+ assert result.exit_code == 0
+
+
+def test_setup_spack_env(runner, mock_spack_manager):
+ """Test setup_spack_env with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(setup_spack_env, obj=mock_spack_manager)
+ mock_spack_manager.setup_spack_env.assert_called_once_with()
+ assert result.exit_code == 0
+
+
+def test_concretize(runner, mock_spack_manager):
+ """Test install_spack with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(concretize, obj=mock_spack_manager)
+ mock_spack_manager.concretize_spack_env.assert_called_once_with()
+ assert result.exit_code == 0
+
+
+def test_install_packages_1(runner, mock_spack_manager):
+ """Test install_packages with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(install_packages, obj=mock_spack_manager)
+ mock_spack_manager.install_packages.assert_called_once_with(jobs=2)
+ assert result.exit_code == 0
+
+
+def test_install_packages(runner, mock_spack_manager):
+ """Test install_packages with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(install_packages, ['--jobs', 3], obj=mock_spack_manager)
+ mock_spack_manager.install_packages.assert_called_once_with(jobs=3)
+ assert result.exit_code == 0
+
+
+@patch('dedal.cli.spack_manager_api.resolve_path')
+@patch('dedal.cli.spack_manager_api.SpackDescriptor')
+def test_add_spack_repo(mock_spack_descriptor, mock_resolve_path, mock_load_config, mock_save_config,
+ mocked_session_path, runner):
+ """Test adding a spack repository with mocks."""
+ expected_config = {'repos': [SpackDescriptor(env_name='test-repo')]}
+ repo_name = 'test-repo'
+ path = '/path'
+ git_path = 'https://example.com/repo.git'
+ mock_resolve_path.return_value = '/resolved/path'
+ mock_load_config.return_value = expected_config
+ mock_repo_instance = MagicMock()
+ mock_spack_descriptor.return_value = mock_repo_instance
+
+ with patch('dedal.cli.spack_manager_api.SESSION_CONFIG_PATH', mocked_session_path):
+ result = runner.invoke(add_spack_repo, ['--repo_name', repo_name, '--path', path, '--git_path', git_path])
+
+ assert result.exit_code == 0
+ assert 'dedal setup_spack_env must be reran after each repo is added' in result.output
+ mock_resolve_path.assert_called_once_with(path)
+ mock_spack_descriptor.assert_called_once_with(repo_name, '/resolved/path', git_path)
+ assert mock_repo_instance in expected_config['repos']
+ mock_save_config.assert_called_once_with(expected_config, mocked_session_path)
+
+
+def test_set_config(runner, mock_save_config, mocked_session_path):
+ """Test set_config."""
+ with patch('dedal.cli.spack_manager_api.SESSION_CONFIG_PATH', mocked_session_path):
+ result = runner.invoke(set_config, ['--env_name', 'test', '--system_name', 'sys'])
+
+ expected_config = {
+ 'use_cache': False,
+ 'env_name': 'test',
+ 'env_path': None,
+ 'env_git_path': None,
+ 'install_dir': None,
+ 'upstream_instance': None,
+ 'system_name': 'sys',
+ 'concretization_dir': None,
+ 'buildcache_dir': None,
+ 'gpg_name': None,
+ 'gpg_mail': None,
+ 'use_spack_global': False,
+ 'repos': [],
+ 'cache_version_concretize': 'v1',
+ 'cache_version_build': 'v1',
+ }
+
+ mock_save_config.assert_called_once()
+ saved_config, saved_path = mock_save_config.call_args[0]
+ assert saved_path == mocked_session_path
+ assert saved_config == expected_config
+ assert result.exit_code == 0
+ assert 'Configuration saved.' in result.output
--
GitLab