From 464f9abd8c149e6029397bf0008af335add2ec0a Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Mon, 27 Jan 2025 12:51:28 +0200
Subject: [PATCH 01/11] esd: added logger class and fixed bug
---
dedal/logger/logger_config.py | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
create mode 100644 dedal/logger/logger_config.py
diff --git a/dedal/logger/logger_config.py b/dedal/logger/logger_config.py
new file mode 100644
index 00000000..3ca3b000
--- /dev/null
+++ b/dedal/logger/logger_config.py
@@ -0,0 +1,33 @@
+import logging
+
+
+class LoggerConfig:
+ """
+ This class sets up logging with a file handler
+ and a stream handler, ensuring consistent
+ and formatted log messages.
+ """
+ def __init__(self, log_file):
+ self.log_file = log_file
+ self._configure_logger()
+
+ def _configure_logger(self):
+ formatter = logging.Formatter(
+ fmt='%(asctime)s - %(levelname)s - %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S'
+ )
+
+ file_handler = logging.FileHandler(self.log_file)
+ file_handler.setFormatter(formatter)
+
+ stream_handler = logging.StreamHandler()
+ stream_handler.setFormatter(formatter)
+
+ self.logger = logging.getLogger(__name__)
+ self.logger.setLevel(logging.DEBUG)
+
+ self.logger.addHandler(file_handler)
+ self.logger.addHandler(stream_handler)
+
+ def get_logger(self):
+ return self.logger
--
GitLab
From 3c9425117675a42f06f614c563fb6177092fd0a4 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 11:25:05 +0200
Subject: [PATCH 02/11] esd-spack-installation: setup; tests; custom exceptions
---
.gitlab-ci.yml | 18 +-
dedal/build_cache/BuildCacheManager.py | 6 +-
dedal/specfile_storage_path_source.py | 6 +-
dedal/tests/spack_from_scratch_test.py | 72 +++++++
dedal/tests/spack_install_test.py | 21 ++
dedal/utils/bootstrap.sh | 6 +
dedal/utils/utils.py | 33 +++
esd/error_handling/exceptions.py | 18 ++
esd/model/SpackModel.py | 12 ++
esd/model/__init__.py | 0
esd/spack_manager/SpackManager.py | 193 ++++++++++++++++++
esd/spack_manager/__init__.py | 0
esd/spack_manager/enums/SpackManagerEnum.py | 6 +
esd/spack_manager/enums/__init__.py | 0
.../factory/SpackManagerBuildCache.py | 19 ++
.../factory/SpackManagerCreator.py | 14 ++
.../factory/SpackManagerScratch.py | 15 ++
esd/spack_manager/factory/__init__.py | 0
pyproject.toml | 5 +-
19 files changed, 430 insertions(+), 14 deletions(-)
create mode 100644 dedal/tests/spack_from_scratch_test.py
create mode 100644 dedal/tests/spack_install_test.py
create mode 100644 dedal/utils/bootstrap.sh
create mode 100644 esd/error_handling/exceptions.py
create mode 100644 esd/model/SpackModel.py
create mode 100644 esd/model/__init__.py
create mode 100644 esd/spack_manager/SpackManager.py
create mode 100644 esd/spack_manager/__init__.py
create mode 100644 esd/spack_manager/enums/SpackManagerEnum.py
create mode 100644 esd/spack_manager/enums/__init__.py
create mode 100644 esd/spack_manager/factory/SpackManagerBuildCache.py
create mode 100644 esd/spack_manager/factory/SpackManagerCreator.py
create mode 100644 esd/spack_manager/factory/SpackManagerScratch.py
create mode 100644 esd/spack_manager/factory/__init__.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7cdc2157..4f15b9ab 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,10 +1,11 @@
stages:
- - build
- test
+ - build
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
+
build-wheel:
stage: build
tags:
@@ -21,18 +22,23 @@ build-wheel:
expire_in: 1 week
-testing:
+testing-pytest:
stage: test
tags:
- docker-runner
- image: python:latest
+ image: ubuntu:22.04
script:
- - pip install -e .
- - pytest ./dedal/tests/ --junitxml=test-results.xml
+ - chmod +x dedal/utils/bootstrap.sh
+ - ./dedal/utils/bootstrap.sh
+ - pip install .
+ - pytest ./dedal/tests/ -s --junitxml=test-results.xml
artifacts:
when: always
reports:
junit: test-results.xml
paths:
- test-results.xml
- expire_in: 1 week
\ No newline at end of file
+ - .dedal.log
+ - .generate_cache.log
+ expire_in: 1 week
+
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index 2da39e25..dbd50bf9 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -40,8 +40,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
- target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
+ target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
try:
self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
self.client.push(
@@ -51,7 +51,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self.logger.info(f"Successfully pushed {sub_path.name}")
+ self.logger.info(f"Successfully pushed {sub_path.env_name}")
except Exception as e:
self.logger.error(
f"An error occurred while pushing: {e}")
diff --git a/dedal/specfile_storage_path_source.py b/dedal/specfile_storage_path_source.py
index 6e8a8889..4d2ff658 100644
--- a/dedal/specfile_storage_path_source.py
+++ b/dedal/specfile_storage_path_source.py
@@ -49,14 +49,14 @@ for rspec in data:
format_string = "{name}-{version}"
pretty_name = pkg.spec.format_path(format_string)
- cosmetic_path = os.path.join(pkg.name, pretty_name)
+ cosmetic_path = os.path.join(pkg.env_name, pretty_name)
to_be_fetched.add(str(spack.mirror.mirror_archive_paths(pkg.fetcher, cosmetic_path).storage_path))
for resource in pkg._get_needed_resources():
- pretty_resource_name = fsys.polite_filename(f"{resource.name}-{pkg.version}")
+ pretty_resource_name = fsys.polite_filename(f"{resource.env_name}-{pkg.version}")
to_be_fetched.add(str(spack.mirror.mirror_archive_paths(resource.fetcher, pretty_resource_name).storage_path))
for patch in ss.patches:
if isinstance(patch, spack.patch.UrlPatch):
- to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.name).storage_path))
+ to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.env_name).storage_path))
for elem in to_be_fetched:
print(elem)
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
new file mode 100644
index 00000000..cca0d2e9
--- /dev/null
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -0,0 +1,72 @@
+from pathlib import Path
+
+import pytest
+
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+def test_spack_repo_exists_1():
+ spack_manager = SpackManagerScratch()
+ assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
+
+
+def test_spack_repo_exists_2():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('ebrains-spack-builds', install_dir)
+ spack_manager = SpackManagerScratch(env=env)
+ with pytest.raises(NoSpackEnvironmentException):
+ spack_manager.spack_repo_exists(env.env_name)
+
+
+# def test_spack_repo_exists_3():
+# install_dir = Path('./install').resolve()
+# env = SpackModel('ebrains-spack-builds', install_dir)
+# spack_manager = SpackManagerScratch(env=env)
+# spack_manager.setup_spack_env()
+# assert spack_manager.spack_repo_exists(env.env_name) == False
+
+
+def test_spack_from_scratch_setup_1():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_2():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_3():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('new_env1', install_dir)
+ repo = env
+ spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ with pytest.raises(BashCommandException):
+ spack_manager.setup_spack_env()
+
+
+def test_spack_from_scratch_setup_4():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('new_env2', install_dir)
+ spack_manager = SpackManagerScratch(env=env)
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_env_exists() == True
+
+
+def test_spack_not_a_valid_repo():
+ env = SpackModel('ebrains-spack-builds', Path(), None)
+ repo = env
+ spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
+ with pytest.raises(NoSpackEnvironmentException):
+ spack_manager.add_spack_repo(repo.path, repo.env_name)
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py
new file mode 100644
index 00000000..34f68323
--- /dev/null
+++ b/dedal/tests/spack_install_test.py
@@ -0,0 +1,21 @@
+import pytest
+
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+# we need this test to run first so that spack is installed only once
+@pytest.mark.run(order=1)
+def test_spack_install_scratch():
+ spack_manager = SpackManagerScratch()
+ spack_manager.install_spack(spack_version="v0.21.1")
+ installed_spack_version = spack_manager.get_spack_installed_version()
+ required_version = "0.21.1"
+ assert required_version == installed_spack_version
+
+
+def test_spack_install_buildcache():
+ spack_manager = SpackManagerBuildCache()
+ installed_spack_version = spack_manager.get_spack_installed_version()
+ required_version = "0.21.1"
+ assert required_version == installed_spack_version
diff --git a/dedal/utils/bootstrap.sh b/dedal/utils/bootstrap.sh
new file mode 100644
index 00000000..9b7d0131
--- /dev/null
+++ b/dedal/utils/bootstrap.sh
@@ -0,0 +1,6 @@
+# Minimal prerequisites for installing the esd_library
+# pip must be installed on the OS
+echo "Bootstrapping..."
+apt update
+apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+python3 -m pip install --upgrade pip setuptools wheel
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 811d258e..48c500c3 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -1,6 +1,10 @@
+import logging
import shutil
+import subprocess
from pathlib import Path
+from esd.error_handling.exceptions import BashCommandException
+
def clean_up(dirs: list[str], logging, ignore_errors=True):
"""
@@ -18,3 +22,32 @@ def clean_up(dirs: list[str], logging, ignore_errors=True):
raise e
else:
logging.info(f"{cleanup_dir} does not exist")
+
+
+def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+ exception=None, **kwargs):
+ try:
+ logger.debug(f'{debug_msg}: args: {args}')
+ return subprocess.run(args, **kwargs)
+ except subprocess.CalledProcessError as e:
+ if exception_msg is not None:
+ logger.error(f"{exception_msg}: {e}")
+ if exception is not None:
+ raise exception(f'{exception_msg} : {e}')
+ else:
+ return None
+
+
+def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
+ if not dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ git_path, dir
+ , check=True, logger=logger,
+ debug_msg=f'Cloned repository {repo_name}',
+ exception_msg=f'Failed to clone repository: {repo_name}',
+ exception=BashCommandException)
+ else:
+ logger.debug(f'Repository {repo_name} already cloned.')
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
new file mode 100644
index 00000000..79f8051f
--- /dev/null
+++ b/esd/error_handling/exceptions.py
@@ -0,0 +1,18 @@
+class SpackException(Exception):
+
+ def __init__(self, message):
+ super().__init__(message)
+ self.message = str(message)
+
+ def __str__(self):
+ return self.message
+
+class BashCommandException(SpackException):
+ """
+ To be thrown when an invalid input is received.
+ """
+
+class NoSpackEnvironmentException(SpackException):
+ """
+ To be thrown when an invalid input is received.
+ """
\ No newline at end of file
diff --git a/esd/model/SpackModel.py b/esd/model/SpackModel.py
new file mode 100644
index 00000000..4b065dba
--- /dev/null
+++ b/esd/model/SpackModel.py
@@ -0,0 +1,12 @@
+from pathlib import Path
+
+
+class SpackModel:
+ """"
+ Provides details about the spack environment
+ """
+
+ def __init__(self, env_name: str, path: Path, git_path: str = None):
+ self.env_name = env_name
+ self.path = path
+ self.git_path = git_path
diff --git a/esd/model/__init__.py b/esd/model/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
new file mode 100644
index 00000000..340b1b95
--- /dev/null
+++ b/esd/spack_manager/SpackManager.py
@@ -0,0 +1,193 @@
+import os
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.logger.logger_builder import get_logger
+from esd.model.SpackModel import SpackModel
+from esd.utils.utils import run_command, git_clone_repo
+
+
+class SpackManager(ABC):
+ """
+ This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
+ Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
+
+ Attributes:
+ -----------
+ env : SpackModel
+ spack environment details
+ repos : list[SpackModel]
+ upstream_instance : str
+ path to Spack instance to use as upstream (optional)
+ """
+
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
+ if repos is None:
+ repos = []
+ self.repos = repos
+ self.env = env
+ self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
+ self.install_dir.mkdir(parents=True, exist_ok=True)
+ self.env_path = None
+ if self.env and self.env.path:
+ self.env.path = self.env.path.resolve()
+ self.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = self.env.path / self.env.env_name
+ self.upstream_instance = upstream_instance
+ self.spack_dir = self.install_dir / "spack"
+ self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ self.logger = logger
+ self.system_name = system_name
+
+ @abstractmethod
+ def concretize_spack_env(self, force=True):
+ pass
+
+ @abstractmethod
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
+
+ def create_fetch_spack_environment(self):
+
+ if self.env.git_path:
+ git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
+ else:
+ os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+ check=True, logger=self.logger,
+ debug_msg=f"Created {self.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.env.env_name} spack environment",
+ exception=BashCommandException)
+
+ def setup_spack_env(self):
+ """
+ This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
+ """
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ if self.system_name:
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
+ os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_dir.exists() and self.spack_dir.is_dir():
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
+ bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
+ else:
+ self.logger.error(f'Invalid installation path: {self.spack_dir}')
+ # Restart the bash after adding environment variables
+ self.create_fetch_spack_environment()
+ if self.install_dir.exists():
+ for repo in self.repos:
+ repo_dir = self.install_dir / repo.path / repo.env_name
+ git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
+ if not self.spack_repo_exists(repo.env_name):
+ self.add_spack_repo(repo.path, repo.env_name)
+ self.logger.debug(f'Added spack repository {repo.env_name}')
+ else:
+ self.logger.debug(f'Spack repository {repo.env_name} already added')
+
+ def spack_repo_exists(self, repo_name: str) -> bool | None:
+ """Check if the given Spack repository exists."""
+ if self.env is None:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if {repo_name} exists')
+ if result is None:
+ return False
+ else:
+ if self.spack_env_exists():
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if repository {repo_name} was added')
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+ if result is None:
+ return False
+ return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
+
+ def spack_env_exists(self):
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if environment {self.env.env_name} exists')
+ if result is None:
+ return False
+ return True
+
+ def add_spack_repo(self, repo_path: Path, repo_name: str):
+ """Add the Spack repository if it does not exist."""
+ if self.spack_env_exists():
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ exception=BashCommandException)
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+
+ def get_spack_installed_version(self):
+ spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ capture_output=True, text=True, check=True,
+ logger=self.logger,
+ debug_msg=f"Getting spack version",
+ exception_msg=f"Error retrieving Spack version")
+ if spack_version:
+ return spack_version.stdout.strip().split()[0]
+ return None
+
+ def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+ try:
+ user = os.getlogin()
+ except OSError:
+ user = None
+
+ self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
+ if not self.spack_dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ "--branch", spack_version, spack_repo, self.spack_dir
+ , check=True, logger=self.logger)
+ self.logger.debug("Cloned spack")
+ else:
+ self.logger.debug("Spack already cloned.")
+
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ # ensure the file exists before opening it
+ if not os.path.exists(bashrc_path):
+ open(bashrc_path, "w").close()
+ # add spack setup commands to .bashrc
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
+ bashrc.write(f"source {self.spack_setup_script}\n")
+ self.logger.info("Added Spack PATH to .bashrc")
+ if user:
+ run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
+ debug_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
+ self.logger.info("Spack install completed")
+ # Restart Bash after the installation ends
+ os.system("exec bash")
+
+ # Configure upstream Spack instance if specified
+ if self.upstream_instance:
+ upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
+ with open(upstreams_yaml_path, "w") as file:
+ file.write(f"""upstreams:
+ upstream-spack-instance:
+ install_tree: {self.upstream_instance}/spack/opt/spack
+ """)
+ self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_manager/__init__.py b/esd/spack_manager/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
new file mode 100644
index 00000000..a2435839
--- /dev/null
+++ b/esd/spack_manager/enums/SpackManagerEnum.py
@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class SpackManagerEnum(Enum):
+ FROM_SCRATCH = "from_scratch",
+ FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/enums/__init__.py b/esd/spack_manager/enums/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
new file mode 100644
index 00000000..38151c6d
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerBuildCache.py
@@ -0,0 +1,19 @@
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerBuildCache(SpackManager):
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
+ super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
+
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
new file mode 100644
index 00000000..9728467f
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -0,0 +1,14 @@
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+class SpackManagerCreator:
+ @staticmethod
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
+ upstream_instance: str):
+ if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
+ return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
+ return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
+
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
new file mode 100644
index 00000000..5a79797c
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -0,0 +1,15 @@
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerScratch(SpackManager):
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
+ super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+ def concretize_spack_env(self, force=True):
+ pass
+
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_manager/factory/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/pyproject.toml b/pyproject.toml
index 757f370c..b6679ca1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,15 +1,15 @@
[build-system]
-requires = ["setuptools", "setuptools-scm"]
+requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "dedal"
+version = "0.1.0"
authors = [
{name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
{name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"},
]
description = "This package provides all the necessary tools to create an Ebrains Software Distribution environment"
-version = "0.1.0"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
@@ -18,6 +18,7 @@ dependencies = [
"ruamel.yaml",
"pytest",
"pytest-mock",
+ "pytest-ordering",
]
[tool.setuptools.data-files]
--
GitLab
From 93d07dbfff3742fcd5fe8f8a7949e288c3a4056c Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 21:32:38 +0200
Subject: [PATCH 03/11] esd-spack-installation: added concretization step and
tests
---
README.md | 1 +
dedal/tests/spack_from_scratch_test.py | 97 ++++++++++++++-----
dedal/tests/spack_install_test.py | 12 +--
dedal/tests/utils_test.py | 18 ++++
dedal/utils/utils.py | 4 +
esd/error_handling/exceptions.py | 14 ++-
.../factory/SpackManagerCreator.py | 10 +-
.../factory/SpackManagerScratch.py | 15 ++-
8 files changed, 132 insertions(+), 39 deletions(-)
diff --git a/README.md b/README.md
index 62e00c68..86ab9c2f 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,4 @@
# ~~Yashchiki~~Koutakia
For now, this repository provides helpers for the EBRAINS container image build flow.
+The lowest spack version which should be used with this library is v0.22.0
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index cca0d2e9..d1aca83c 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -4,62 +4,70 @@ import pytest
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+from esd.utils.utils import file_exists_and_not_empty
def test_spack_repo_exists_1():
- spack_manager = SpackManagerScratch()
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
-def test_spack_repo_exists_2():
- install_dir = Path('./install').resolve()
+def test_spack_repo_exists_2(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerScratch(env=env)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
with pytest.raises(NoSpackEnvironmentException):
spack_manager.spack_repo_exists(env.env_name)
-# def test_spack_repo_exists_3():
-# install_dir = Path('./install').resolve()
-# env = SpackModel('ebrains-spack-builds', install_dir)
-# spack_manager = SpackManagerScratch(env=env)
-# spack_manager.setup_spack_env()
-# assert spack_manager.spack_repo_exists(env.env_name) == False
+def test_spack_repo_exists_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == False
-def test_spack_from_scratch_setup_1():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_1(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
- spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == True
+ assert spack_manager.spack_repo_exists(env.env_name) == False
-def test_spack_from_scratch_setup_2():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_2(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True
-def test_spack_from_scratch_setup_3():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_3(tmp_path):
+ install_dir = tmp_path
env = SpackModel('new_env1', install_dir)
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
with pytest.raises(BashCommandException):
spack_manager.setup_spack_env()
-def test_spack_from_scratch_setup_4():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_4(tmp_path):
+ install_dir = tmp_path
env = SpackModel('new_env2', install_dir)
- spack_manager = SpackManagerScratch(env=env)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
assert spack_manager.spack_env_exists() == True
@@ -67,6 +75,47 @@ def test_spack_from_scratch_setup_4():
def test_spack_not_a_valid_repo():
env = SpackModel('ebrains-spack-builds', Path(), None)
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo],
+ system_name='ebrainslab')
with pytest.raises(NoSpackEnvironmentException):
spack_manager.add_spack_repo(repo.path, repo.env_name)
+
+
+def test_spack_from_scratch_concretize_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_2(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == False
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py
index 34f68323..9a32d5c7 100644
--- a/dedal/tests/spack_install_test.py
+++ b/dedal/tests/spack_install_test.py
@@ -4,18 +4,18 @@ from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCa
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-# we need this test to run first so that spack is installed only once
+SPACK_VERSION = "0.22.0"
+
+# we need this test to run first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version="v0.21.1")
+ spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
def test_spack_install_buildcache():
spack_manager = SpackManagerBuildCache()
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/utils_test.py b/dedal/tests/utils_test.py
index 14795726..256b8218 100644
--- a/dedal/tests/utils_test.py
+++ b/dedal/tests/utils_test.py
@@ -61,3 +61,21 @@ def test_clean_up_nonexistent_dirs(mocker):
for dir_path in nonexistent_dirs:
mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
+
+
+def test_file_does_not_exist(tmp_path: Path):
+ non_existent_file = tmp_path / "non_existent.txt"
+ assert not file_exists_and_not_empty(non_existent_file)
+
+
+def test_file_exists_but_empty(tmp_path: Path):
+ empty_file = tmp_path / "empty.txt"
+ # Create an empty file
+ empty_file.touch()
+ assert not file_exists_and_not_empty(empty_file)
+
+
+def test_file_exists_and_not_empty(tmp_path: Path):
+ non_empty_file = tmp_path / "non_empty.txt"
+ non_empty_file.write_text("Some content")
+ assert file_exists_and_not_empty(non_empty_file)
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 48c500c3..173347d0 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -51,3 +51,7 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
exception=BashCommandException)
else:
logger.debug(f'Repository {repo_name} already cloned.')
+
+
+def file_exists_and_not_empty(file: Path) -> bool:
+ return file.is_file() and file.stat().st_size > 0
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 79f8051f..d6de666b 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -7,12 +7,20 @@ class SpackException(Exception):
def __str__(self):
return self.message
+
class BashCommandException(SpackException):
"""
- To be thrown when an invalid input is received.
+ To be thrown when a bash command has failed
"""
+
class NoSpackEnvironmentException(SpackException):
"""
- To be thrown when an invalid input is received.
- """
\ No newline at end of file
+ To be thrown when an operation on a spack environment is executed without the environment being activated or existent
+ """
+
+
+class SpackConcertizeException(SpackException):
+ """
+ To be thrown when the spack concretization step fails
+ """
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
index 9728467f..6eb26a04 100644
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -1,3 +1,4 @@
+from esd.model.SpackModel import SpackModel
from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
@@ -5,10 +6,9 @@ from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
class SpackManagerCreator:
@staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
- upstream_instance: str):
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ return SpackManagerScratch(env, repos, upstream_instance, system_name)
elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
-
+ return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 5a79797c..2ec22705 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -1,6 +1,8 @@
+from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
from esd.spack_manager.SpackManager import SpackManager
from esd.logger.logger_builder import get_logger
+from esd.utils.utils import run_command
class SpackManagerScratch(SpackManager):
@@ -9,7 +11,18 @@ class SpackManagerScratch(SpackManager):
super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
def concretize_spack_env(self, force=True):
- pass
+ force = '--force' if force else ''
+ if self.spack_env_exists():
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Concertization step for {self.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.env.env_name}',
+ exception=SpackConcertizeException)
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
pass
--
GitLab
From d0a57d8fffcb8286c5f4eb220a9590b55002f8b5 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 6 Feb 2025 10:10:33 +0200
Subject: [PATCH 04/11] esd-spack-installation: additional methods
---
.gitlab-ci.yml | 1 +
dedal/tests/spack_from_scratch_test.py | 83 +++++++++++++++++-----
esd/error_handling/exceptions.py | 9 ++-
esd/spack_manager/SpackManager.py | 51 +++++++++----
esd/spack_manager/wrapper/__init__.py | 0
esd/spack_manager/wrapper/spack_wrapper.py | 15 ++++
6 files changed, 124 insertions(+), 35 deletions(-)
create mode 100644 esd/spack_manager/wrapper/__init__.py
create mode 100644 esd/spack_manager/wrapper/spack_wrapper.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4f15b9ab..bd5669bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,6 +6,7 @@ variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
+
build-wheel:
stage: build
tags:
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index d1aca83c..2131e7df 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -1,14 +1,13 @@
+import os
from pathlib import Path
-
import pytest
-
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
from esd.utils.utils import file_exists_and_not_empty
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
def test_spack_repo_exists_1():
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
@@ -34,9 +33,9 @@ def test_spack_repo_exists_3(tmp_path):
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- system_name='ebrainslab')
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == False
@@ -44,11 +43,11 @@ def test_spack_from_scratch_setup_1(tmp_path):
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True
@@ -58,8 +57,8 @@ def test_spack_from_scratch_setup_3(tmp_path):
env = SpackModel('new_env1', install_dir)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ repos=[repo, repo],
+ system_name='ebrainslab')
with pytest.raises(BashCommandException):
spack_manager.setup_spack_env()
@@ -76,8 +75,8 @@ def test_spack_not_a_valid_repo():
env = SpackModel('ebrains-spack-builds', Path(), None)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo],
- system_name='ebrainslab')
+ repos=[repo],
+ system_name='ebrainslab')
with pytest.raises(NoSpackEnvironmentException):
spack_manager.add_spack_repo(repo.path, repo.env_name)
@@ -85,7 +84,7 @@ def test_spack_not_a_valid_repo():
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -98,7 +97,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -111,11 +110,59 @@ def test_spack_from_scratch_concretize_2(tmp_path):
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
concretization_file_path = spack_manager.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == False
+
+
+def test_spack_from_scratch_concretize_4(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_5(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_6(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ repo = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_7(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ 'https://gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ repo = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index d6de666b..9d11b5fa 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -14,13 +14,18 @@ class BashCommandException(SpackException):
"""
-class NoSpackEnvironmentException(SpackException):
+class NoSpackEnvironmentException(BashCommandException):
"""
To be thrown when an operation on a spack environment is executed without the environment being activated or existent
"""
-class SpackConcertizeException(SpackException):
+class SpackConcertizeException(BashCommandException):
"""
To be thrown when the spack concretization step fails
"""
+
+class SpackInstallPackagesException(BashCommandException):
+ """
+ To be thrown when the spack fails to install spack packages
+ """
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index 340b1b95..a7f46c27 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -1,10 +1,13 @@
import os
+import re
from abc import ABC, abstractmethod
from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
+ SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
+from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
from esd.utils.utils import run_command, git_clone_repo
@@ -103,10 +106,10 @@ class SpackManager(ABC):
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -124,18 +127,35 @@ class SpackManager(ABC):
return False
return True
+ @no_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
- exception=BashCommandException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ exception=BashCommandException)
+
+ @no_spack_env
+ def get_compiler_version(self):
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ check=True, logger=self.logger,
+ capture_output=True, text=True,
+ debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ exception=BashCommandException)
+ # todo add error handling and tests
+ if result.stdout is None:
+ self.logger.debug('No gcc found for {self.env.env_name}')
+ return None
+
+ # Find the first occurrence of a GCC compiler using regex
+ match = re.search(r"gcc@([\d\.]+)", result.stdout)
+ gcc_version = match.group(1)
+ self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
@@ -147,6 +167,7 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
+
def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/spack_manager/wrapper/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/spack_manager/wrapper/spack_wrapper.py
new file mode 100644
index 00000000..d075a317
--- /dev/null
+++ b/esd/spack_manager/wrapper/spack_wrapper.py
@@ -0,0 +1,15 @@
+import functools
+
+from esd.error_handling.exceptions import NoSpackEnvironmentException
+
+
+def no_spack_env(method):
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.spack_env_exists():
+ return method(self, *args, **kwargs) # Call the method with 'self'
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+
+ return wrapper
--
GitLab
From eee415e3245e7232e71f3ac11630b407f01d5390 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Thu, 6 Feb 2025 18:14:49 +0200
Subject: [PATCH 05/11] esd-spack-installation: fixed passing access token to
tests; added log file for spack install step
---
dedal/tests/spack_from_scratch_test.py | 37 +++++++++-------------
esd/spack_manager/SpackManager.py | 25 +++++++++++----
esd/spack_manager/wrapper/spack_wrapper.py | 2 +-
3 files changed, 35 insertions(+), 29 deletions(-)
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index 2131e7df..e5627409 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -8,6 +8,10 @@ from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
from esd.utils.utils import file_exists_and_not_empty
SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
+
+
def test_spack_repo_exists_1():
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
@@ -32,8 +36,7 @@ def test_spack_repo_exists_3(tmp_path):
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
system_name='ebrainslab')
spack_manager.setup_spack_env()
@@ -42,8 +45,7 @@ def test_spack_from_scratch_setup_1(tmp_path):
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
repos=[repo, repo],
@@ -83,8 +85,7 @@ def test_spack_not_a_valid_repo():
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -96,8 +97,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -109,8 +109,7 @@ def test_spack_from_scratch_concretize_2(tmp_path):
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
repos=[repo, repo],
@@ -122,8 +121,7 @@ def test_spack_from_scratch_concretize_3(tmp_path):
def test_spack_from_scratch_concretize_4(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=False)
@@ -133,8 +131,7 @@ def test_spack_from_scratch_concretize_4(tmp_path):
def test_spack_from_scratch_concretize_5(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=True)
@@ -144,10 +141,8 @@ def test_spack_from_scratch_concretize_5(tmp_path):
def test_spack_from_scratch_concretize_6(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
- repo = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=False)
@@ -157,10 +152,8 @@ def test_spack_from_scratch_concretize_6(tmp_path):
def test_spack_from_scratch_concretize_7(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- 'https://gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
- repo = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=True)
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index a7f46c27..d534b30a 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -2,12 +2,13 @@ import os
import re
from abc import ABC, abstractmethod
from pathlib import Path
+from tabnanny import check
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
+from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
from esd.utils.utils import run_command, git_clone_repo
@@ -53,7 +54,6 @@ class SpackManager(ABC):
pass
def create_fetch_spack_environment(self):
-
if self.env.git_path:
git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
else:
@@ -127,7 +127,7 @@ class SpackManager(ABC):
return False
return True
- @no_spack_env
+ @check_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
@@ -137,7 +137,7 @@ class SpackManager(ABC):
exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
exception=BashCommandException)
- @no_spack_env
+ @check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
@@ -167,8 +167,21 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
-
- def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, fresh=False):
+ signed = '' if signed else '--no-check-signature'
+ fresh = '--fresh' if fresh else ''
+ with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack install --env {self.env.env_name} -v {signed} --j {jobs} {fresh}',
+ stdout=log_file,
+ capture_output=True, text=True, check=True,
+ logger=self.logger,
+ debug_msg=f"Installing spack packages for {self.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ exception=SpackInstallPackagesException)
+
+ def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/spack_manager/wrapper/spack_wrapper.py
index d075a317..c2f9c116 100644
--- a/esd/spack_manager/wrapper/spack_wrapper.py
+++ b/esd/spack_manager/wrapper/spack_wrapper.py
@@ -3,7 +3,7 @@ import functools
from esd.error_handling.exceptions import NoSpackEnvironmentException
-def no_spack_env(method):
+def check_spack_env(method):
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
if self.spack_env_exists():
--
GitLab
From 8b4b484ae4ac77a7a8f8a01bd8826e5d00759979 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 7 Feb 2025 18:50:52 +0200
Subject: [PATCH 06/11] esd-spack-installation: spack install method; major
refactoring; fixed bug on updating env vars in .bashrc
---
dedal/build_cache/BuildCacheManager.py | 62 ++---
.../__init__.py => dedal/cli/SpackManager.py | 0
dedal/tests/spack_from_scratch_test.py | 211 +++++++++++-------
dedal/tests/spack_install_test.py | 21 +-
dedal/tests/testing_variables.py | 6 +
dedal/utils/utils.py | 47 +++-
esd/configuration/SpackConfig.py | 25 +++
.../enums => configuration}/__init__.py | 0
.../factory => error_handling}/__init__.py | 0
.../{SpackModel.py => SpackDescriptor.py} | 5 +-
.../SpackOperation.py} | 153 ++++++-------
esd/spack_factory/SpackOperationCreator.py | 14 ++
esd/spack_factory/SpackOperationUseCache.py | 19 ++
.../wrapper => spack_factory}/__init__.py | 0
esd/spack_manager/enums/SpackManagerEnum.py | 6 -
.../factory/SpackManagerBuildCache.py | 19 --
.../factory/SpackManagerCreator.py | 14 --
.../factory/SpackManagerScratch.py | 28 ---
esd/wrapper/__init__.py | 0
.../wrapper/spack_wrapper.py | 0
20 files changed, 350 insertions(+), 280 deletions(-)
rename esd/spack_manager/__init__.py => dedal/cli/SpackManager.py (100%)
create mode 100644 dedal/tests/testing_variables.py
create mode 100644 esd/configuration/SpackConfig.py
rename esd/{spack_manager/enums => configuration}/__init__.py (100%)
rename esd/{spack_manager/factory => error_handling}/__init__.py (100%)
rename esd/model/{SpackModel.py => SpackDescriptor.py} (57%)
rename esd/{spack_manager/SpackManager.py => spack_factory/SpackOperation.py} (56%)
create mode 100644 esd/spack_factory/SpackOperationCreator.py
create mode 100644 esd/spack_factory/SpackOperationUseCache.py
rename esd/{spack_manager/wrapper => spack_factory}/__init__.py (100%)
delete mode 100644 esd/spack_manager/enums/SpackManagerEnum.py
delete mode 100644 esd/spack_manager/factory/SpackManagerBuildCache.py
delete mode 100644 esd/spack_manager/factory/SpackManagerCreator.py
delete mode 100644 esd/spack_manager/factory/SpackManagerScratch.py
create mode 100644 esd/wrapper/__init__.py
rename esd/{spack_manager => }/wrapper/spack_wrapper.py (100%)
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index dbd50bf9..55fa10cb 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -12,48 +12,51 @@ class BuildCacheManager(BuildCacheManagerInterface):
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', insecure=False):
- self.logger = get_logger(__name__, BuildCacheManager.__name__)
- self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.registry_project = os.environ.get("REGISTRY_PROJECT")
+ def __init__(self, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
+ auth_backend='basic',
+ insecure=False):
+ self._logger = get_logger(__name__, BuildCacheManager.__name__)
+ self._registry_project = registry_project
- self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
- self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
+ self._registry_username = registry_username
+ self._registry_password = registry_password
- self.registry_host = str(os.environ.get("REGISTRY_HOST"))
+ self._registry_host = registry_host
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
- self.client.login(username=self._registry_username, password=self._registry_password)
- self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
+ self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend,
+ insecure=insecure)
+ self._client.login(username=self._registry_username, password=self._registry_password)
+ self.cache_version = cache_version
+ self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/{self.cache_version}'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / out_dir
+ build_cache_path = out_dir.resolve()
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- self.logger.error(f"Path {build_cache_path} not found.")
+ self._logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
- target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
+ target = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{str(sub_path.name)}"
try:
- self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
- self.client.push(
+ self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self._client.push(
files=[str(sub_path)],
target=target,
# save in manifest the relative path for reconstruction
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self.logger.info(f"Successfully pushed {sub_path.env_name}")
+ self._logger.info(f"Successfully pushed {sub_path.name}")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"An error occurred while pushing: {e}")
# todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
# clean_up([str(build_cache_path)], self.logger)
@@ -63,37 +66,38 @@ class BuildCacheManager(BuildCacheManagerInterface):
This method retrieves all tags from an OCI Registry
"""
try:
- return self.client.get_tags(self.oci_registry_path)
+ return self._client.get_tags(self._oci_registry_path)
except Exception as e:
- self.logger.error(f"Failed to list tags: {e}")
+ self._logger.error(f"Failed to list tags: {e}")
return None
def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / in_dir
+ build_cache_path = in_dir.resolve()
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
+ ref = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
+ self._client.get_manifest(
+ f'{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}')[
'annotations'][
'path']
try:
- self.client.pull(
+ self._client.pull(
ref,
# missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- self.logger.info(f"Successfully pulled artifact {tag}.")
+ self._logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"Failed to pull artifact {tag} : {e}")
def delete(self):
@@ -106,8 +110,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
tags = self.list_tags()
if tags is not None:
try:
- self.client.delete_tags(self.oci_registry_path, tags)
- self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
+ self._client.delete_tags(self._oci_registry_path, tags)
+ self._logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- self.logger.error(
+ self._logger.error(
f"Failed to delete artifacts: {e}")
diff --git a/esd/spack_manager/__init__.py b/dedal/cli/SpackManager.py
similarity index 100%
rename from esd/spack_manager/__init__.py
rename to dedal/cli/SpackManager.py
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index e5627409..cdc405e7 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -1,161 +1,204 @@
-import os
from pathlib import Path
import pytest
+from esd.configuration.SpackConfig import SpackConfig
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
-from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
+from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
+from esd.model.SpackDescriptor import SpackDescriptor
+from esd.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
from esd.utils.utils import file_exists_and_not_empty
-SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
-test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
-ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
-
-
def test_spack_repo_exists_1():
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
- assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
+ spack_operation = SpackOperationCreator.get_spack_operator()
+ spack_operation.install_spack()
+ assert spack_operation.spack_repo_exists('ebrains-spack-builds') == False
def test_spack_repo_exists_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
with pytest.raises(NoSpackEnvironmentException):
- spack_manager.spack_repo_exists(env.env_name)
+ spack_operation.spack_repo_exists(env.env_name)
def test_spack_repo_exists_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == False
+ env = SpackDescriptor('ebrains-spack-builds', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ print(spack_operation.get_spack_installed_version())
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == False
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == False
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == False
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == True
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == True
def test_spack_from_scratch_setup_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('new_env1', install_dir)
+ env = SpackDescriptor('new_env1', install_dir)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
with pytest.raises(BashCommandException):
- spack_manager.setup_spack_env()
+ spack_operation.setup_spack_env()
def test_spack_from_scratch_setup_4(tmp_path):
install_dir = tmp_path
- env = SpackModel('new_env2', install_dir)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- assert spack_manager.spack_env_exists() == True
+ env = SpackDescriptor('new_env2', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_env_exists() == True
def test_spack_not_a_valid_repo():
- env = SpackModel('ebrains-spack-builds', Path(), None)
+ env = SpackDescriptor('ebrains-spack-builds', Path(), None)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo],
- system_name='ebrainslab')
- with pytest.raises(NoSpackEnvironmentException):
- spack_manager.add_spack_repo(repo.path, repo.env_name)
+ config = SpackConfig(env=env, system_name='ebrainslab')
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ with pytest.raises(BashCommandException):
+ spack_operation.add_spack_repo(repo.path, repo.env_name)
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=False)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == False
def test_spack_from_scratch_concretize_4(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=False)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_5(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_6(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=False)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_7(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_install(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
+ install_result = spack_operation.install_packages(jobs=2, signed=False, fresh=True, debug=False)
+ assert install_result.returncode == 0
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py
index 9a32d5c7..28f8268e 100644
--- a/dedal/tests/spack_install_test.py
+++ b/dedal/tests/spack_install_test.py
@@ -1,21 +1,12 @@
import pytest
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.tests.testing_variables import SPACK_VERSION
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-SPACK_VERSION = "0.22.0"
-
-# we need this test to run first so that spack is installed only once for all the tests
+# run this test first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
- spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
- installed_spack_version = spack_manager.get_spack_installed_version()
- assert SPACK_VERSION == installed_spack_version
-
-
-def test_spack_install_buildcache():
- spack_manager = SpackManagerBuildCache()
- installed_spack_version = spack_manager.get_spack_installed_version()
+ spack_operation = SpackOperation()
+ spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
+ installed_spack_version = spack_operation.get_spack_installed_version()
assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/testing_variables.py b/dedal/tests/testing_variables.py
new file mode 100644
index 00000000..ab95bfa1
--- /dev/null
+++ b/dedal/tests/testing_variables.py
@@ -0,0 +1,6 @@
+import os
+
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
+SPACK_VERSION = "0.22.0"
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 173347d0..033cbc54 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -1,33 +1,35 @@
import logging
+import os
import shutil
import subprocess
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException
+import re
-def clean_up(dirs: list[str], logging, ignore_errors=True):
+def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
for cleanup_dir in dirs:
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
- logging.info(f"Removing {cleanup_dir}")
+ logger.info(f"Removing {cleanup_dir}")
try:
shutil.rmtree(Path(cleanup_dir))
except OSError as e:
- logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ logger.error(f"Failed to remove {cleanup_dir}: {e}")
if not ignore_errors:
raise e
else:
- logging.info(f"{cleanup_dir} does not exist")
+ logger.info(f"{cleanup_dir} does not exist")
-def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
exception=None, **kwargs):
try:
- logger.debug(f'{debug_msg}: args: {args}')
+ logger.info(f'{info_msg}: args: {args}')
return subprocess.run(args, **kwargs)
except subprocess.CalledProcessError as e:
if exception_msg is not None:
@@ -46,12 +48,41 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
"-c", "feature.manyFiles=true",
git_path, dir
, check=True, logger=logger,
- debug_msg=f'Cloned repository {repo_name}',
+ info_msg=f'Cloned repository {repo_name}',
exception_msg=f'Failed to clone repository: {repo_name}',
exception=BashCommandException)
else:
- logger.debug(f'Repository {repo_name} already cloned.')
+ logger.info(f'Repository {repo_name} already cloned.')
def file_exists_and_not_empty(file: Path) -> bool:
return file.is_file() and file.stat().st_size > 0
+
+
+def log_command(results, log_file: str):
+ with open(log_file, "w") as log_file:
+ log_file.write(results.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(results.stderr)
+
+
+def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
+ logger: logging = logging.getLogger(__name__)):
+ """Update or add an environment variable in ~/.bashrc."""
+ with open(bashrc_path, "r") as file:
+ lines = file.readlines()
+ pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
+ updated = False
+ # Modify the existing variable if found
+ for i, line in enumerate(lines):
+ if pattern.match(line):
+ lines[i] = f'export {var_name}={value}\n'
+ updated = True
+ break
+ if not updated:
+ lines.append(f'\nexport {var_name}={value}\n')
+ logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
+ else:
+ logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
+ with open(bashrc_path, "w") as file:
+ file.writelines(lines)
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
new file mode 100644
index 00000000..93a2e874
--- /dev/null
+++ b/esd/configuration/SpackConfig.py
@@ -0,0 +1,25 @@
+import os
+from pathlib import Path
+from esd.model import SpackDescriptor
+
+
+class SpackConfig:
+ def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
+ install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
+ concretization_dir: Path = None, buildcache_dir: Path = None):
+ self.env = env
+ if repos is None:
+ self.repos = []
+ else:
+ self.repos = repos
+ self.install_dir = install_dir
+ self.upstream_instance = upstream_instance
+ self.system_name = system_name
+ self.concretization_dir = concretization_dir
+ self.buildcache_dir = buildcache_dir
+
+ def add_repo(self, repo: SpackDescriptor):
+ if self.repos is None:
+ self.repos = []
+ else:
+ self.repos.append(repo)
diff --git a/esd/spack_manager/enums/__init__.py b/esd/configuration/__init__.py
similarity index 100%
rename from esd/spack_manager/enums/__init__.py
rename to esd/configuration/__init__.py
diff --git a/esd/spack_manager/factory/__init__.py b/esd/error_handling/__init__.py
similarity index 100%
rename from esd/spack_manager/factory/__init__.py
rename to esd/error_handling/__init__.py
diff --git a/esd/model/SpackModel.py b/esd/model/SpackDescriptor.py
similarity index 57%
rename from esd/model/SpackModel.py
rename to esd/model/SpackDescriptor.py
index 4b065dba..70e484fb 100644
--- a/esd/model/SpackModel.py
+++ b/esd/model/SpackDescriptor.py
@@ -1,12 +1,13 @@
+import os
from pathlib import Path
-class SpackModel:
+class SpackDescriptor:
""""
Provides details about the spack environment
"""
- def __init__(self, env_name: str, path: Path, git_path: str = None):
+ def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
self.env_name = env_name
self.path = path
self.git_path = git_path
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_factory/SpackOperation.py
similarity index 56%
rename from esd/spack_manager/SpackManager.py
rename to esd/spack_factory/SpackOperation.py
index d534b30a..29f44f49 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -1,68 +1,59 @@
import os
import re
+import subprocess
from abc import ABC, abstractmethod
from pathlib import Path
-from tabnanny import check
-
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException
+ SpackInstallPackagesException, SpackConcertizeException
from esd.logger.logger_builder import get_logger
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo
+from esd.configuration.SpackConfig import SpackConfig
+from esd.tests.testing_variables import SPACK_VERSION
+from esd.wrapper.spack_wrapper import check_spack_env
+from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-class SpackManager(ABC):
+class SpackOperation(ABC):
"""
This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
Attributes:
-----------
- env : SpackModel
+ env : SpackDescriptor
spack environment details
- repos : list[SpackModel]
+ repos : list[SpackDescriptor]
upstream_instance : str
path to Spack instance to use as upstream (optional)
"""
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
- if repos is None:
- repos = []
- self.repos = repos
- self.env = env
- self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
- self.install_dir.mkdir(parents=True, exist_ok=True)
- self.env_path = None
- if self.env and self.env.path:
- self.env.path = self.env.path.resolve()
- self.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = self.env.path / self.env.env_name
- self.upstream_instance = upstream_instance
- self.spack_dir = self.install_dir / "spack"
- self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
+ self.spack_config = spack_config
+ self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.spack_config.install_dir / 'spack'
+ self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
self.logger = logger
- self.system_name = system_name
+ if self.spack_config.env and spack_config.env.path:
+ self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
@abstractmethod
def concretize_spack_env(self, force=True):
pass
- @abstractmethod
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
-
def create_fetch_spack_environment(self):
- if self.env.git_path:
- git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
+ if self.spack_config.env.git_path:
+ git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
+ self.spack_config.env.git_path,
+ logger=self.logger)
else:
- os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
check=True, logger=self.logger,
- debug_msg=f"Created {self.env.env_name} spack environment",
- exception_msg=f"Failed to create {self.env.env_name} spack environment",
+ info_msg=f"Created {self.spack_config.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
exception=BashCommandException)
def setup_spack_env(self):
@@ -70,22 +61,20 @@ class SpackManager(ABC):
This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
"""
bashrc_path = os.path.expanduser("~/.bashrc")
- if self.system_name:
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
- os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_config.system_name:
+ set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
+ os.environ['SYSTEMNAME'] = self.spack_config.system_name
if self.spack_dir.exists() and self.spack_dir.is_dir():
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
- bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
else:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
# Restart the bash after adding environment variables
self.create_fetch_spack_environment()
- if self.install_dir.exists():
- for repo in self.repos:
- repo_dir = self.install_dir / repo.path / repo.env_name
+ if self.spack_config.install_dir.exists():
+ for repo in self.spack_config.repos:
+ repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
if not self.spack_repo_exists(repo.env_name):
self.add_spack_repo(repo.path, repo.env_name)
@@ -95,21 +84,21 @@ class SpackManager(ABC):
def spack_repo_exists(self, repo_name: str) -> bool | None:
"""Check if the given Spack repository exists."""
- if self.env is None:
+ if self.spack_config.env is None:
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if {repo_name} exists')
+ info_msg=f'Checking if {repo_name} exists')
if result is None:
return False
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ f'{self.spack_command_on_env} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ info_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -119,10 +108,10 @@ class SpackManager(ABC):
def spack_env_exists(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+ self.spack_command_on_env,
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if environment {self.env.env_name} exists')
+ info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
if result is None:
return False
return True
@@ -131,20 +120,20 @@ class SpackManager(ABC):
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
exception=BashCommandException)
@check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ f'{self.spack_command_on_env} && spack compiler list',
check=True, logger=self.logger,
capture_output=True, text=True,
- debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
- exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
exception=BashCommandException)
# todo add error handling and tests
if result.stdout is None:
@@ -154,34 +143,48 @@ class SpackManager(ABC):
# Find the first occurrence of a GCC compiler using regex
match = re.search(r"gcc@([\d\.]+)", result.stdout)
gcc_version = match.group(1)
- self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
capture_output=True, text=True, check=True,
logger=self.logger,
- debug_msg=f"Getting spack version",
+ info_msg=f"Getting spack version",
exception_msg=f"Error retrieving Spack version")
if spack_version:
return spack_version.stdout.strip().split()[0]
return None
@check_spack_env
- def install_packages(self, jobs: int, signed=True, fresh=False):
+ def concretize_spack_env(self, force=True):
+ force = '--force' if force else ''
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Concertization step for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
+ exception=SpackConcertizeException)
+
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
fresh = '--fresh' if fresh else ''
- with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack install --env {self.env.env_name} -v {signed} --j {jobs} {fresh}',
- stdout=log_file,
- capture_output=True, text=True, check=True,
- logger=self.logger,
- debug_msg=f"Installing spack packages for {self.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.env.env_name}",
- exception=SpackInstallPackagesException)
-
- def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
+ exception=SpackInstallPackagesException)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
+ return install_result
+
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
@@ -210,18 +213,18 @@ class SpackManager(ABC):
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
- debug_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
+ info_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
self.logger.info("Spack install completed")
# Restart Bash after the installation ends
os.system("exec bash")
# Configure upstream Spack instance if specified
- if self.upstream_instance:
+ if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
with open(upstreams_yaml_path, "w") as file:
file.write(f"""upstreams:
upstream-spack-instance:
- install_tree: {self.upstream_instance}/spack/opt/spack
+ install_tree: {self.spack_config.upstream_instance}/spack/opt/spack
""")
self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_factory/SpackOperationCreator.py b/esd/spack_factory/SpackOperationCreator.py
new file mode 100644
index 00000000..8369c5ca
--- /dev/null
+++ b/esd/spack_factory/SpackOperationCreator.py
@@ -0,0 +1,14 @@
+from esd.configuration.SpackConfig import SpackConfig
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+
+
+class SpackOperationCreator:
+ @staticmethod
+ def get_spack_operator(spack_config: SpackConfig = None):
+ if spack_config is None:
+ return SpackOperation(SpackConfig())
+ elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
+ return SpackOperation(spack_config)
+ else:
+ return SpackOperationUseCache(spack_config)
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
new file mode 100644
index 00000000..15a3822f
--- /dev/null
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -0,0 +1,19 @@
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationUseCache(SpackOperation):
+ """
+ This class uses caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ super().__init__(spack_config, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/spack_factory/__init__.py
similarity index 100%
rename from esd/spack_manager/wrapper/__init__.py
rename to esd/spack_factory/__init__.py
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
deleted file mode 100644
index a2435839..00000000
--- a/esd/spack_manager/enums/SpackManagerEnum.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from enum import Enum
-
-
-class SpackManagerEnum(Enum):
- FROM_SCRATCH = "from_scratch",
- FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
deleted file mode 100644
index 38151c6d..00000000
--- a/esd/spack_manager/factory/SpackManagerBuildCache.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-
-
-class SpackManagerBuildCache(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- def concretize_spack_env(self, force=True):
- pass
-
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
deleted file mode 100644
index 6eb26a04..00000000
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-
-class SpackManagerCreator:
- @staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env, repos, upstream_instance, system_name)
- elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
deleted file mode 100644
index 2ec22705..00000000
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-from esd.utils.utils import run_command
-
-
-class SpackManagerScratch(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def concretize_spack_env(self, force=True):
- force = '--force' if force else ''
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Concertization step for {self.env.env_name}',
- exception_msg=f'Failed the concertization step for {self.env.env_name}',
- exception=SpackConcertizeException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
-
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
diff --git a/esd/wrapper/__init__.py b/esd/wrapper/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/wrapper/spack_wrapper.py
similarity index 100%
rename from esd/spack_manager/wrapper/spack_wrapper.py
rename to esd/wrapper/spack_wrapper.py
--
GitLab
From ebda7beeef6e7460c6eb129a37c6eb0e503edaf4 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 14 Feb 2025 11:28:46 +0200
Subject: [PATCH 07/11] esd-spack-installation: added additional spack commands
---
esd/configuration/GpgConfig.py | 7 +++
esd/configuration/SpackConfig.py | 11 +++-
esd/error_handling/exceptions.py | 10 ++++
esd/spack_factory/SpackOperation.py | 61 +++++++++++++++++----
esd/spack_factory/SpackOperationUseCache.py | 15 ++++-
5 files changed, 91 insertions(+), 13 deletions(-)
create mode 100644 esd/configuration/GpgConfig.py
diff --git a/esd/configuration/GpgConfig.py b/esd/configuration/GpgConfig.py
new file mode 100644
index 00000000..a8f0c2d3
--- /dev/null
+++ b/esd/configuration/GpgConfig.py
@@ -0,0 +1,7 @@
+class GpgConfig:
+ """
+ Configuration for gpg key used by spack
+ """
+ def __init__(self, gpg_name='example', gpg_mail='example@example.com'):
+ self.name = gpg_name
+ self.mail = gpg_mail
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
index 93a2e874..b6178760 100644
--- a/esd/configuration/SpackConfig.py
+++ b/esd/configuration/SpackConfig.py
@@ -1,22 +1,31 @@
import os
from pathlib import Path
+
+from esd.configuration.GpgConfig import GpgConfig
from esd.model import SpackDescriptor
class SpackConfig:
def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
- concretization_dir: Path = None, buildcache_dir: Path = None):
+ concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None):
self.env = env
if repos is None:
self.repos = []
else:
self.repos = repos
self.install_dir = install_dir
+ if self.install_dir:
+ os.makedirs(self.install_dir, exist_ok=True)
self.upstream_instance = upstream_instance
self.system_name = system_name
self.concretization_dir = concretization_dir
+ if self.concretization_dir:
+ os.makedirs(self.concretization_dir, exist_ok=True)
self.buildcache_dir = buildcache_dir
+ if self.buildcache_dir:
+ os.makedirs(self.buildcache_dir, exist_ok=True)
+ self.gpg = gpg
def add_repo(self, repo: SpackDescriptor):
if self.repos is None:
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 9d11b5fa..0256f886 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -29,3 +29,13 @@ class SpackInstallPackagesException(BashCommandException):
"""
To be thrown when the spack fails to install spack packages
"""
+
+class SpackMirrorException(BashCommandException):
+ """
+ To be thrown when the spack add mirror command fails
+ """
+
+class SpackGpgException(BashCommandException):
+ """
+ To be thrown when the spack fails to create gpg keys
+ """
diff --git a/esd/spack_factory/SpackOperation.py b/esd/spack_factory/SpackOperation.py
index 29f44f49..a0b21a1c 100644
--- a/esd/spack_factory/SpackOperation.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -1,10 +1,9 @@
import os
import re
import subprocess
-from abc import ABC, abstractmethod
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException, SpackConcertizeException
+ SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
from esd.logger.logger_builder import get_logger
from esd.configuration.SpackConfig import SpackConfig
from esd.tests.testing_variables import SPACK_VERSION
@@ -12,7 +11,7 @@ from esd.wrapper.spack_wrapper import check_spack_env
from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-class SpackOperation(ABC):
+class SpackOperation:
"""
This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
@@ -38,10 +37,6 @@ class SpackOperation(ABC):
self.env_path = spack_config.env.path / spack_config.env.env_name
self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
- @abstractmethod
- def concretize_spack_env(self, force=True):
- pass
-
def create_fetch_spack_environment(self):
if self.spack_config.env.git_path:
git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
@@ -65,8 +60,10 @@ class SpackOperation(ABC):
set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
os.environ['SYSTEMNAME'] = self.spack_config.system_name
if self.spack_dir.exists() and self.spack_dir.is_dir():
- set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
- set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
else:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
@@ -160,13 +157,55 @@ class SpackOperation(ABC):
def concretize_spack_env(self, force=True):
force = '--force' if force else ''
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ f'{self.spack_command_on_env} && spack concretize {force}',
check=True,
- capture_output=True, text=True, logger=self.logger,
+ logger=self.logger,
info_msg=f'Concertization step for {self.spack_config.env.env_name}',
exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
exception=SpackConcertizeException)
+ def create_gpg_keys(self):
+ if self.spack_config.gpg:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.env_name}',
+ exception=SpackGpgException)
+ else:
+ raise SpackGpgException('No GPG configuration was defined is spack configuration')
+
+ def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
+ autopush = '--autopush' if autopush else ''
+ signed = '--signed' if signed else ''
+ if global_mirror:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException)
+ else:
+ check_spack_env(
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException))
+
+ def remove_mirror(self, mirror_name: str):
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Removing mirror {mirror_name}',
+ exception_msg=f'Failed to remove mirror {mirror_name}',
+ exception=SpackMirrorException)
+
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
index 15a3822f..313522d2 100644
--- a/esd/spack_factory/SpackOperationUseCache.py
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -1,3 +1,5 @@
+import os
+from esd.build_cache.BuildCacheManager import BuildCacheManager
from esd.logger.logger_builder import get_logger
from esd.spack_factory.SpackOperation import SpackOperation
from esd.configuration.SpackConfig import SpackConfig
@@ -8,8 +10,19 @@ class SpackOperationUseCache(SpackOperation):
This class uses caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
+ cache_version_build='cache'):
super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=cache_version_build)
def setup_spack_env(self):
super().setup_spack_env()
--
GitLab
From 59ab786c33be27d4f003c87ea1baae518496c20c Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 18 Feb 2025 13:45:13 +0200
Subject: [PATCH 08/11] esd-spack-installation: package renaming to dedal
---
.gitlab-ci.yml | 2 -
{esd => dedal}/configuration/GpgConfig.py | 0
{esd => dedal}/configuration/SpackConfig.py | 4 +-
{esd => dedal}/configuration/__init__.py | 0
{esd => dedal}/error_handling/__init__.py | 0
{esd => dedal}/error_handling/exceptions.py | 0
{esd => dedal}/model/SpackDescriptor.py | 0
{esd => dedal}/model/__init__.py | 0
.../spack_factory/SpackOperation.py | 12 +--
.../spack_factory/SpackOperationCreator.py | 6 +-
.../spack_factory/SpackOperationUseCache.py | 8 +-
{esd => dedal}/spack_factory/__init__.py | 0
dedal/tests/spack_from_scratch_test.py | 12 +--
dedal/tests/spack_install_test.py | 4 +-
dedal/tests/utils_test.py | 73 ++++++++++++++++++-
dedal/utils/utils.py | 14 +++-
{esd => dedal}/wrapper/__init__.py | 0
{esd => dedal}/wrapper/spack_wrapper.py | 2 +-
18 files changed, 108 insertions(+), 29 deletions(-)
rename {esd => dedal}/configuration/GpgConfig.py (100%)
rename {esd => dedal}/configuration/SpackConfig.py (92%)
rename {esd => dedal}/configuration/__init__.py (100%)
rename {esd => dedal}/error_handling/__init__.py (100%)
rename {esd => dedal}/error_handling/exceptions.py (100%)
rename {esd => dedal}/model/SpackDescriptor.py (100%)
rename {esd => dedal}/model/__init__.py (100%)
rename {esd => dedal}/spack_factory/SpackOperation.py (97%)
rename {esd => dedal}/spack_factory/SpackOperationCreator.py (67%)
rename {esd => dedal}/spack_factory/SpackOperationUseCache.py (86%)
rename {esd => dedal}/spack_factory/__init__.py (100%)
rename {esd => dedal}/wrapper/__init__.py (100%)
rename {esd => dedal}/wrapper/spack_wrapper.py (85%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bd5669bd..2b497048 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,7 +6,6 @@ variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
-
build-wheel:
stage: build
tags:
@@ -40,6 +39,5 @@ testing-pytest:
paths:
- test-results.xml
- .dedal.log
- - .generate_cache.log
expire_in: 1 week
diff --git a/esd/configuration/GpgConfig.py b/dedal/configuration/GpgConfig.py
similarity index 100%
rename from esd/configuration/GpgConfig.py
rename to dedal/configuration/GpgConfig.py
diff --git a/esd/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py
similarity index 92%
rename from esd/configuration/SpackConfig.py
rename to dedal/configuration/SpackConfig.py
index b6178760..0d470679 100644
--- a/esd/configuration/SpackConfig.py
+++ b/dedal/configuration/SpackConfig.py
@@ -1,8 +1,8 @@
import os
from pathlib import Path
-from esd.configuration.GpgConfig import GpgConfig
-from esd.model import SpackDescriptor
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.model import SpackDescriptor
class SpackConfig:
diff --git a/esd/configuration/__init__.py b/dedal/configuration/__init__.py
similarity index 100%
rename from esd/configuration/__init__.py
rename to dedal/configuration/__init__.py
diff --git a/esd/error_handling/__init__.py b/dedal/error_handling/__init__.py
similarity index 100%
rename from esd/error_handling/__init__.py
rename to dedal/error_handling/__init__.py
diff --git a/esd/error_handling/exceptions.py b/dedal/error_handling/exceptions.py
similarity index 100%
rename from esd/error_handling/exceptions.py
rename to dedal/error_handling/exceptions.py
diff --git a/esd/model/SpackDescriptor.py b/dedal/model/SpackDescriptor.py
similarity index 100%
rename from esd/model/SpackDescriptor.py
rename to dedal/model/SpackDescriptor.py
diff --git a/esd/model/__init__.py b/dedal/model/__init__.py
similarity index 100%
rename from esd/model/__init__.py
rename to dedal/model/__init__.py
diff --git a/esd/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
similarity index 97%
rename from esd/spack_factory/SpackOperation.py
rename to dedal/spack_factory/SpackOperation.py
index a0b21a1c..ecfbb8e5 100644
--- a/esd/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -2,13 +2,13 @@ import os
import re
import subprocess
from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
+from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
-from esd.logger.logger_builder import get_logger
-from esd.configuration.SpackConfig import SpackConfig
-from esd.tests.testing_variables import SPACK_VERSION
-from esd.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
+from dedal.logger.logger_builder import get_logger
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.tests.testing_variables import SPACK_VERSION
+from dedal.wrapper.spack_wrapper import check_spack_env
+from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
class SpackOperation:
diff --git a/esd/spack_factory/SpackOperationCreator.py b/dedal/spack_factory/SpackOperationCreator.py
similarity index 67%
rename from esd/spack_factory/SpackOperationCreator.py
rename to dedal/spack_factory/SpackOperationCreator.py
index 8369c5ca..54517a84 100644
--- a/esd/spack_factory/SpackOperationCreator.py
+++ b/dedal/spack_factory/SpackOperationCreator.py
@@ -1,6 +1,6 @@
-from esd.configuration.SpackConfig import SpackConfig
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
class SpackOperationCreator:
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
similarity index 86%
rename from esd/spack_factory/SpackOperationUseCache.py
rename to dedal/spack_factory/SpackOperationUseCache.py
index 313522d2..efb9af76 100644
--- a/esd/spack_factory/SpackOperationUseCache.py
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -1,8 +1,8 @@
import os
-from esd.build_cache.BuildCacheManager import BuildCacheManager
-from esd.logger.logger_builder import get_logger
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.configuration.SpackConfig import SpackConfig
+from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.logger.logger_builder import get_logger
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.configuration.SpackConfig import SpackConfig
class SpackOperationUseCache(SpackOperation):
diff --git a/esd/spack_factory/__init__.py b/dedal/spack_factory/__init__.py
similarity index 100%
rename from esd/spack_factory/__init__.py
rename to dedal/spack_factory/__init__.py
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index cdc405e7..2fec80f7 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -1,11 +1,11 @@
from pathlib import Path
import pytest
-from esd.configuration.SpackConfig import SpackConfig
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
-from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
-from esd.model.SpackDescriptor import SpackDescriptor
-from esd.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
-from esd.utils.utils import file_exists_and_not_empty
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
+from dedal.utils.utils import file_exists_and_not_empty
def test_spack_repo_exists_1():
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py
index 28f8268e..564d5c6a 100644
--- a/dedal/tests/spack_install_test.py
+++ b/dedal/tests/spack_install_test.py
@@ -1,6 +1,6 @@
import pytest
-from esd.spack_factory.SpackOperation import SpackOperation
-from esd.tests.testing_variables import SPACK_VERSION
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.tests.testing_variables import SPACK_VERSION
# run this test first so that spack is installed only once for all the tests
diff --git a/dedal/tests/utils_test.py b/dedal/tests/utils_test.py
index 256b8218..cd478606 100644
--- a/dedal/tests/utils_test.py
+++ b/dedal/tests/utils_test.py
@@ -1,7 +1,9 @@
+import subprocess
+
import pytest
from pathlib import Path
-
-from dedal.utils.utils import clean_up
+from unittest.mock import mock_open, patch, MagicMock
+from dedal.utils.utils import clean_up, file_exists_and_not_empty, log_command, run_command
@pytest.fixture
@@ -79,3 +81,70 @@ def test_file_exists_and_not_empty(tmp_path: Path):
non_empty_file = tmp_path / "non_empty.txt"
non_empty_file.write_text("Some content")
assert file_exists_and_not_empty(non_empty_file)
+
+
+def test_log_command():
+ results = MagicMock()
+ results.stdout = "Test output"
+ results.stderr = "Test error"
+ mock_file = mock_open()
+
+ with patch("builtins.open", mock_file):
+ log_command(results, "logfile.log")
+
+ mock_file.assert_called_once_with("logfile.log", "w")
+ handle = mock_file()
+ handle.write.assert_any_call("Test output")
+ handle.write.assert_any_call("\n--- STDERR ---\n")
+ handle.write.assert_any_call("Test error")
+
+
+def test_run_command_success(mocker):
+ mock_subprocess = mocker.patch("subprocess.run", return_value=MagicMock(returncode=0))
+ mock_logger = MagicMock()
+ result = run_command('bash', '-c', 'echo hello', logger=mock_logger, info_msg="Running echo")
+ mock_logger.info.assert_called_with("Running echo: args: ('bash', '-c', 'echo hello')")
+ mock_subprocess.assert_called_once_with(('bash', '-c', 'echo hello'))
+ assert result.returncode == 0
+
+
+def test_run_command_not_found(mocker):
+ mocker.patch("subprocess.run", side_effect=FileNotFoundError)
+ mock_logger = MagicMock()
+ run_command("invalid_command", logger=mock_logger)
+ mock_logger.error.assert_called_with("Command not found. Please check the command syntax.")
+
+
+def test_run_command_permission_error(mocker):
+ mocker.patch("subprocess.run", side_effect=PermissionError)
+ mock_logger = MagicMock()
+ run_command("restricted_command", logger=mock_logger)
+ mock_logger.error.assert_called_with("Permission denied. Try running with appropriate permissions.")
+
+
+def test_run_command_timeout(mocker):
+ mocker.patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="test", timeout=5))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger)
+ mock_logger.error.assert_called_with("Command timed out. Try increasing the timeout duration.")
+
+
+def test_run_command_os_error(mocker):
+ mocker.patch("subprocess.run", side_effect=OSError("OS Error"))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger)
+ mock_logger.error.assert_called_with("OS error occurred: OS Error")
+
+
+def test_run_command_unexpected_exception(mocker):
+ mocker.patch("subprocess.run", side_effect=Exception("Unexpected Error"))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger)
+ mock_logger.error.assert_called_with("An unexpected error occurred: Unexpected Error")
+
+
+def test_run_command_called_process_error(mocker):
+ mocker.patch("subprocess.run", side_effect=subprocess.CalledProcessError(1, "test"))
+ mock_logger = MagicMock()
+ run_command("test", logger=mock_logger, exception_msg="Process failed")
+ mock_logger.error.assert_called_with("Process failed: Command 'test' returned non-zero exit status 1.")
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 033cbc54..9fc82ad5 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -4,7 +4,7 @@ import shutil
import subprocess
from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException
+from dedal.error_handling.exceptions import BashCommandException
import re
@@ -38,6 +38,18 @@ def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', e
raise exception(f'{exception_msg} : {e}')
else:
return None
+ except FileNotFoundError:
+ logger.error(f"Command not found. Please check the command syntax.")
+ except PermissionError:
+ logger.error(f"Permission denied. Try running with appropriate permissions.")
+ except subprocess.TimeoutExpired:
+ logger.error(f"Command timed out. Try increasing the timeout duration.")
+ except ValueError:
+ logger.error(f"Invalid argument passed to subprocess. Check function parameters.")
+ except OSError as e:
+ logger.error(f"OS error occurred: {e}")
+ except Exception as e:
+ logger.error(f"An unexpected error occurred: {e}")
def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
diff --git a/esd/wrapper/__init__.py b/dedal/wrapper/__init__.py
similarity index 100%
rename from esd/wrapper/__init__.py
rename to dedal/wrapper/__init__.py
diff --git a/esd/wrapper/spack_wrapper.py b/dedal/wrapper/spack_wrapper.py
similarity index 85%
rename from esd/wrapper/spack_wrapper.py
rename to dedal/wrapper/spack_wrapper.py
index c2f9c116..018cad48 100644
--- a/esd/wrapper/spack_wrapper.py
+++ b/dedal/wrapper/spack_wrapper.py
@@ -1,6 +1,6 @@
import functools
-from esd.error_handling.exceptions import NoSpackEnvironmentException
+from dedal.error_handling.exceptions import NoSpackEnvironmentException
def check_spack_env(method):
--
GitLab
From ccf28f621364c9b33b7e0237e83b20011d8ce007 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 20 Feb 2025 13:00:31 +0200
Subject: [PATCH 09/11] esd-spack-install: test package restructure
---
dedal/tests/integration_tests/__init__.py | 0
dedal/tests/{ => integration_tests}/spack_from_scratch_test.py | 0
dedal/tests/{ => integration_tests}/spack_install_test.py | 0
dedal/tests/unit_tests/__init__.py | 0
dedal/tests/{ => unit_tests}/utils_test.py | 0
5 files changed, 0 insertions(+), 0 deletions(-)
create mode 100644 dedal/tests/integration_tests/__init__.py
rename dedal/tests/{ => integration_tests}/spack_from_scratch_test.py (100%)
rename dedal/tests/{ => integration_tests}/spack_install_test.py (100%)
create mode 100644 dedal/tests/unit_tests/__init__.py
rename dedal/tests/{ => unit_tests}/utils_test.py (100%)
diff --git a/dedal/tests/integration_tests/__init__.py b/dedal/tests/integration_tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
similarity index 100%
rename from dedal/tests/spack_from_scratch_test.py
rename to dedal/tests/integration_tests/spack_from_scratch_test.py
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
similarity index 100%
rename from dedal/tests/spack_install_test.py
rename to dedal/tests/integration_tests/spack_install_test.py
diff --git a/dedal/tests/unit_tests/__init__.py b/dedal/tests/unit_tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/dedal/tests/utils_test.py b/dedal/tests/unit_tests/utils_test.py
similarity index 100%
rename from dedal/tests/utils_test.py
rename to dedal/tests/unit_tests/utils_test.py
--
GitLab
From 2b1be1143535c27c88be9fc906e1676625a00d74 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Fri, 21 Feb 2025 10:49:01 +0200
Subject: [PATCH 10/11] esd-spack-installation: README
---
.env | 9 +++
README.md | 55 ++++++++++++++++++-
dedal/spack_factory/SpackOperationUseCache.py | 4 +-
dedal/tests/testing_variables.py | 2 +-
4 files changed, 64 insertions(+), 6 deletions(-)
create mode 100644 .env
diff --git a/.env b/.env
new file mode 100644
index 00000000..93e62653
--- /dev/null
+++ b/.env
@@ -0,0 +1,9 @@
+BUILDCACHE_OCI_HOST=""
+BUILDCACHE_OCI_PASSWORD=""
+BUILDCACHE_OCI_PROJECT=""
+BUILDCACHE_OCI_USERNAME=""
+
+CONCRETIZE_OCI_HOST=""
+CONCRETIZE_OCI_PASSWORD=""
+CONCRETIZE_OCI_PROJECT=""
+CONCRETIZE_OCI_USERNAME""
diff --git a/README.md b/README.md
index 86ab9c2f..dd68dcfa 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,53 @@
-# ~~Yashchiki~~Koutakia
+# Dedal
-For now, this repository provides helpers for the EBRAINS container image build flow.
-The lowest spack version which should be used with this library is v0.22.0
+This repository provides functionalities to easily ```managed spack environments``` and ```helpers for the container image build flow```.
+
+This library runs only on different kinds of linux distribution operating systems.
+
+The lowest ```spack version``` compatible with this library is ```v0.23.0```.
+
+
+ This repository also provied CLI interface. For more informations, after installing this library, call dedal --help.
+
+
+**Setting up the needed environment variables**
+ The ````<checkout path>\dedal\.env```` file contains the environment variables required for OCI registry used for caching.
+ Ensure that you edit the ````<checkout path>\dedal\.env```` file to match your environment.
+ The following provides an explanation of the various environment variables:
+
+
+ # OCI Registry Configuration Sample for concretization caches
+ # =============================
+ # The following variables configure the Harbor docker OCI registry (EBRAINS) used for caching.
+
+ # The hostname of the OCI registry. e.g. docker-registry.ebrains.eu
+ CONCRETIZE__OCI_HOST="docker-registry.ebrains.eu"
+
+ # The project name in the Docker registry.
+ CONCRETIZE__OCI_PROJECT="concretize_caches"
+
+ # The username used for authentication with the Docker registry.
+ CONCRETIZE__OCI_USERNAME="robot$concretize-cache-test+user"
+
+ # The password used for authentication with the Docker registry.
+ CONCRETIZE__OCI_HOST="###ACCESS_TOKEN###"
+
+
+ # OCI Registry Configuration Sample for binary caches
+ # =============================
+ # The following variables configure the Harbor docker OCI registry (EBRAINS) used for caching.
+
+ # The hostname of the OCI registry. e.g. docker-registry.ebrains.eu
+ BUILDCACHE_OCI_HOST="docker-registry.ebrains.eu"
+
+ # The project name in the Docker registry.
+ BUILDCACHE_OCI_PROJECT="binary-cache-test"
+
+ # The username used for authentication with the Docker registry.
+ BUILDCACHE_OCI_USERNAME="robot$binary-cache-test+user"
+
+ # The password used for authentication with the Docker registry.
+ BUILDCACHE_OCI_HOST="###ACCESS_TOKEN###"
+
+For both concretization and binary caches, the cache version can be changed via the attributes ```cache_version_concretize``` and ```cache_version_build```.
+The default values are ```v1```.
diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
index efb9af76..41a9094c 100644
--- a/dedal/spack_factory/SpackOperationUseCache.py
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -10,8 +10,8 @@ class SpackOperationUseCache(SpackOperation):
This class uses caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
- cache_version_build='cache'):
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='v1',
+ cache_version_build='v1'):
super().__init__(spack_config, logger=get_logger(__name__))
self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
os.environ.get('CONCRETIZE_OCI_PROJECT'),
diff --git a/dedal/tests/testing_variables.py b/dedal/tests/testing_variables.py
index ab95bfa1..e441a286 100644
--- a/dedal/tests/testing_variables.py
+++ b/dedal/tests/testing_variables.py
@@ -1,6 +1,6 @@
import os
ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
-SPACK_VERSION = "0.22.0"
+SPACK_VERSION = "0.23.0"
SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
--
GitLab
From 66d4453c7ffd647c3e1783936b0f33bb2182f8d6 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 26 Feb 2025 16:06:29 +0200
Subject: [PATCH 11/11] esd-spack-installation: fixing tests
---
.../integration_tests/spack_from_scratch_test.py | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
index 2fec80f7..d080bc7b 100644
--- a/dedal/tests/integration_tests/spack_from_scratch_test.py
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -8,13 +8,7 @@ from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_buil
from dedal.utils.utils import file_exists_and_not_empty
-def test_spack_repo_exists_1():
- spack_operation = SpackOperationCreator.get_spack_operator()
- spack_operation.install_spack()
- assert spack_operation.spack_repo_exists('ebrains-spack-builds') == False
-
-
-def test_spack_repo_exists_2(tmp_path):
+def test_spack_repo_exists_1(tmp_path):
install_dir = tmp_path
env = SpackDescriptor('ebrains-spack-builds', install_dir)
config = SpackConfig(env=env, install_dir=install_dir)
@@ -24,7 +18,7 @@ def test_spack_repo_exists_2(tmp_path):
spack_operation.spack_repo_exists(env.env_name)
-def test_spack_repo_exists_3(tmp_path):
+def test_spack_repo_exists_2(tmp_path):
install_dir = tmp_path
env = SpackDescriptor('ebrains-spack-builds', install_dir)
config = SpackConfig(env=env, install_dir=install_dir)
@@ -91,6 +85,8 @@ def test_spack_not_a_valid_repo():
spack_operation.add_spack_repo(repo.path, repo.env_name)
+@pytest.mark.skip(
+ reason="Skipping the concretization step because it may freeze when numerous Spack packages are added to the environment.")
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
@@ -107,6 +103,8 @@ def test_spack_from_scratch_concretize_1(tmp_path):
assert file_exists_and_not_empty(concretization_file_path) == True
+@pytest.mark.skip(
+ reason="Skipping the concretization step because it may freeze when numerous Spack packages are added to the environment.")
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
--
GitLab