From 33a04b84900890071e6bbe075e6e8ff6aeb4e207 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 11:20:49 +0200
Subject: [PATCH] esd-spack-installation: fixing spack env activation method
and added error handling;
---
.gitlab-ci.yml | 39 ++--
...ge_build_cache.py => BuildCacheManager.py} | 61 +++---
esd/build_cache/BuildCacheManagerInterface.py | 17 ++
esd/error_handling/exceptions.py | 13 ++
esd/logger/logger_builder.py | 51 +++++
esd/logger/logger_config.py | 33 ----
esd/logger/logging.conf | 55 ++++++
esd/model/SpackModel.py | 12 ++
esd/model/__init__.py | 0
esd/spack_manager/SpackManager.py | 180 ++++++++++++++++++
esd/spack_manager/__init__.py | 0
esd/spack_manager/enums/SpackManagerEnum.py | 6 +
esd/spack_manager/enums/__init__.py | 0
.../factory/SpackManagerBuildCache.py | 19 ++
.../factory/SpackManagerCreator.py | 14 ++
.../factory/SpackManagerScratch.py | 15 ++
esd/spack_manager/factory/__init__.py | 0
esd/specfile_storage_path_source.py | 6 +-
esd/tests/spack_from_scratch_test.py | 63 ++++++
esd/tests/spack_install_test.py | 21 ++
esd/utils/bootstrap.sh | 6 +
esd/utils/utils.py | 31 ++-
pyproject.toml | 12 +-
23 files changed, 564 insertions(+), 90 deletions(-)
rename esd/build_cache/{manage_build_cache.py => BuildCacheManager.py} (66%)
create mode 100644 esd/build_cache/BuildCacheManagerInterface.py
create mode 100644 esd/error_handling/exceptions.py
create mode 100644 esd/logger/logger_builder.py
delete mode 100644 esd/logger/logger_config.py
create mode 100644 esd/logger/logging.conf
create mode 100644 esd/model/SpackModel.py
create mode 100644 esd/model/__init__.py
create mode 100644 esd/spack_manager/SpackManager.py
create mode 100644 esd/spack_manager/__init__.py
create mode 100644 esd/spack_manager/enums/SpackManagerEnum.py
create mode 100644 esd/spack_manager/enums/__init__.py
create mode 100644 esd/spack_manager/factory/SpackManagerBuildCache.py
create mode 100644 esd/spack_manager/factory/SpackManagerCreator.py
create mode 100644 esd/spack_manager/factory/SpackManagerScratch.py
create mode 100644 esd/spack_manager/factory/__init__.py
create mode 100644 esd/tests/spack_from_scratch_test.py
create mode 100644 esd/tests/spack_install_test.py
create mode 100644 esd/utils/bootstrap.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7dea24a3..0ce02907 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,37 +1,42 @@
stages:
- - build
- test
+ - build
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
-build-spack-env-on-runner:
+build-wheel:
stage: build
tags:
- - docker-runner # esd_image
- image: $BUILD_ENV_DOCKER_IMAGE
+ - docker-runner
+ image: python:latest
+ before_script:
+ - python -m pip install --upgrade pip setuptools wheel build
script:
- - /usr/sbin/sysctl user.max_user_namespaces
- - /usr/sbin/sysctl kernel.unprivileged_userns_clone
- #- buildah build --isolation=chroot --runtime=crun --network=host --storage-opt="network.network_backend=cni" --storage-opt="overlay.mount_program=/usr/bin/fuse-overlayfs" -f Dockerfile .
- #- export APPTAINER_VERSION="1.3.6"
- #- |
- # mkdir -p apptainer-install/
- # curl -s https://raw.githubusercontent.com/apptainer/apptainer/main/tools/install-unprivileged.sh | bash -s - apptainer-install/
- - apptainer version
+ - python -m build --sdist --wheel
+ artifacts:
+ paths:
+ - dist/*.whl
+ - dist/*.tar.gz
+ expire_in: 1 week
+
-testing:
+testing-pytest:
stage: test
tags:
- docker-runner
- image: python:latest # $BUILD_ENV_DOCKER_IMAGE
+ image: ubuntu:22.04
script:
- - pip install -e .
- - pytest ./esd/tests/ --junitxml=test-results.xml
+ - chmod +x esd/utils/bootstrap.sh
+ - ./esd/utils/bootstrap.sh
+ - pip install .
+ - pytest ./esd/tests/ -s --junitxml=test-results.xml
artifacts:
when: always
reports:
junit: test-results.xml
paths:
- test-results.xml
- expire_in: 1 week
\ No newline at end of file
+ - .esd.log
+ expire_in: 1 week
+
diff --git a/esd/build_cache/manage_build_cache.py b/esd/build_cache/BuildCacheManager.py
similarity index 66%
rename from esd/build_cache/manage_build_cache.py
rename to esd/build_cache/BuildCacheManager.py
index bc0054d6..e1bd6824 100644
--- a/esd/build_cache/manage_build_cache.py
+++ b/esd/build_cache/BuildCacheManager.py
@@ -2,18 +2,19 @@ import os
import oras.client
from pathlib import Path
-from esd.logger.logger_config import LoggerConfig
+from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from esd.logger.logger_builder import get_logger
from esd.utils.utils import clean_up
-class BuildCacheManager:
+class BuildCacheManager(BuildCacheManagerInterface):
"""
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', log_path='./', insecure=False):
+ def __init__(self, auth_backend='basic', insecure=False):
+ self.logger = get_logger(__name__, BuildCacheManager.__name__)
self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.log_file = Path(log_path) / "log_oras.txt"
self.registry_project = os.environ.get("REGISTRY_PROJECT")
self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
@@ -27,24 +28,22 @@ class BuildCacheManager:
self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
self.client.login(username=self._registry_username, password=self._registry_password)
self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
- self.LOGGER = LoggerConfig(self.log_file).get_logger()
- def oci_registry_upload_build_cache(self, cache_dir: Path):
+ def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / cache_dir
+ build_cache_path = self.home_path / out_dir
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- raise FileNotFoundError(
- f"BuildCacheManager::oci_registry_upload_build_cache::Path {build_cache_path} not found.")
+ self.logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
- target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
+ target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
try:
- self.LOGGER.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
self.client.push(
files=[str(sub_path)],
target=target,
@@ -52,31 +51,31 @@ class BuildCacheManager:
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self.LOGGER.info(f"Successfully pushed {sub_path.name}")
+ self.logger.info(f"Successfully pushed {sub_path.env_name}")
except Exception as e:
- self.LOGGER.error(
- f"BuildCacheManager::registry_upload_build_cache::An error occurred while pushing: {e}")
- # delete the build cache after being pushed to the OCI Registry
- clean_up([str(build_cache_path)], self.LOGGER)
+ self.logger.error(
+ f"An error occurred while pushing: {e}")
+ # todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
+ # clean_up([str(build_cache_path)], self.logger)
- def oci_registry_get_tags(self):
+ def list_tags(self):
"""
This method retrieves all tags from an OCI Registry
"""
try:
return self.client.get_tags(self.oci_registry_path)
except Exception as e:
- self.LOGGER.error(f"BuildCacheManager::oci_registry_get_tags::Failed to list tags: {e}")
+ self.logger.error(f"Failed to list tags: {e}")
return None
- def oci_registry_download_build_cache(self, cache_dir: Path):
+ def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / cache_dir
+ build_cache_path = self.home_path / in_dir
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
- tags = self.oci_registry_get_tags()
+ tags = self.list_tags()
if tags is not None:
for tag in tags:
ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
@@ -88,27 +87,27 @@ class BuildCacheManager:
try:
self.client.pull(
ref,
- # missing dirs to outdir are created automatically by OrasClient pull method
+ # missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- self.LOGGER.info(f"Successfully pulled artifact {tag}.")
+ self.logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- self.LOGGER.error(
- f"BuildCacheManager::registry_download_build_cache::Failed to pull artifact {tag} : {e}")
+ self.logger.error(
+ f"Failed to pull artifact {tag} : {e}")
- def oci_registry_delete_build_cache(self):
+ def delete(self):
"""
Deletes all artifacts from an OCI Registry based on their tags.
This method removes artifacts identified by their tags in the specified OCI Registry.
It requires appropriate permissions to delete artifacts from the registry.
If the registry or user does not have the necessary delete permissions, the operation might fail.
"""
- tags = self.oci_registry_get_tags()
+ tags = self.list_tags()
if tags is not None:
try:
self.client.delete_tags(self.oci_registry_path, tags)
- self.LOGGER.info(f"Successfully deleted all artifacts form OCI registry.")
+ self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- self.LOGGER.error(
- f"BuildCacheManager::registry_delete_build_cache::Failed to delete artifacts: {e}")
+ self.logger.error(
+ f"Failed to delete artifacts: {e}")
diff --git a/esd/build_cache/BuildCacheManagerInterface.py b/esd/build_cache/BuildCacheManagerInterface.py
new file mode 100644
index 00000000..3016590b
--- /dev/null
+++ b/esd/build_cache/BuildCacheManagerInterface.py
@@ -0,0 +1,17 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class BuildCacheManagerInterface(ABC):
+
+ @abstractmethod
+ def upload(self, out_dir: Path):
+ pass
+
+ @abstractmethod
+ def download(self, in_dir: Path):
+ pass
+
+ @abstractmethod
+ def delete(self):
+ pass
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
new file mode 100644
index 00000000..2acca54e
--- /dev/null
+++ b/esd/error_handling/exceptions.py
@@ -0,0 +1,13 @@
+class SpackException(Exception):
+
+ def __init__(self, message):
+ super().__init__(message)
+ self.message = str(message)
+
+ def __str__(self):
+ return self.message
+
+class BashCommandException(SpackException):
+ """
+ To be thrown when an invalid input is received.
+ """
diff --git a/esd/logger/logger_builder.py b/esd/logger/logger_builder.py
new file mode 100644
index 00000000..c15a2f0e
--- /dev/null
+++ b/esd/logger/logger_builder.py
@@ -0,0 +1,51 @@
+import os
+import inspect
+import weakref
+import logging
+import logging.config
+
+
+class LoggerBuilder(object):
+ """
+ Class taking care of uniform Python logger initialization.
+ It uses the Python native logging package.
+ It's purpose is just to offer a common mechanism for initializing all modules in a package.
+ """
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(self, config_file_name='logging.conf'):
+ """
+ Prepare Python logger based on a configuration file.
+ :param: config_file_name - name of the logging configuration relative to the current package
+ """
+ current_folder = os.path.dirname(inspect.getfile(self.__class__))
+ config_file_path = os.path.join(current_folder, config_file_name)
+ logging.config.fileConfig(config_file_path, disable_existing_loggers=False)
+ self._loggers = weakref.WeakValueDictionary()
+
+ def build_logger(self, parent_module, parent_class):
+ """
+ Build a logger instance and return it
+ """
+ logger_key = f'{parent_module}.{parent_class}' if parent_class else parent_module
+ self._loggers[logger_key] = logger = logging.getLogger(logger_key)
+ return logger
+
+ def set_loggers_level(self, level):
+ for logger in self._loggers.values():
+ logger.setLevel(level)
+
+
+def get_logger(parent_module='', parent_class=None):
+ """
+ Function to retrieve a new Python logger instance for current module.
+
+ :param parent_module: module name for which to create logger.
+ :param parent_class: class name for which to create logger.
+ """
+ return LoggerBuilder().build_logger(parent_module, parent_class)
diff --git a/esd/logger/logger_config.py b/esd/logger/logger_config.py
deleted file mode 100644
index 3ca3b000..00000000
--- a/esd/logger/logger_config.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import logging
-
-
-class LoggerConfig:
- """
- This class sets up logging with a file handler
- and a stream handler, ensuring consistent
- and formatted log messages.
- """
- def __init__(self, log_file):
- self.log_file = log_file
- self._configure_logger()
-
- def _configure_logger(self):
- formatter = logging.Formatter(
- fmt='%(asctime)s - %(levelname)s - %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S'
- )
-
- file_handler = logging.FileHandler(self.log_file)
- file_handler.setFormatter(formatter)
-
- stream_handler = logging.StreamHandler()
- stream_handler.setFormatter(formatter)
-
- self.logger = logging.getLogger(__name__)
- self.logger.setLevel(logging.DEBUG)
-
- self.logger.addHandler(file_handler)
- self.logger.addHandler(stream_handler)
-
- def get_logger(self):
- return self.logger
diff --git a/esd/logger/logging.conf b/esd/logger/logging.conf
new file mode 100644
index 00000000..d95ba87c
--- /dev/null
+++ b/esd/logger/logging.conf
@@ -0,0 +1,55 @@
+############################################
+## ESD - logging configuration. ##
+############################################
+[loggers]
+keys=root, esd, oras
+
+[handlers]
+keys=consoleHandler, fileHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=WARNING
+handlers=consoleHandler, fileHandler
+propagate=0
+
+############################################
+## esd specific logging ##
+############################################
+[logger_esd]
+level=DEBUG
+handlers=consoleHandler, fileHandler
+qualname=esd
+propagate=0
+
+[logger_oras]
+level=ERROR
+handlers=consoleHandler
+qualname=oras
+propagate=0
+
+############################################
+## Handlers ##
+############################################
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=handlers.TimedRotatingFileHandler
+level=INFO
+formatter=simpleFormatter
+args=('.esd.log', 'midnight', 1, 30, None, False, False)
+
+############################################
+## Formatters ##
+############################################
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(levelname)s - %(name)s::%(funcName)s - %(message)s
+datefmt = %d-%m-%Y %I:%M:%S
\ No newline at end of file
diff --git a/esd/model/SpackModel.py b/esd/model/SpackModel.py
new file mode 100644
index 00000000..4b065dba
--- /dev/null
+++ b/esd/model/SpackModel.py
@@ -0,0 +1,12 @@
+from pathlib import Path
+
+
+class SpackModel:
+ """"
+ Provides details about the spack environment
+ """
+
+ def __init__(self, env_name: str, path: Path, git_path: str = None):
+ self.env_name = env_name
+ self.path = path
+ self.git_path = git_path
diff --git a/esd/model/__init__.py b/esd/model/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
new file mode 100644
index 00000000..cdf4a1d3
--- /dev/null
+++ b/esd/spack_manager/SpackManager.py
@@ -0,0 +1,180 @@
+import os
+import subprocess
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from esd.error_handling.exceptions import BashCommandException
+from esd.logger.logger_builder import get_logger
+from esd.model.SpackModel import SpackModel
+from esd.utils.utils import run_command, git_clone_repo
+
+
+class SpackManager(ABC):
+ """
+ This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
+ Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
+
+ Attributes:
+ -----------
+ env : SpackModel
+ spack environment details
+ repos : list[SpackModel]
+ upstream_instance : str
+ path to Spack instance to use as upstream (optional)
+ """
+
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
+ if repos is None:
+ self.repos = list()
+ self.env = env
+ self.upstream_instance = upstream_instance
+ self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd())
+ self.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.install_dir / "spack"
+ self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ self.logger = logger
+ self.system_name = system_name
+
+ @abstractmethod
+ def concretize_spack_env(self, force=True):
+ pass
+
+ @abstractmethod
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
+
+ def create_fetch_spack_environment(self):
+ env_dir = self.install_dir / self.env.path / self.env.env_name
+ if self.env.git_path:
+ try:
+ git_clone_repo(self.env.env_name, env_dir, self.env.git_path, logger=self.logger)
+ except subprocess.CalledProcessError as e:
+ self.logger.exception(f'Failed to clone repository: {self.env.env_name}: {e}')
+ raise BashCommandException(f'Failed to clone repository: {self.env.env_name}: {e}')
+ else:
+ try:
+ os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env create -d {self.env.path}/{self.env.env_name}',
+ check=True, logger=self.logger)
+ self.logger.debug(f"Created {self.env.env_name} spack environment")
+ except subprocess.CalledProcessError as e:
+ self.logger.error(f"Failed to create {self.env.env_name} spack environment")
+ raise BashCommandException(f"Failed to create {self.env.env_name} spack environment")
+
+ def setup_spack_env(self):
+ """
+ This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
+ """
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ if self.system_name:
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
+ os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_dir.exists() and self.spack_dir.is_dir():
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
+ bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
+ else:
+ self.logger.error(f'Invalid installation path: {self.spack_dir}')
+ # Restart the bash after adding environment variables
+ self.create_fetch_spack_environment()
+ if self.install_dir.exists():
+ for repo in self.repos:
+ repo_dir = self.install_dir / repo.path / repo.env_name
+ git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
+ if not self.spack_repo_exists(repo.env_name):
+ self.add_spack_repo(repo.path, repo.env_name)
+ self.logger.debug(f'Added spack repository {repo.env_name}')
+ else:
+ self.logger.debug(f'Spack repository {repo.env_name} already added')
+
+ def spack_repo_exists(self, repo_name: str) -> bool:
+ """Check if the given Spack repository exists."""
+ if self.env is None:
+ try:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger)
+ except subprocess.CalledProcessError:
+ return False
+ else:
+ try:
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger)
+ except subprocess.CalledProcessError:
+ return False
+ return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
+
+ def add_spack_repo(self, repo_path: Path, repo_name: str):
+ """Add the Spack repository if it does not exist."""
+ repo_path = repo_path.resolve().as_posix()
+ try:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger)
+ self.logger.debug(f"Added {repo_name} to spack environment {self.env.env_name}")
+ except subprocess.CalledProcessError as e:
+ self.logger.error(f"Failed to add {repo_name} to spack environment {self.env.env_name}")
+ raise BashCommandException(f"Failed to add {repo_name} to spack environment {self.env.env_name}: {e}")
+
+ def get_spack_installed_version(self):
+ try:
+ spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ capture_output=True, text=True, check=True,
+ logger=self.logger)
+ spack_version = spack_version.stdout.strip().split()[0]
+ self.logger.debug(f"Getting spack version: {spack_version}")
+ return spack_version
+ except subprocess.SubprocessError as e:
+ self.logger.error(f"Error retrieving Spack version: {e}")
+ return None
+
+ def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+ try:
+ user = os.getlogin()
+ except OSError:
+ user = None
+
+ self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
+ if not self.spack_dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ "--branch", spack_version, spack_repo, self.spack_dir
+ , check=True, logger=self.logger)
+ self.logger.debug("Cloned spack")
+ else:
+ self.logger.debug("Spack already cloned.")
+
+ bashrc_path = os.path.expanduser("~/.bashrc")
+ # ensure the file exists before opening it
+ if not os.path.exists(bashrc_path):
+ open(bashrc_path, "w").close()
+ # add spack setup commands to .bashrc
+ with open(bashrc_path, "a") as bashrc:
+ bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
+ bashrc.write(f"source {self.spack_setup_script}\n")
+ self.logger.info("Added Spack PATH to .bashrc")
+ if user:
+ run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger)
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger)
+ self.logger.info("Spack install completed")
+ # Restart Bash after the installation ends
+ os.system("exec bash")
+
+ # Configure upstream Spack instance if specified
+ if self.upstream_instance:
+ upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
+ with open(upstreams_yaml_path, "w") as file:
+ file.write(f"""upstreams:
+ upstream-spack-instance:
+ install_tree: {self.upstream_instance}/spack/opt/spack
+ """)
+ self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_manager/__init__.py b/esd/spack_manager/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
new file mode 100644
index 00000000..a2435839
--- /dev/null
+++ b/esd/spack_manager/enums/SpackManagerEnum.py
@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class SpackManagerEnum(Enum):
+ FROM_SCRATCH = "from_scratch",
+ FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/enums/__init__.py b/esd/spack_manager/enums/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
new file mode 100644
index 00000000..38151c6d
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerBuildCache.py
@@ -0,0 +1,19 @@
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerBuildCache(SpackManager):
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
+ super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
+
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
new file mode 100644
index 00000000..9728467f
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -0,0 +1,14 @@
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+class SpackManagerCreator:
+ @staticmethod
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
+ upstream_instance: str):
+ if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
+ return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
+ return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
+
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
new file mode 100644
index 00000000..5a79797c
--- /dev/null
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -0,0 +1,15 @@
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerScratch(SpackManager):
+ def __init__(self, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
+ super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+ def concretize_spack_env(self, force=True):
+ pass
+
+ def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+ pass
diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_manager/factory/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/specfile_storage_path_source.py b/esd/specfile_storage_path_source.py
index 6e8a8889..4d2ff658 100644
--- a/esd/specfile_storage_path_source.py
+++ b/esd/specfile_storage_path_source.py
@@ -49,14 +49,14 @@ for rspec in data:
format_string = "{name}-{version}"
pretty_name = pkg.spec.format_path(format_string)
- cosmetic_path = os.path.join(pkg.name, pretty_name)
+ cosmetic_path = os.path.join(pkg.env_name, pretty_name)
to_be_fetched.add(str(spack.mirror.mirror_archive_paths(pkg.fetcher, cosmetic_path).storage_path))
for resource in pkg._get_needed_resources():
- pretty_resource_name = fsys.polite_filename(f"{resource.name}-{pkg.version}")
+ pretty_resource_name = fsys.polite_filename(f"{resource.env_name}-{pkg.version}")
to_be_fetched.add(str(spack.mirror.mirror_archive_paths(resource.fetcher, pretty_resource_name).storage_path))
for patch in ss.patches:
if isinstance(patch, spack.patch.UrlPatch):
- to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.name).storage_path))
+ to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.env_name).storage_path))
for elem in to_be_fetched:
print(elem)
diff --git a/esd/tests/spack_from_scratch_test.py b/esd/tests/spack_from_scratch_test.py
new file mode 100644
index 00000000..4981b760
--- /dev/null
+++ b/esd/tests/spack_from_scratch_test.py
@@ -0,0 +1,63 @@
+from pathlib import Path
+
+import pytest
+
+from esd.error_handling.exceptions import BashCommandException
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+def test_spack_repo_exists_1():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('ebrains-spack-builds', install_dir)
+ spack_manager = SpackManagerScratch(env=env)
+ assert spack_manager.spack_repo_exists(env.env_name) == False
+
+
+def test_spack_repo_exists_2():
+ spack_manager = SpackManagerScratch()
+ assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
+
+
+def test_spack_from_scratch_setup_1():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ spack_manager = SpackManagerScratch(env, [env], system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_2():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerScratch(env, [repo, repo], system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_3():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('new_environment', install_dir, )
+ repo = env
+ spack_manager = SpackManagerScratch(env, [repo, repo], system_name='ebrainslab')
+ with pytest.raises(BashCommandException):
+ spack_manager.setup_spack_env()
+
+
+def test_spack_from_scratch_setup_4():
+ install_dir = Path('./install').resolve()
+ env = SpackModel('new_environment', install_dir, )
+ spack_manager = SpackManagerScratch(env, system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_bash_error():
+ env = SpackModel('ebrains-spack-builds', Path(), None)
+ repo = env
+ spack_manager = SpackManagerScratch(env, [repo], system_name='ebrainslab')
+ with pytest.raises(BashCommandException):
+ spack_manager.add_spack_repo(repo.path, repo.env_name)
diff --git a/esd/tests/spack_install_test.py b/esd/tests/spack_install_test.py
new file mode 100644
index 00000000..34f68323
--- /dev/null
+++ b/esd/tests/spack_install_test.py
@@ -0,0 +1,21 @@
+import pytest
+
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+# we need this test to run first so that spack is installed only once
+@pytest.mark.run(order=1)
+def test_spack_install_scratch():
+ spack_manager = SpackManagerScratch()
+ spack_manager.install_spack(spack_version="v0.21.1")
+ installed_spack_version = spack_manager.get_spack_installed_version()
+ required_version = "0.21.1"
+ assert required_version == installed_spack_version
+
+
+def test_spack_install_buildcache():
+ spack_manager = SpackManagerBuildCache()
+ installed_spack_version = spack_manager.get_spack_installed_version()
+ required_version = "0.21.1"
+ assert required_version == installed_spack_version
diff --git a/esd/utils/bootstrap.sh b/esd/utils/bootstrap.sh
new file mode 100644
index 00000000..9b7d0131
--- /dev/null
+++ b/esd/utils/bootstrap.sh
@@ -0,0 +1,6 @@
+# Minimal prerequisites for installing the esd_library
+# pip must be installed on the OS
+echo "Bootstrapping..."
+apt update
+apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+python3 -m pip install --upgrade pip setuptools wheel
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index c1aa4c0d..d229e345 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -1,8 +1,10 @@
+import logging
import shutil
+import subprocess
from pathlib import Path
-def clean_up(dirs: list[str], logging):
+def clean_up(dirs: list[str], logging, ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
@@ -10,6 +12,31 @@ def clean_up(dirs: list[str], logging):
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
logging.info(f"Removing {cleanup_dir}")
- shutil.rmtree(Path(cleanup_dir))
+ try:
+ shutil.rmtree(Path(cleanup_dir))
+ except OSError as e:
+ logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ if not ignore_errors:
+ raise e
else:
logging.info(f"{cleanup_dir} does not exist")
+
+
+def run_command(*args, logger: None, **kwargs):
+ if logger is None:
+ logger = logging.getLogger(__name__)
+ logger.debug(f'{args}')
+ return subprocess.run(args, **kwargs)
+
+
+def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging):
+ if not dir.exists():
+ run_command(
+ "git", "clone", "--depth", "1",
+ "-c", "advice.detachedHead=false",
+ "-c", "feature.manyFiles=true",
+ git_path, dir
+ , check=True, logger=logger)
+ logger.debug(f'Cloned repository {repo_name}')
+ else:
+ logger.debug(f'Repository {repo_name} already cloned.')
diff --git a/pyproject.toml b/pyproject.toml
index cd4afe54..abcbe05d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,15 +1,15 @@
[build-system]
-requires = ["setuptools", "setuptools-scm"]
+requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
-name = "esd"
+name = "esd-tools"
+version = "0.1.0"
authors = [
{name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
{name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"},
]
description = "This package provides all the necessary tools to create an Ebrains Software Distribution environment"
-version = "0.1.0"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
@@ -18,4 +18,8 @@ dependencies = [
"ruamel.yaml",
"pytest",
"pytest-mock",
-]
\ No newline at end of file
+ "pytest-ordering",
+]
+
+[tool.setuptools.data-files]
+"esd-tools" = ["esd/logger/logging.conf"]
\ No newline at end of file
--
GitLab