diff --git a/esd/build_cache/BuildCacheManager.py b/esd/build_cache/BuildCacheManager.py
index e1bd6824321c5a2b483800bc4b39824aa4715493..cccb584695b9c59f0b4ca1ab2eb981f5c9ad9242 100644
--- a/esd/build_cache/BuildCacheManager.py
+++ b/esd/build_cache/BuildCacheManager.py
@@ -12,48 +12,51 @@ class BuildCacheManager(BuildCacheManagerInterface):
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', insecure=False):
- self.logger = get_logger(__name__, BuildCacheManager.__name__)
- self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.registry_project = os.environ.get("REGISTRY_PROJECT")
+ def __init__(self, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
+ auth_backend='basic',
+ insecure=False):
+ self._logger = get_logger(__name__, BuildCacheManager.__name__)
+ self._registry_project = registry_project
- self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
- self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
+ self._registry_username = registry_username
+ self._registry_password = registry_password
- self.registry_host = str(os.environ.get("REGISTRY_HOST"))
+ self._registry_host = registry_host
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
- self.client.login(username=self._registry_username, password=self._registry_password)
- self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
+ self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend,
+ insecure=insecure)
+ self._client.login(username=self._registry_username, password=self._registry_password)
+ self.cache_version = cache_version
+ self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/{self.cache_version}'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / out_dir
+ build_cache_path = out_dir.resolve()
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- self.logger.error(f"Path {build_cache_path} not found.")
+ self._logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
- target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
+ target = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{str(sub_path.name)}"
try:
- self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
- self.client.push(
+ self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self._client.push(
files=[str(sub_path)],
target=target,
# save in manifest the relative path for reconstruction
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self.logger.info(f"Successfully pushed {sub_path.env_name}")
+ self._logger.info(f"Successfully pushed {sub_path.name}")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"An error occurred while pushing: {e}")
# todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
# clean_up([str(build_cache_path)], self.logger)
@@ -63,37 +66,38 @@ class BuildCacheManager(BuildCacheManagerInterface):
This method retrieves all tags from an OCI Registry
"""
try:
- return self.client.get_tags(self.oci_registry_path)
+ return self._client.get_tags(self._oci_registry_path)
except Exception as e:
- self.logger.error(f"Failed to list tags: {e}")
+ self._logger.error(f"Failed to list tags: {e}")
return None
def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / in_dir
+ build_cache_path = in_dir.resolve()
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
+ ref = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
+ self._client.get_manifest(
+ f'{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}')[
'annotations'][
'path']
try:
- self.client.pull(
+ self._client.pull(
ref,
# missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- self.logger.info(f"Successfully pulled artifact {tag}.")
+ self._logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"Failed to pull artifact {tag} : {e}")
def delete(self):
@@ -106,8 +110,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
tags = self.list_tags()
if tags is not None:
try:
- self.client.delete_tags(self.oci_registry_path, tags)
- self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
+ self._client.delete_tags(self._oci_registry_path, tags)
+ self._logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- self.logger.error(
+ self._logger.error(
f"Failed to delete artifacts: {e}")
diff --git a/esd/cli/SpackManager.py b/esd/cli/SpackManager.py
new file mode 100644
index 0000000000000000000000000000000000000000..11eefce81a0458aa22b18e061a25bd6245e1c650
--- /dev/null
+++ b/esd/cli/SpackManager.py
@@ -0,0 +1,2 @@
+class SpackManager:
+ pass
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
new file mode 100644
index 0000000000000000000000000000000000000000..93a2e87423b871b0b9b4f583efec79e328739edf
--- /dev/null
+++ b/esd/configuration/SpackConfig.py
@@ -0,0 +1,25 @@
+import os
+from pathlib import Path
+from esd.model import SpackDescriptor
+
+
+class SpackConfig:
+ def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
+ install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
+ concretization_dir: Path = None, buildcache_dir: Path = None):
+ self.env = env
+ if repos is None:
+ self.repos = []
+ else:
+ self.repos = repos
+ self.install_dir = install_dir
+ self.upstream_instance = upstream_instance
+ self.system_name = system_name
+ self.concretization_dir = concretization_dir
+ self.buildcache_dir = buildcache_dir
+
+ def add_repo(self, repo: SpackDescriptor):
+ if self.repos is None:
+ self.repos = []
+ else:
+ self.repos.append(repo)
diff --git a/esd/spack_manager/__init__.py b/esd/configuration/__init__.py
similarity index 100%
rename from esd/spack_manager/__init__.py
rename to esd/configuration/__init__.py
diff --git a/esd/spack_manager/enums/__init__.py b/esd/error_handling/__init__.py
similarity index 100%
rename from esd/spack_manager/enums/__init__.py
rename to esd/error_handling/__init__.py
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 9d11b5fa46e1d87c80e258753dcfa425ea367d34..0256f886ab0cf4b958ac12d59d6fcea2d5f568ec 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -29,3 +29,13 @@ class SpackInstallPackagesException(BashCommandException):
"""
To be thrown when the spack fails to install spack packages
"""
+
+class SpackMirrorException(BashCommandException):
+ """
+ To be thrown when the spack add mirror command fails
+ """
+
+class SpackGpgException(BashCommandException):
+ """
+ To be thrown when the spack fails to create gpg keys
+ """
diff --git a/esd/model/SpackModel.py b/esd/model/SpackDescriptor.py
similarity index 57%
rename from esd/model/SpackModel.py
rename to esd/model/SpackDescriptor.py
index 4b065dba06f558ce21a0354257d77aa595bcaeb1..70e484fb3d39e4333389682d14a32ac46c08a912 100644
--- a/esd/model/SpackModel.py
+++ b/esd/model/SpackDescriptor.py
@@ -1,12 +1,13 @@
+import os
from pathlib import Path
-class SpackModel:
+class SpackDescriptor:
""""
Provides details about the spack environment
"""
- def __init__(self, env_name: str, path: Path, git_path: str = None):
+ def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
self.env_name = env_name
self.path = path
self.git_path = git_path
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_factory/SpackOperation.py
similarity index 51%
rename from esd/spack_manager/SpackManager.py
rename to esd/spack_factory/SpackOperation.py
index 65a21d6e224cb7b3c0d4150de1fefa7adbe97b4f..904582f45b6c6360edadc588c5851e2911e2f4b2 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -1,65 +1,54 @@
import os
import re
import subprocess
-from abc import ABC, abstractmethod
from pathlib import Path
-from tabnanny import check
-
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException
+ SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
from esd.logger.logger_builder import get_logger
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo
+from esd.configuration.SpackConfig import SpackConfig
+from esd.tests.testing_variables import SPACK_VERSION
+from esd.wrapper.spack_wrapper import check_spack_env
+from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-class SpackManager(ABC):
+class SpackOperation:
"""
This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
Attributes:
-----------
- env : SpackModel
+ env : SpackDescriptor
spack environment details
- repos : list[SpackModel]
+ repos : list[SpackDescriptor]
upstream_instance : str
path to Spack instance to use as upstream (optional)
"""
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
- if repos is None:
- repos = []
- self.repos = repos
- self.env = env
- self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
- self.install_dir.mkdir(parents=True, exist_ok=True)
- self.env_path = None
- if self.env and self.env.path:
- self.env.path = self.env.path.resolve()
- self.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = self.env.path / self.env.env_name
- self.upstream_instance = upstream_instance
- self.spack_dir = self.install_dir / "spack"
- self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
+ self.spack_config = spack_config
+ self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.spack_config.install_dir / 'spack'
+ self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
self.logger = logger
- self.system_name = system_name
-
- @abstractmethod
- def concretize_spack_env(self, force=True):
- pass
+ if self.spack_config.env and spack_config.env.path:
+ self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
def create_fetch_spack_environment(self):
- if self.env.git_path:
- git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
+ if self.spack_config.env.git_path:
+ git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
+ self.spack_config.env.git_path,
+ logger=self.logger)
else:
- os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
check=True, logger=self.logger,
- debug_msg=f"Created {self.env.env_name} spack environment",
- exception_msg=f"Failed to create {self.env.env_name} spack environment",
+ info_msg=f"Created {self.spack_config.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
exception=BashCommandException)
def setup_spack_env(self):
@@ -67,22 +56,22 @@ class SpackManager(ABC):
This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
"""
bashrc_path = os.path.expanduser("~/.bashrc")
- if self.system_name:
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
- os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_config.system_name:
+ set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
+ os.environ['SYSTEMNAME'] = self.spack_config.system_name
if self.spack_dir.exists() and self.spack_dir.is_dir():
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
- bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
else:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
# Restart the bash after adding environment variables
self.create_fetch_spack_environment()
- if self.install_dir.exists():
- for repo in self.repos:
- repo_dir = self.install_dir / repo.path / repo.env_name
+ if self.spack_config.install_dir.exists():
+ for repo in self.spack_config.repos:
+ repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
if not self.spack_repo_exists(repo.env_name):
self.add_spack_repo(repo.path, repo.env_name)
@@ -92,21 +81,21 @@ class SpackManager(ABC):
def spack_repo_exists(self, repo_name: str) -> bool | None:
"""Check if the given Spack repository exists."""
- if self.env is None:
+ if self.spack_config.env is None:
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if {repo_name} exists')
+ info_msg=f'Checking if {repo_name} exists')
if result is None:
return False
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ f'{self.spack_command_on_env} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ info_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -116,10 +105,10 @@ class SpackManager(ABC):
def spack_env_exists(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+ self.spack_command_on_env,
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if environment {self.env.env_name} exists')
+ info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
if result is None:
return False
return True
@@ -128,20 +117,20 @@ class SpackManager(ABC):
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
exception=BashCommandException)
@check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ f'{self.spack_command_on_env} && spack compiler list',
check=True, logger=self.logger,
capture_output=True, text=True,
- debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
- exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
exception=BashCommandException)
# todo add error handling and tests
if result.stdout is None:
@@ -151,40 +140,87 @@ class SpackManager(ABC):
# Find the first occurrence of a GCC compiler using regex
match = re.search(r"gcc@([\d\.]+)", result.stdout)
gcc_version = match.group(1)
- self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
capture_output=True, text=True, check=True,
logger=self.logger,
- debug_msg=f"Getting spack version",
+ info_msg=f"Getting spack version",
exception_msg=f"Error retrieving Spack version")
if spack_version:
return spack_version.stdout.strip().split()[0]
return None
+ @check_spack_env
+ def concretize_spack_env(self, force=True):
+ force = '--force' if force else ''
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack concretize {force}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Concertization step for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
+ exception=SpackConcertizeException)
+
+ def create_gpg_keys(self, gpg_name='example', gpg_mail='example@example.com'):
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack gpg init && spack gpg create {gpg_name} {gpg_mail}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.env_name}',
+ exception=SpackGpgException)
+
+ def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
+ autopush = '--autopush' if autopush else ''
+ signed = '--signed' if signed else ''
+ if global_mirror:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException)
+ else:
+ check_spack_env(
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException))
+
+ def remove_mirror(self, mirror_name: str):
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Removing mirror {mirror_name}',
+ exception_msg=f'Failed to remove mirror {mirror_name}',
+ exception=SpackMirrorException)
+
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
fresh = '--fresh' if fresh else ''
debug = '--debug' if debug else ''
install_result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
logger=self.logger,
- debug_msg=f"Installing spack packages for {self.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
exception=SpackInstallPackagesException)
- with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
- log_file.write(install_result.stdout)
- log_file.write("\n--- STDERR ---\n")
- log_file.write(install_result.stderr)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
- def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
@@ -213,18 +249,18 @@ class SpackManager(ABC):
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
- debug_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
+ info_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
self.logger.info("Spack install completed")
# Restart Bash after the installation ends
os.system("exec bash")
# Configure upstream Spack instance if specified
- if self.upstream_instance:
+ if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
with open(upstreams_yaml_path, "w") as file:
file.write(f"""upstreams:
upstream-spack-instance:
- install_tree: {self.upstream_instance}/spack/opt/spack
+ install_tree: {self.spack_config.upstream_instance}/spack/opt/spack
""")
self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_factory/SpackOperationCreator.py b/esd/spack_factory/SpackOperationCreator.py
new file mode 100644
index 0000000000000000000000000000000000000000..8369c5ca0efb2865e706f1968882fc53a4e4d3d9
--- /dev/null
+++ b/esd/spack_factory/SpackOperationCreator.py
@@ -0,0 +1,14 @@
+from esd.configuration.SpackConfig import SpackConfig
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+
+
+class SpackOperationCreator:
+ @staticmethod
+ def get_spack_operator(spack_config: SpackConfig = None):
+ if spack_config is None:
+ return SpackOperation(SpackConfig())
+ elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
+ return SpackOperation(spack_config)
+ else:
+ return SpackOperationUseCache(spack_config)
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
new file mode 100644
index 0000000000000000000000000000000000000000..313522d235515955124ba44536976d995e17ff77
--- /dev/null
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -0,0 +1,32 @@
+import os
+from esd.build_cache.BuildCacheManager import BuildCacheManager
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationUseCache(SpackOperation):
+ """
+ This class uses caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
+ cache_version_build='cache'):
+ super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=cache_version_build)
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_factory/__init__.py
similarity index 100%
rename from esd/spack_manager/factory/__init__.py
rename to esd/spack_factory/__init__.py
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
deleted file mode 100644
index a24358394d19ee1903835f7eafea8f8e8c964fa6..0000000000000000000000000000000000000000
--- a/esd/spack_manager/enums/SpackManagerEnum.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from enum import Enum
-
-
-class SpackManagerEnum(Enum):
- FROM_SCRATCH = "from_scratch",
- FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
deleted file mode 100644
index 5b66f5c51bb6bb3377749998c1e9377fdcd70f94..0000000000000000000000000000000000000000
--- a/esd/spack_manager/factory/SpackManagerBuildCache.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-
-
-class SpackManagerBuildCache(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- def concretize_spack_env(self, force=True):
- pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
deleted file mode 100644
index 6eb26a04cfbb4fc7be66e1a2900d9698bda91b5b..0000000000000000000000000000000000000000
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-
-class SpackManagerCreator:
- @staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env, repos, upstream_instance, system_name)
- elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
deleted file mode 100644
index 3dbc25f68e4bb64204b6901a356b6feaca054c9a..0000000000000000000000000000000000000000
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-from esd.utils.utils import run_command
-
-
-class SpackManagerScratch(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def concretize_spack_env(self, force=True):
- force = '--force' if force else ''
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Concertization step for {self.env.env_name}',
- exception_msg=f'Failed the concertization step for {self.env.env_name}',
- exception=SpackConcertizeException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
diff --git a/esd/tests/spack_from_scratch_test.py b/esd/tests/spack_from_scratch_test.py
index 782d6be8f7504ad2116b761b15504290de0663e9..cdc405e744fb70389ce474fa04a59c973da8e58d 100644
--- a/esd/tests/spack_from_scratch_test.py
+++ b/esd/tests/spack_from_scratch_test.py
@@ -1,173 +1,204 @@
-import os
from pathlib import Path
import pytest
+from esd.configuration.SpackConfig import SpackConfig
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
-from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
+from esd.spack_factory.SpackOperationCreator import SpackOperationCreator
+from esd.model.SpackDescriptor import SpackDescriptor
+from esd.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
from esd.utils.utils import file_exists_and_not_empty
-SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
-test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
-ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
-
def test_spack_repo_exists_1():
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
- assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
+ spack_operation = SpackOperationCreator.get_spack_operator()
+ spack_operation.install_spack()
+ assert spack_operation.spack_repo_exists('ebrains-spack-builds') == False
def test_spack_repo_exists_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
with pytest.raises(NoSpackEnvironmentException):
- spack_manager.spack_repo_exists(env.env_name)
+ spack_operation.spack_repo_exists(env.env_name)
def test_spack_repo_exists_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == False
+ env = SpackDescriptor('ebrains-spack-builds', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ print(spack_operation.get_spack_installed_version())
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == False
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == False
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == False
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == True
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_repo_exists(env.env_name) == True
def test_spack_from_scratch_setup_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('new_env1', install_dir)
+ env = SpackDescriptor('new_env1', install_dir)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
with pytest.raises(BashCommandException):
- spack_manager.setup_spack_env()
+ spack_operation.setup_spack_env()
def test_spack_from_scratch_setup_4(tmp_path):
install_dir = tmp_path
- env = SpackModel('new_env2', install_dir)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- assert spack_manager.spack_env_exists() == True
+ env = SpackDescriptor('new_env2', install_dir)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.spack_env_exists() == True
def test_spack_not_a_valid_repo():
- env = SpackModel('ebrains-spack-builds', Path(), None)
+ env = SpackDescriptor('ebrains-spack-builds', Path(), None)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo],
- system_name='ebrainslab')
- with pytest.raises(NoSpackEnvironmentException):
- spack_manager.add_spack_repo(repo.path, repo.env_name)
+ config = SpackConfig(env=env, system_name='ebrainslab')
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ with pytest.raises(BashCommandException):
+ spack_operation.add_spack_repo(repo.path, repo.env_name)
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=False)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
- spack_manager.setup_spack_env()
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
+ config.add_repo(repo)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == False
def test_spack_from_scratch_concretize_4(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=False)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_5(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_6(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=False)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=False)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_concretize_7(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_install(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
- repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
- spack_manager.setup_spack_env()
- spack_manager.concretize_spack_env(force=True)
- concretization_file_path = spack_manager.env_path / 'spack.lock'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True)
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
- install_result = spack_manager.install_packages(jobs=2, signed=False, fresh=True, debug=False)
+ install_result = spack_operation.install_packages(jobs=2, signed=False, fresh=True, debug=False)
assert install_result.returncode == 0
diff --git a/esd/tests/spack_install_test.py b/esd/tests/spack_install_test.py
index 9a32d5c77ea97070b8eb1e1876cadfa4c19957bf..28f8268e668ee2036930b9724ea72cd0320e82d4 100644
--- a/esd/tests/spack_install_test.py
+++ b/esd/tests/spack_install_test.py
@@ -1,21 +1,12 @@
import pytest
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.tests.testing_variables import SPACK_VERSION
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-SPACK_VERSION = "0.22.0"
-
-# we need this test to run first so that spack is installed only once for all the tests
+# run this test first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
- spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
- installed_spack_version = spack_manager.get_spack_installed_version()
- assert SPACK_VERSION == installed_spack_version
-
-
-def test_spack_install_buildcache():
- spack_manager = SpackManagerBuildCache()
- installed_spack_version = spack_manager.get_spack_installed_version()
+ spack_operation = SpackOperation()
+ spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
+ installed_spack_version = spack_operation.get_spack_installed_version()
assert SPACK_VERSION == installed_spack_version
diff --git a/esd/tests/testing_variables.py b/esd/tests/testing_variables.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab95bfa1e02658b30cef40bd6654915a51d33ad3
--- /dev/null
+++ b/esd/tests/testing_variables.py
@@ -0,0 +1,6 @@
+import os
+
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
+SPACK_VERSION = "0.22.0"
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 173347d06b08f6092a06b4590fd969d0fb66535e..033cbc54bbcd33a5f6c59399227d71b620e7544f 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -1,33 +1,35 @@
import logging
+import os
import shutil
import subprocess
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException
+import re
-def clean_up(dirs: list[str], logging, ignore_errors=True):
+def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
for cleanup_dir in dirs:
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
- logging.info(f"Removing {cleanup_dir}")
+ logger.info(f"Removing {cleanup_dir}")
try:
shutil.rmtree(Path(cleanup_dir))
except OSError as e:
- logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ logger.error(f"Failed to remove {cleanup_dir}: {e}")
if not ignore_errors:
raise e
else:
- logging.info(f"{cleanup_dir} does not exist")
+ logger.info(f"{cleanup_dir} does not exist")
-def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
exception=None, **kwargs):
try:
- logger.debug(f'{debug_msg}: args: {args}')
+ logger.info(f'{info_msg}: args: {args}')
return subprocess.run(args, **kwargs)
except subprocess.CalledProcessError as e:
if exception_msg is not None:
@@ -46,12 +48,41 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
"-c", "feature.manyFiles=true",
git_path, dir
, check=True, logger=logger,
- debug_msg=f'Cloned repository {repo_name}',
+ info_msg=f'Cloned repository {repo_name}',
exception_msg=f'Failed to clone repository: {repo_name}',
exception=BashCommandException)
else:
- logger.debug(f'Repository {repo_name} already cloned.')
+ logger.info(f'Repository {repo_name} already cloned.')
def file_exists_and_not_empty(file: Path) -> bool:
return file.is_file() and file.stat().st_size > 0
+
+
+def log_command(results, log_file: str):
+ with open(log_file, "w") as log_file:
+ log_file.write(results.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(results.stderr)
+
+
+def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
+ logger: logging = logging.getLogger(__name__)):
+ """Update or add an environment variable in ~/.bashrc."""
+ with open(bashrc_path, "r") as file:
+ lines = file.readlines()
+ pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
+ updated = False
+ # Modify the existing variable if found
+ for i, line in enumerate(lines):
+ if pattern.match(line):
+ lines[i] = f'export {var_name}={value}\n'
+ updated = True
+ break
+ if not updated:
+ lines.append(f'\nexport {var_name}={value}\n')
+ logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
+ else:
+ logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
+ with open(bashrc_path, "w") as file:
+ file.writelines(lines)
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/wrapper/__init__.py
similarity index 100%
rename from esd/spack_manager/wrapper/__init__.py
rename to esd/wrapper/__init__.py
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/wrapper/spack_wrapper.py
similarity index 100%
rename from esd/spack_manager/wrapper/spack_wrapper.py
rename to esd/wrapper/spack_wrapper.py