diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index 839ac52400cae71fa760672321de51761c9fd1a8..cccb584695b9c59f0b4ca1ab2eb981f5c9ad9242 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -12,36 +12,39 @@ class BuildCacheManager(BuildCacheManagerInterface):
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', insecure=False):
+ def __init__(self, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
+ auth_backend='basic',
+ insecure=False):
self._logger = get_logger(__name__, BuildCacheManager.__name__)
- self._home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self._registry_project = os.environ.get("REGISTRY_PROJECT")
+ self._registry_project = registry_project
- self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
- self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
+ self._registry_username = registry_username
+ self._registry_password = registry_password
- self._registry_host = str(os.environ.get("REGISTRY_HOST"))
+ self._registry_host = registry_host
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend, insecure=insecure)
+ self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend,
+ insecure=insecure)
self._client.login(username=self._registry_username, password=self._registry_password)
- self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/cache'
+ self.cache_version = cache_version
+ self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/{self.cache_version}'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self._home_path / out_dir
+ build_cache_path = out_dir.resolve()
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
self._logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
- rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
- target = f"{self._registry_host}/{self._registry_project}/cache:{str(sub_path.env_name)}"
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
+ target = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{str(sub_path.name)}"
try:
self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
self._client.push(
@@ -51,7 +54,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self._logger.info(f"Successfully pushed {sub_path.env_name}")
+ self._logger.info(f"Successfully pushed {sub_path.name}")
except Exception as e:
self._logger.error(
f"An error occurred while pushing: {e}")
@@ -72,16 +75,17 @@ class BuildCacheManager(BuildCacheManagerInterface):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self._home_path / in_dir
+ build_cache_path = in_dir.resolve()
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self._registry_host}/{self._registry_project}/cache:{tag}"
+ ref = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self._client.get_manifest(f'{self._registry_host}/{self._registry_project}/cache:{tag}')[
+ self._client.get_manifest(
+ f'{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}')[
'annotations'][
'path']
try:
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 9a32d5c77ea97070b8eb1e1876cadfa4c19957bf..28f8268e668ee2036930b9724ea72cd0320e82d4 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -1,21 +1,12 @@
import pytest
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.tests.testing_variables import SPACK_VERSION
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-SPACK_VERSION = "0.22.0"
-
-# we need this test to run first so that spack is installed only once for all the tests
+# run this test first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
- spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
- installed_spack_version = spack_manager.get_spack_installed_version()
- assert SPACK_VERSION == installed_spack_version
-
-
-def test_spack_install_buildcache():
- spack_manager = SpackManagerBuildCache()
- installed_spack_version = spack_manager.get_spack_installed_version()
+ spack_operation = SpackOperation()
+ spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
+ installed_spack_version = spack_operation.get_spack_installed_version()
assert SPACK_VERSION == installed_spack_version
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
new file mode 100644
index 0000000000000000000000000000000000000000..93a2e87423b871b0b9b4f583efec79e328739edf
--- /dev/null
+++ b/esd/configuration/SpackConfig.py
@@ -0,0 +1,25 @@
+import os
+from pathlib import Path
+from esd.model import SpackDescriptor
+
+
+class SpackConfig:
+ def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
+ install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
+ concretization_dir: Path = None, buildcache_dir: Path = None):
+ self.env = env
+ if repos is None:
+ self.repos = []
+ else:
+ self.repos = repos
+ self.install_dir = install_dir
+ self.upstream_instance = upstream_instance
+ self.system_name = system_name
+ self.concretization_dir = concretization_dir
+ self.buildcache_dir = buildcache_dir
+
+ def add_repo(self, repo: SpackDescriptor):
+ if self.repos is None:
+ self.repos = []
+ else:
+ self.repos.append(repo)
diff --git a/esd/spack_manager/__init__.py b/esd/configuration/__init__.py
similarity index 100%
rename from esd/spack_manager/__init__.py
rename to esd/configuration/__init__.py
diff --git a/esd/spack_manager/enums/__init__.py b/esd/error_handling/__init__.py
similarity index 100%
rename from esd/spack_manager/enums/__init__.py
rename to esd/error_handling/__init__.py
diff --git a/esd/model/SpackModel.py b/esd/model/SpackDescriptor.py
similarity index 57%
rename from esd/model/SpackModel.py
rename to esd/model/SpackDescriptor.py
index 4b065dba06f558ce21a0354257d77aa595bcaeb1..70e484fb3d39e4333389682d14a32ac46c08a912 100644
--- a/esd/model/SpackModel.py
+++ b/esd/model/SpackDescriptor.py
@@ -1,12 +1,13 @@
+import os
from pathlib import Path
-class SpackModel:
+class SpackDescriptor:
""""
Provides details about the spack environment
"""
- def __init__(self, env_name: str, path: Path, git_path: str = None):
+ def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
self.env_name = env_name
self.path = path
self.git_path = git_path
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_factory/SpackOperation.py
similarity index 62%
rename from esd/spack_manager/SpackManager.py
rename to esd/spack_factory/SpackOperation.py
index bf381c394ca276e3e32664caa2319feec5077e7e..29f44f4904ac8b112bd3ca5eb27748e8b525e32c 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -3,63 +3,57 @@ import re
import subprocess
from abc import ABC, abstractmethod
from pathlib import Path
-from tabnanny import check
-
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException
+ SpackInstallPackagesException, SpackConcertizeException
from esd.logger.logger_builder import get_logger
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo, log_command
+from esd.configuration.SpackConfig import SpackConfig
+from esd.tests.testing_variables import SPACK_VERSION
+from esd.wrapper.spack_wrapper import check_spack_env
+from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-class SpackManager(ABC):
+class SpackOperation(ABC):
"""
This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
Attributes:
-----------
- env : SpackModel
+ env : SpackDescriptor
spack environment details
- repos : list[SpackModel]
+ repos : list[SpackDescriptor]
upstream_instance : str
path to Spack instance to use as upstream (optional)
"""
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
- if repos is None:
- repos = []
- self.repos = repos
- self.env = env
- self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
- self.install_dir.mkdir(parents=True, exist_ok=True)
- self.env_path = None
- if self.env and self.env.path:
- self.env.path = self.env.path.resolve()
- self.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = self.env.path / self.env.env_name
- self.upstream_instance = upstream_instance
- self.spack_dir = self.install_dir / "spack"
- self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
+ self.spack_config = spack_config
+ self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_dir = self.spack_config.install_dir / 'spack'
+ self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
self.logger = logger
- self.system_name = system_name
+ if self.spack_config.env and spack_config.env.path:
+ self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
+ self.env_path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
@abstractmethod
def concretize_spack_env(self, force=True):
pass
def create_fetch_spack_environment(self):
- if self.env.git_path:
- git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
+ if self.spack_config.env.git_path:
+ git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
+ self.spack_config.env.git_path,
+ logger=self.logger)
else:
- os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+ os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
check=True, logger=self.logger,
- debug_msg=f"Created {self.env.env_name} spack environment",
- exception_msg=f"Failed to create {self.env.env_name} spack environment",
+ info_msg=f"Created {self.spack_config.env.env_name} spack environment",
+ exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
exception=BashCommandException)
def setup_spack_env(self):
@@ -67,22 +61,20 @@ class SpackManager(ABC):
This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
"""
bashrc_path = os.path.expanduser("~/.bashrc")
- if self.system_name:
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
- os.environ['SYSTEMNAME'] = self.system_name
+ if self.spack_config.system_name:
+ set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
+ os.environ['SYSTEMNAME'] = self.spack_config.system_name
if self.spack_dir.exists() and self.spack_dir.is_dir():
- with open(bashrc_path, "a") as bashrc:
- bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
- bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
else:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
# Restart the bash after adding environment variables
self.create_fetch_spack_environment()
- if self.install_dir.exists():
- for repo in self.repos:
- repo_dir = self.install_dir / repo.path / repo.env_name
+ if self.spack_config.install_dir.exists():
+ for repo in self.spack_config.repos:
+ repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
if not self.spack_repo_exists(repo.env_name):
self.add_spack_repo(repo.path, repo.env_name)
@@ -92,21 +84,21 @@ class SpackManager(ABC):
def spack_repo_exists(self, repo_name: str) -> bool | None:
"""Check if the given Spack repository exists."""
- if self.env is None:
+ if self.spack_config.env is None:
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if {repo_name} exists')
+ info_msg=f'Checking if {repo_name} exists')
if result is None:
return False
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ f'{self.spack_command_on_env} && spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ info_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -116,10 +108,10 @@ class SpackManager(ABC):
def spack_env_exists(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+ self.spack_command_on_env,
check=True,
capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if environment {self.env.env_name} exists')
+ info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
if result is None:
return False
return True
@@ -128,20 +120,20 @@ class SpackManager(ABC):
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
exception=BashCommandException)
@check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ f'{self.spack_command_on_env} && spack compiler list',
check=True, logger=self.logger,
capture_output=True, text=True,
- debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
- exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
exception=BashCommandException)
# todo add error handling and tests
if result.stdout is None:
@@ -151,37 +143,48 @@ class SpackManager(ABC):
# Find the first occurrence of a GCC compiler using regex
match = re.search(r"gcc@([\d\.]+)", result.stdout)
gcc_version = match.group(1)
- self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
capture_output=True, text=True, check=True,
logger=self.logger,
- debug_msg=f"Getting spack version",
+ info_msg=f"Getting spack version",
exception_msg=f"Error retrieving Spack version")
if spack_version:
return spack_version.stdout.strip().split()[0]
return None
+ @check_spack_env
+ def concretize_spack_env(self, force=True):
+ force = '--force' if force else ''
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ info_msg=f'Concertization step for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
+ exception=SpackConcertizeException)
+
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
fresh = '--fresh' if fresh else ''
debug = '--debug' if debug else ''
install_result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
logger=self.logger,
- debug_msg=f"Installing spack packages for {self.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
exception=SpackInstallPackagesException)
log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
- def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
@@ -210,18 +213,18 @@ class SpackManager(ABC):
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
- debug_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
+ info_msg='Adding permissions to the logged in user')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
self.logger.info("Spack install completed")
# Restart Bash after the installation ends
os.system("exec bash")
# Configure upstream Spack instance if specified
- if self.upstream_instance:
+ if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
with open(upstreams_yaml_path, "w") as file:
file.write(f"""upstreams:
upstream-spack-instance:
- install_tree: {self.upstream_instance}/spack/opt/spack
+ install_tree: {self.spack_config.upstream_instance}/spack/opt/spack
""")
self.logger.info("Added upstream spack instance")
diff --git a/esd/spack_factory/SpackOperationCreator.py b/esd/spack_factory/SpackOperationCreator.py
new file mode 100644
index 0000000000000000000000000000000000000000..8369c5ca0efb2865e706f1968882fc53a4e4d3d9
--- /dev/null
+++ b/esd/spack_factory/SpackOperationCreator.py
@@ -0,0 +1,14 @@
+from esd.configuration.SpackConfig import SpackConfig
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+
+
+class SpackOperationCreator:
+ @staticmethod
+ def get_spack_operator(spack_config: SpackConfig = None):
+ if spack_config is None:
+ return SpackOperation(SpackConfig())
+ elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
+ return SpackOperation(spack_config)
+ else:
+ return SpackOperationUseCache(spack_config)
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
new file mode 100644
index 0000000000000000000000000000000000000000..15a3822fc982788b3df997b6dfb15e1efa5100b1
--- /dev/null
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -0,0 +1,19 @@
+from esd.logger.logger_builder import get_logger
+from esd.spack_factory.SpackOperation import SpackOperation
+from esd.configuration.SpackConfig import SpackConfig
+
+
+class SpackOperationUseCache(SpackOperation):
+ """
+ This class uses caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ super().__init__(spack_config, logger=get_logger(__name__))
+
+ def setup_spack_env(self):
+ super().setup_spack_env()
+ # todo add buildcache to the spack environment
+
+ def concretize_spack_env(self, force=True):
+ pass
diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_factory/__init__.py
similarity index 100%
rename from esd/spack_manager/factory/__init__.py
rename to esd/spack_factory/__init__.py
diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py
deleted file mode 100644
index a24358394d19ee1903835f7eafea8f8e8c964fa6..0000000000000000000000000000000000000000
--- a/esd/spack_manager/enums/SpackManagerEnum.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from enum import Enum
-
-
-class SpackManagerEnum(Enum):
- FROM_SCRATCH = "from_scratch",
- FROM_BUILDCACHE = "from_buildcache",
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
deleted file mode 100644
index 5b66f5c51bb6bb3377749998c1e9377fdcd70f94..0000000000000000000000000000000000000000
--- a/esd/spack_manager/factory/SpackManagerBuildCache.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-
-
-class SpackManagerBuildCache(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def setup_spack_env(self):
- super().setup_spack_env()
- # todo add buildcache to the spack environment
-
- def concretize_spack_env(self, force=True):
- pass
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
deleted file mode 100644
index 6eb26a04cfbb4fc7be66e1a2900d9698bda91b5b..0000000000000000000000000000000000000000
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
-from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-
-
-class SpackManagerCreator:
- @staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env, repos, upstream_instance, system_name)
- elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
deleted file mode 100644
index 3dbc25f68e4bb64204b6901a356b6feaca054c9a..0000000000000000000000000000000000000000
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
-from esd.model.SpackModel import SpackModel
-from esd.spack_manager.SpackManager import SpackManager
-from esd.logger.logger_builder import get_logger
-from esd.utils.utils import run_command
-
-
-class SpackManagerScratch(SpackManager):
- def __init__(self, env: SpackModel = None, repos=None,
- upstream_instance=None, system_name: str = None):
- super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
-
- def concretize_spack_env(self, force=True):
- force = '--force' if force else ''
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Concertization step for {self.env.env_name}',
- exception_msg=f'Failed the concertization step for {self.env.env_name}',
- exception=SpackConcertizeException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 3ce70f252a4b5ff701c76b59eeaf3d1c703799d2..033cbc54bbcd33a5f6c59399227d71b620e7544f 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -5,30 +5,31 @@ import subprocess
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException
+import re
-def clean_up(dirs: list[str], logging, ignore_errors=True):
+def clean_up(dirs: list[str], logger: logging = logging.getLogger(__name__), ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
for cleanup_dir in dirs:
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
- logging.info(f"Removing {cleanup_dir}")
+ logger.info(f"Removing {cleanup_dir}")
try:
shutil.rmtree(Path(cleanup_dir))
except OSError as e:
- logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ logger.error(f"Failed to remove {cleanup_dir}: {e}")
if not ignore_errors:
raise e
else:
- logging.info(f"{cleanup_dir} does not exist")
+ logger.info(f"{cleanup_dir} does not exist")
-def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+def run_command(*args, logger=logging.getLogger(__name__), info_msg: str = '', exception_msg: str = None,
exception=None, **kwargs):
try:
- logger.debug(f'{debug_msg}: args: {args}')
+ logger.info(f'{info_msg}: args: {args}')
return subprocess.run(args, **kwargs)
except subprocess.CalledProcessError as e:
if exception_msg is not None:
@@ -47,11 +48,11 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
"-c", "feature.manyFiles=true",
git_path, dir
, check=True, logger=logger,
- debug_msg=f'Cloned repository {repo_name}',
+ info_msg=f'Cloned repository {repo_name}',
exception_msg=f'Failed to clone repository: {repo_name}',
exception=BashCommandException)
else:
- logger.debug(f'Repository {repo_name} already cloned.')
+ logger.info(f'Repository {repo_name} already cloned.')
def file_exists_and_not_empty(file: Path) -> bool:
@@ -63,3 +64,25 @@ def log_command(results, log_file: str):
log_file.write(results.stdout)
log_file.write("\n--- STDERR ---\n")
log_file.write(results.stderr)
+
+
+def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
+ logger: logging = logging.getLogger(__name__)):
+ """Update or add an environment variable in ~/.bashrc."""
+ with open(bashrc_path, "r") as file:
+ lines = file.readlines()
+ pattern = re.compile(rf'^\s*export\s+{var_name}=.*$')
+ updated = False
+ # Modify the existing variable if found
+ for i, line in enumerate(lines):
+ if pattern.match(line):
+ lines[i] = f'export {var_name}={value}\n'
+ updated = True
+ break
+ if not updated:
+ lines.append(f'\nexport {var_name}={value}\n')
+ logger.info(f"Added in {bashrc_path} with: export {var_name}={value}")
+ else:
+ logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
+ with open(bashrc_path, "w") as file:
+ file.writelines(lines)
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/wrapper/__init__.py
similarity index 100%
rename from esd/spack_manager/wrapper/__init__.py
rename to esd/wrapper/__init__.py
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/wrapper/spack_wrapper.py
similarity index 100%
rename from esd/spack_manager/wrapper/spack_wrapper.py
rename to esd/wrapper/spack_wrapper.py