diff --git a/esd/configuration/GpgConfig.py b/esd/configuration/GpgConfig.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8f0c2d3bc0f39db8c5b251d9ffc6f6fa3a577ec
--- /dev/null
+++ b/esd/configuration/GpgConfig.py
@@ -0,0 +1,7 @@
+class GpgConfig:
+ """
+ Configuration for gpg key used by spack
+ """
+ def __init__(self, gpg_name='example', gpg_mail='example@example.com'):
+ self.name = gpg_name
+ self.mail = gpg_mail
diff --git a/esd/configuration/SpackConfig.py b/esd/configuration/SpackConfig.py
index 93a2e87423b871b0b9b4f583efec79e328739edf..b6178760ae724b8ecc5fd8c7ad576343c0624922 100644
--- a/esd/configuration/SpackConfig.py
+++ b/esd/configuration/SpackConfig.py
@@ -1,22 +1,31 @@
import os
from pathlib import Path
+
+from esd.configuration.GpgConfig import GpgConfig
from esd.model import SpackDescriptor
class SpackConfig:
def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
- concretization_dir: Path = None, buildcache_dir: Path = None):
+ concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None):
self.env = env
if repos is None:
self.repos = []
else:
self.repos = repos
self.install_dir = install_dir
+ if self.install_dir:
+ os.makedirs(self.install_dir, exist_ok=True)
self.upstream_instance = upstream_instance
self.system_name = system_name
self.concretization_dir = concretization_dir
+ if self.concretization_dir:
+ os.makedirs(self.concretization_dir, exist_ok=True)
self.buildcache_dir = buildcache_dir
+ if self.buildcache_dir:
+ os.makedirs(self.buildcache_dir, exist_ok=True)
+ self.gpg = gpg
def add_repo(self, repo: SpackDescriptor):
if self.repos is None:
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 9d11b5fa46e1d87c80e258753dcfa425ea367d34..0256f886ab0cf4b958ac12d59d6fcea2d5f568ec 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -29,3 +29,13 @@ class SpackInstallPackagesException(BashCommandException):
"""
To be thrown when the spack fails to install spack packages
"""
+
+class SpackMirrorException(BashCommandException):
+ """
+ To be thrown when the spack add mirror command fails
+ """
+
+class SpackGpgException(BashCommandException):
+ """
+ To be thrown when the spack fails to create gpg keys
+ """
diff --git a/esd/spack_factory/SpackOperation.py b/esd/spack_factory/SpackOperation.py
index 29f44f4904ac8b112bd3ca5eb27748e8b525e32c..a0b21a1c8934ac604388be139b5e9b9ec2f0b5b4 100644
--- a/esd/spack_factory/SpackOperation.py
+++ b/esd/spack_factory/SpackOperation.py
@@ -1,10 +1,9 @@
import os
import re
import subprocess
-from abc import ABC, abstractmethod
from pathlib import Path
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException, SpackConcertizeException
+ SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
from esd.logger.logger_builder import get_logger
from esd.configuration.SpackConfig import SpackConfig
from esd.tests.testing_variables import SPACK_VERSION
@@ -12,7 +11,7 @@ from esd.wrapper.spack_wrapper import check_spack_env
from esd.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable
-class SpackOperation(ABC):
+class SpackOperation:
"""
This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
@@ -38,10 +37,6 @@ class SpackOperation(ABC):
self.env_path = spack_config.env.path / spack_config.env.env_name
self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
- @abstractmethod
- def concretize_spack_env(self, force=True):
- pass
-
def create_fetch_spack_environment(self):
if self.spack_config.env.git_path:
git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
@@ -65,8 +60,10 @@ class SpackOperation(ABC):
set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger)
os.environ['SYSTEMNAME'] = self.spack_config.system_name
if self.spack_dir.exists() and self.spack_dir.is_dir():
- set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
- set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
+ set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path,
+ logger=self.logger)
self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
else:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
@@ -160,13 +157,55 @@ class SpackOperation(ABC):
def concretize_spack_env(self, force=True):
force = '--force' if force else ''
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ f'{self.spack_command_on_env} && spack concretize {force}',
check=True,
- capture_output=True, text=True, logger=self.logger,
+ logger=self.logger,
info_msg=f'Concertization step for {self.spack_config.env.env_name}',
exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
exception=SpackConcertizeException)
+ def create_gpg_keys(self):
+ if self.spack_config.gpg:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
+ exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.env_name}',
+ exception=SpackGpgException)
+ else:
+ raise SpackGpgException('No GPG configuration was defined is spack configuration')
+
+ def add_mirror(self, mirror_name: str, mirror_path: Path, signed=False, autopush=False, global_mirror=False):
+ autopush = '--autopush' if autopush else ''
+ signed = '--signed' if signed else ''
+ if global_mirror:
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException)
+ else:
+ check_spack_env(
+ run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Added mirror {mirror_name}',
+ exception_msg=f'Failed to add mirror {mirror_name}',
+ exception=SpackMirrorException))
+
+ def remove_mirror(self, mirror_name: str):
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+ check=True,
+ logger=self.logger,
+ info_msg=f'Removing mirror {mirror_name}',
+ exception_msg=f'Failed to remove mirror {mirror_name}',
+ exception=SpackMirrorException)
+
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
signed = '' if signed else '--no-check-signature'
diff --git a/esd/spack_factory/SpackOperationUseCache.py b/esd/spack_factory/SpackOperationUseCache.py
index 15a3822fc982788b3df997b6dfb15e1efa5100b1..313522d235515955124ba44536976d995e17ff77 100644
--- a/esd/spack_factory/SpackOperationUseCache.py
+++ b/esd/spack_factory/SpackOperationUseCache.py
@@ -1,3 +1,5 @@
+import os
+from esd.build_cache.BuildCacheManager import BuildCacheManager
from esd.logger.logger_builder import get_logger
from esd.spack_factory.SpackOperation import SpackOperation
from esd.configuration.SpackConfig import SpackConfig
@@ -8,8 +10,19 @@ class SpackOperationUseCache(SpackOperation):
This class uses caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='cache',
+ cache_version_build='cache'):
super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=cache_version_build)
def setup_spack_env(self):
super().setup_spack_env()