diff --git a/.env b/.env
index 93e626530f31129609801d8e4bb3e61003ad3362..786f93e2274be154065ab3b3f4e9f937c8a2c102 100644
--- a/.env
+++ b/.env
@@ -2,8 +2,7 @@ BUILDCACHE_OCI_HOST=""
BUILDCACHE_OCI_PASSWORD=""
BUILDCACHE_OCI_PROJECT=""
BUILDCACHE_OCI_USERNAME=""
-
CONCRETIZE_OCI_HOST=""
CONCRETIZE_OCI_PASSWORD=""
CONCRETIZE_OCI_PROJECT=""
-CONCRETIZE_OCI_USERNAME""
+CONCRETIZE_OCI_USERNAME=""
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2b497048d572c8df8798ae2d62fc8e13589add33..4f15b9ab2407e9d56f56ed2acb0abceab1dddf9e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,5 +39,6 @@ testing-pytest:
paths:
- test-results.xml
- .dedal.log
+ - .generate_cache.log
expire_in: 1 week
diff --git a/README.md b/README.md
index dd68dcfa9fecfd80b6cf28a890a71a80c4bed9b1..5dc3fcc178a6fa777d80e5ced20dab708ed7f69b 100644
--- a/README.md
+++ b/README.md
@@ -2,14 +2,6 @@
This repository provides functionalities to easily ```managed spack environments``` and ```helpers for the container image build flow```.
-This library runs only on different kinds of linux distribution operating systems.
-
-The lowest ```spack version``` compatible with this library is ```v0.23.0```.
-
-
- This repository also provied CLI interface. For more informations, after installing this library, call dedal --help.
-
-
**Setting up the needed environment variables**
The ````<checkout path>\dedal\.env```` file contains the environment variables required for OCI registry used for caching.
Ensure that you edit the ````<checkout path>\dedal\.env```` file to match your environment.
@@ -51,3 +43,11 @@ The lowest ```spack version``` compatible with this library is ```v0.23.0```.
For both concretization and binary caches, the cache version can be changed via the attributes ```cache_version_concretize``` and ```cache_version_build```.
The default values are ```v1```.
+
+Before using this library, the following tool must be installed on Linux distribution:
+````
+ apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+````
+````
+ python3 -m pip install --upgrade pip setuptools wheel
+````
diff --git a/dedal/bll/SpackManager.py b/dedal/bll/SpackManager.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5fae221c7093bff86a4adf541e4664fda353dff
--- /dev/null
+++ b/dedal/bll/SpackManager.py
@@ -0,0 +1,35 @@
+import os
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackManager:
+ """
+ This class defines the logic used by the CLI
+ """
+
+ def __init__(self, spack_config: SpackConfig = None, use_cache=False):
+ self._spack_config = spack_config
+ self._use_cache = use_cache
+
+ def _get_spack_operation(self):
+ return SpackOperationCreator.get_spack_operator(self._spack_config, self._use_cache)
+
+ def install_spack(self, version: str, bashrc_path=os.path.expanduser("~/.bashrc")):
+ self._get_spack_operation().install_spack(spack_version=f'v{version}', bashrc_path=bashrc_path)
+
+ def add_spack_repo(self, repo: SpackDescriptor):
+ """
+ After additional repo was added, setup_spack_env must be invoked
+ """
+ self._spack_config.add_repo(repo)
+
+ def setup_spack_env(self):
+ self._get_spack_operation().setup_spack_env()
+
+ def concretize_spack_env(self):
+ self._get_spack_operation().concretize_spack_env()
+
+ def install_packages(self, jobs: int):
+ self._get_spack_operation().install_packages(jobs=jobs)
diff --git a/dedal/cli/SpackManager.py b/dedal/bll/__init__.py
similarity index 100%
rename from dedal/cli/SpackManager.py
rename to dedal/bll/__init__.py
diff --git a/dedal/bll/cli_utils.py b/dedal/bll/cli_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..bfc74ed0262aa944c88d722a558a88fccaad0687
--- /dev/null
+++ b/dedal/bll/cli_utils.py
@@ -0,0 +1,23 @@
+import jsonpickle
+import os
+
+
+def save_config(spack_config_data, config_path: str):
+ """Save config to JSON file."""
+ with open(config_path, "w") as data_file:
+ data_file.write(jsonpickle.encode(spack_config_data))
+
+
+def load_config(config_path: str):
+ """Load config from JSON file."""
+ if os.path.exists(config_path):
+ with open(config_path, "r") as data_file:
+ data = jsonpickle.decode(data_file.read())
+ return data
+ return {}
+
+
+def clear_config(config_path: str):
+ """Delete the JSON config file."""
+ if os.path.exists(config_path):
+ os.remove(config_path)
diff --git a/dedal/cli/spack_manager_api.py b/dedal/cli/spack_manager_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..438670282b17437f752ec9fb183e054a9a3d3549
--- /dev/null
+++ b/dedal/cli/spack_manager_api.py
@@ -0,0 +1,152 @@
+import os
+from pathlib import Path
+import click
+import jsonpickle
+
+from dedal.bll.SpackManager import SpackManager
+from dedal.bll.cli_utils import save_config, load_config
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.utils.utils import resolve_path
+
+SESSION_CONFIG_PATH = os.path.expanduser(f'~/tmp/dedal/dedal_session.json')
+os.makedirs(os.path.dirname(SESSION_CONFIG_PATH), exist_ok=True)
+
+
+@click.group()
+@click.pass_context
+def cli(ctx: click.Context):
+ config = load_config(SESSION_CONFIG_PATH)
+ if ctx.invoked_subcommand not in ['set-config', 'install-spack'] and not config:
+ click.echo('No configuration set. Use `set-config` first.')
+ ctx.exit(1)
+ if config:
+ config['env_path'] = resolve_path(config['env_path'])
+ env = SpackDescriptor(config['env_name'], config['env_path'], config['env_git_path'])
+ gpg = GpgConfig(config['gpg_name'], config['gpg_mail']) if config['gpg_name'] and config['gpg_mail'] else None
+ spack_config = SpackConfig(env=env, repos=None, install_dir=config['install_dir'],
+ upstream_instance=config['upstream_instance'],
+ concretization_dir=config['concretization_dir'],
+ buildcache_dir=config['buildcache_dir'],
+ system_name=config['system_name'], gpg=gpg,
+ use_spack_global=config['use_spack_global'])
+ ctx.obj = SpackManager(spack_config, use_cache=config['use_cache'])
+
+
+@cli.command()
+@click.option('--use_cache', is_flag=True, default=False, help='Enables cashing')
+@click.option('--use_spack_global', is_flag=True, default=False, help='Uses spack installed globally on the os')
+@click.option('--env_name', type=str, default=None, help='Environment name')
+@click.option('--env_path', type=str, default=None, help='Environment path to download locally')
+@click.option('--env_git_path', type=str, default=None, help='Git path to download the environment')
+@click.option('--install_dir', type=str,
+ help='Install directory for installing spack; spack environments and repositories are stored here')
+@click.option('--upstream_instance', type=str, default=None, help='Upstream instance for spack environment')
+@click.option('--system_name', type=str, default=None, help='System name; it is used inside the spack environment')
+@click.option('--concretization_dir', type=str, default=None,
+ help='Directory where the concretization caching (spack.lock) will be downloaded')
+@click.option('--buildcache_dir', type=str, default=None,
+ help='Directory where the binary caching is downloaded for the spack packages')
+@click.option('--gpg_name', type=str, default=None, help='Gpg name')
+@click.option('--gpg_mail', type=str, default=None, help='Gpg mail contact address')
+@click.option('--cache_version_concretize', type=str, default='v1', help='Cache version for concretizaion data')
+@click.option('--cache_version_build', type=str, default='v1', help='Cache version for binary caches data')
+def set_config(use_cache, env_name, env_path, env_git_path, install_dir, upstream_instance, system_name,
+ concretization_dir,
+ buildcache_dir, gpg_name, gpg_mail, use_spack_global, cache_version_concretize, cache_version_build):
+ """Set configuration parameters for tahe session."""
+ spack_config_data = {
+ 'use_cache': use_cache,
+ 'env_name': env_name,
+ 'env_path': env_path,
+ 'env_git_path': env_git_path,
+ 'install_dir': install_dir,
+ 'upstream_instance': upstream_instance,
+ 'system_name': system_name,
+ 'concretization_dir': Path(concretization_dir) if concretization_dir else None,
+ 'buildcache_dir': Path(buildcache_dir) if buildcache_dir else None,
+ 'gpg_name': gpg_name,
+ 'gpg_mail': gpg_mail,
+ 'use_spack_global': use_spack_global,
+ 'repos': [],
+ 'cache_version_concretize': cache_version_concretize,
+ 'cache_version_build': cache_version_build,
+ }
+ save_config(spack_config_data, SESSION_CONFIG_PATH)
+ click.echo('Configuration saved.')
+
+
+@click.command()
+def show_config():
+ """Show the current configuration."""
+ config = load_config(SESSION_CONFIG_PATH)
+ if config:
+ click.echo(jsonpickle.encode(config, indent=2))
+ else:
+ click.echo('No configuration set. Use `set-config` first.')
+
+
+@cli.command()
+@click.option('--spack_version', type=str, default='0.23.0', help='Spack version')
+@click.option('--bashrc_path', type=str, default='~/.bashrc', help='Path to .bashrc')
+@click.pass_context
+def install_spack(ctx: click.Context, spack_version: str, bashrc_path: str):
+ """Install spack in the install_dir folder"""
+ if ctx.obj is None:
+ SpackManager().install_spack(spack_version, bashrc_path)
+ else:
+ ctx.obj.install_spack(spack_version, bashrc_path)
+
+
+@cli.command()
+@click.option('--repo_name', type=str, required=True, default=None, help='Repository name')
+@click.option('--path', type=str, required=True, default=None, help='Repository path to download locally')
+@click.option('--git_path', type=str, required=True, default=None, help='Git path to download the repository')
+def add_spack_repo(repo_name: str, path: str, git_path: str = None):
+ """Adds a spack repository to the spack environments. The setup command must be rerun."""
+ path = resolve_path(path)
+ repo = SpackDescriptor(repo_name, path, git_path)
+ config = load_config(SESSION_CONFIG_PATH)
+ config['repos'].append(repo)
+ save_config(config, SESSION_CONFIG_PATH)
+ click.echo('dedal setup_spack_env must be reran after each repo is added for the environment.')
+
+
+@cli.command()
+@click.pass_context
+def setup_spack_env(ctx: click.Context):
+ """Setups a spack environment according to the given configuration."""
+ ctx.obj.setup_spack_env()
+
+
+@cli.command()
+@click.pass_context
+def concretize(ctx: click.Context):
+ """Spack concretization step"""
+ ctx.obj.concretize_spack_env()
+
+
+@cli.command()
+@click.option('--jobs', type=int, default=2, help='Number of parallel jobs for spack installation')
+@click.pass_context
+def install_packages(ctx: click.Context, jobs):
+ """Installs spack packages present in the spack environment defined in configuration"""
+ ctx.obj.install_packages(jobs=jobs)
+
+
+@click.command()
+def clear_config():
+ """Clear stored configuration"""
+ if os.path.exists(SESSION_CONFIG_PATH):
+ os.remove(SESSION_CONFIG_PATH)
+ click.echo('Configuration cleared!')
+ else:
+ click.echo('No configuration to clear.')
+
+
+cli.add_command(show_config)
+cli.add_command(clear_config)
+
+if __name__ == '__main__':
+ cli()
diff --git a/dedal/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py
index 0d4706796ec13d6bf733c3d68459e455a6da53b7..d76783ec507bd1d0d71f4cf14cc068e831f6a357 100644
--- a/dedal/configuration/SpackConfig.py
+++ b/dedal/configuration/SpackConfig.py
@@ -1,31 +1,30 @@
import os
from pathlib import Path
-
from dedal.configuration.GpgConfig import GpgConfig
from dedal.model import SpackDescriptor
+from dedal.utils.utils import resolve_path
class SpackConfig:
def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
- concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None):
+ concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None,
+ use_spack_global=False, cache_version_concretize='v1',
+ cache_version_build='v1'):
self.env = env
if repos is None:
self.repos = []
else:
self.repos = repos
- self.install_dir = install_dir
- if self.install_dir:
- os.makedirs(self.install_dir, exist_ok=True)
self.upstream_instance = upstream_instance
self.system_name = system_name
- self.concretization_dir = concretization_dir
- if self.concretization_dir:
- os.makedirs(self.concretization_dir, exist_ok=True)
- self.buildcache_dir = buildcache_dir
- if self.buildcache_dir:
- os.makedirs(self.buildcache_dir, exist_ok=True)
+ self.concretization_dir = concretization_dir if concretization_dir is None else resolve_path(concretization_dir)
+ self.buildcache_dir = buildcache_dir if buildcache_dir is None else resolve_path(buildcache_dir)
+ self.install_dir = resolve_path(install_dir)
self.gpg = gpg
+ self.use_spack_global = use_spack_global
+ self.cache_version_concretize = cache_version_concretize
+ self.cache_version_build = cache_version_build
def add_repo(self, repo: SpackDescriptor):
if self.repos is None:
diff --git a/dedal/error_handling/exceptions.py b/dedal/error_handling/exceptions.py
index 0256f886ab0cf4b958ac12d59d6fcea2d5f568ec..4653e72f54481e2d86d90afc3ab103744243b37d 100644
--- a/dedal/error_handling/exceptions.py
+++ b/dedal/error_handling/exceptions.py
@@ -39,3 +39,8 @@ class SpackGpgException(BashCommandException):
"""
To be thrown when the spack fails to create gpg keys
"""
+
+class SpackRepoException(BashCommandException):
+ """
+ To be thrown when the spack fails to add a repo
+ """
\ No newline at end of file
diff --git a/dedal/logger/logger_config.py b/dedal/logger/logger_config.py
deleted file mode 100644
index 3ca3b000fd171dde8ceafdc6731dfb02009e845c..0000000000000000000000000000000000000000
--- a/dedal/logger/logger_config.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import logging
-
-
-class LoggerConfig:
- """
- This class sets up logging with a file handler
- and a stream handler, ensuring consistent
- and formatted log messages.
- """
- def __init__(self, log_file):
- self.log_file = log_file
- self._configure_logger()
-
- def _configure_logger(self):
- formatter = logging.Formatter(
- fmt='%(asctime)s - %(levelname)s - %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S'
- )
-
- file_handler = logging.FileHandler(self.log_file)
- file_handler.setFormatter(formatter)
-
- stream_handler = logging.StreamHandler()
- stream_handler.setFormatter(formatter)
-
- self.logger = logging.getLogger(__name__)
- self.logger.setLevel(logging.DEBUG)
-
- self.logger.addHandler(file_handler)
- self.logger.addHandler(stream_handler)
-
- def get_logger(self):
- return self.logger
diff --git a/dedal/model/SpackDescriptor.py b/dedal/model/SpackDescriptor.py
index 70e484fb3d39e4333389682d14a32ac46c08a912..421c4824e00a1eb1315fb4204f31cf6245ffa96c 100644
--- a/dedal/model/SpackDescriptor.py
+++ b/dedal/model/SpackDescriptor.py
@@ -9,5 +9,5 @@ class SpackDescriptor:
def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
self.env_name = env_name
- self.path = path
+ self.path = path if isinstance(path,Path) else Path(path)
self.git_path = git_path
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
index ecfbb8e5084fc1e49b32894aecd47190df36fd98..58dcad8bc08ade485da314d9727f95abb0d85514 100644
--- a/dedal/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -3,7 +3,7 @@ import re
import subprocess
from pathlib import Path
from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
- SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
+ SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException, SpackRepoException
from dedal.logger.logger_builder import get_logger
from dedal.configuration.SpackConfig import SpackConfig
from dedal.tests.testing_variables import SPACK_VERSION
@@ -27,15 +27,23 @@ class SpackOperation:
def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
self.spack_config = spack_config
- self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+ self.spack_config.install_dir = spack_config.install_dir
+ os.makedirs(self.spack_config.install_dir, exist_ok=True)
self.spack_dir = self.spack_config.install_dir / 'spack'
- self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
+
+ self.spack_setup_script = "" if self.spack_config.use_spack_global else f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'} &&"
self.logger = logger
+ self.spack_config.concretization_dir = spack_config.concretization_dir
+ if self.spack_config.concretization_dir:
+ os.makedirs(self.spack_config.concretization_dir, exist_ok=True)
+ self.spack_config.buildcache_dir = spack_config.buildcache_dir
+ if self.spack_config.buildcache_dir:
+ os.makedirs(self.spack_config.buildcache_dir, exist_ok=True)
if self.spack_config.env and spack_config.env.path:
- self.spack_config.env.path = spack_config.env.path.resolve()
+ self.spack_config.env.path = spack_config.env.path
self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
- self.env_path = spack_config.env.path / spack_config.env.env_name
- self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
+ self.env_path: Path = spack_config.env.path / spack_config.env.env_name
+ self.spack_command_on_env = f'{self.spack_setup_script} spack env activate -p {self.env_path}'
def create_fetch_spack_environment(self):
if self.spack_config.env.git_path:
@@ -45,7 +53,7 @@ class SpackOperation:
else:
os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+ f'{self.spack_setup_script} spack env create -d {self.env_path}',
check=True, logger=self.logger,
info_msg=f"Created {self.spack_config.env.env_name} spack environment",
exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
@@ -83,7 +91,7 @@ class SpackOperation:
"""Check if the given Spack repository exists."""
if self.spack_config.env is None:
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack repo list',
+ f'{self.spack_setup_script} spack repo list',
check=True,
capture_output=True, text=True, logger=self.logger,
info_msg=f'Checking if {repo_name} exists')
@@ -121,7 +129,7 @@ class SpackOperation:
check=True, logger=self.logger,
info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
- exception=BashCommandException)
+ exception=SpackRepoException)
@check_spack_env
def get_compiler_version(self):
@@ -144,7 +152,7 @@ class SpackOperation:
return gcc_version
def get_spack_installed_version(self):
- spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+ spack_version = run_command("bash", "-c", f'{self.spack_setup_script} spack --version',
capture_output=True, text=True, check=True,
logger=self.logger,
info_msg=f"Getting spack version",
@@ -159,7 +167,7 @@ class SpackOperation:
run_command("bash", "-c",
f'{self.spack_command_on_env} && spack concretize {force}',
check=True,
- logger=self.logger,
+ logger=self.logger,
info_msg=f'Concertization step for {self.spack_config.env.env_name}',
exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
exception=SpackConcertizeException)
@@ -167,7 +175,7 @@ class SpackOperation:
def create_gpg_keys(self):
if self.spack_config.gpg:
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+ f'{self.spack_setup_script} spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
check=True,
logger=self.logger,
info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
@@ -181,7 +189,7 @@ class SpackOperation:
signed = '--signed' if signed else ''
if global_mirror:
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+ f'{self.spack_setup_script} spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
check=True,
logger=self.logger,
info_msg=f'Added mirror {mirror_name}',
@@ -199,7 +207,7 @@ class SpackOperation:
def remove_mirror(self, mirror_name: str):
run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+ f'{self.spack_setup_script} spack mirror rm {mirror_name}',
check=True,
logger=self.logger,
info_msg=f'Removing mirror {mirror_name}',
@@ -212,7 +220,7 @@ class SpackOperation:
fresh = '--fresh' if fresh else ''
debug = '--debug' if debug else ''
install_result = run_command("bash", "-c",
- f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ f'{self.spack_command_on_env} && spack {debug} install -v {signed} -j {jobs} {fresh}',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
@@ -223,7 +231,8 @@ class SpackOperation:
log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
- def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
+ def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack',
+ bashrc_path=os.path.expanduser("~/.bashrc")):
try:
user = os.getlogin()
except OSError:
@@ -241,23 +250,24 @@ class SpackOperation:
else:
self.logger.debug("Spack already cloned.")
- bashrc_path = os.path.expanduser("~/.bashrc")
# ensure the file exists before opening it
if not os.path.exists(bashrc_path):
open(bashrc_path, "w").close()
# add spack setup commands to .bashrc
with open(bashrc_path, "a") as bashrc:
bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
- bashrc.write(f"source {self.spack_setup_script}\n")
+ spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
+ bashrc.write(f"{spack_setup_script}\n")
self.logger.info("Added Spack PATH to .bashrc")
if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
info_msg='Adding permissions to the logged in user')
- run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
self.logger.info("Spack install completed")
- # Restart Bash after the installation ends
- os.system("exec bash")
-
+ if self.spack_config.use_spack_global is True:
+ # Restart the bash only of the spack is used globally
+ self.logger.info('Restarting bash')
+ run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
+ os.system("exec bash")
# Configure upstream Spack instance if specified
if self.spack_config.upstream_instance:
upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
diff --git a/dedal/spack_factory/SpackOperationCreateCache.py b/dedal/spack_factory/SpackOperationCreateCache.py
new file mode 100644
index 0000000000000000000000000000000000000000..f04eae3ad58722aea18d0c0e51a2fb9922093477
--- /dev/null
+++ b/dedal/spack_factory/SpackOperationCreateCache.py
@@ -0,0 +1,51 @@
+import os
+
+from dedal.error_handling.exceptions import NoSpackEnvironmentException
+
+from dedal.utils.utils import copy_to_tmp, copy_file
+from dedal.wrapper.spack_wrapper import check_spack_env
+from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.logger.logger_builder import get_logger
+from dedal.spack_factory.SpackOperation import SpackOperation
+
+
+class SpackOperationCreateCache(SpackOperation):
+ """
+ This class creates caching for the concretization step and for the installation step.
+ """
+
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
+ super().__init__(spack_config, logger=get_logger(__name__))
+ self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=spack_config.cache_version_concretize)
+ self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=spack_config.cache_version_build)
+
+ @check_spack_env
+ def concretize_spack_env(self):
+ super().concretize_spack_env(force=True)
+ dependency_path = self.spack_config.env.path / self.spack_config.env.env_name / 'spack.lock'
+ copy_file(dependency_path, self.spack_config.concretization_dir, logger=self.logger)
+ self.cache_dependency.upload(self.spack_config.concretization_dir)
+ self.logger.info(f'Created new spack concretization for create cache: {self.spack_config.env.env_name}')
+
+ @check_spack_env
+ def install_packages(self, jobs: int = 2, debug=False):
+ signed = False
+ if self.spack_config.gpg:
+ signed = True
+ self.create_gpg_keys()
+ self.add_mirror('local_cache', self.spack_config.buildcache_dir, signed=signed, autopush=signed,
+ global_mirror=False)
+ self.logger.info(f'Added mirror for {self.spack_config.env.env_name}')
+ super().install_packages(jobs=jobs, signed=signed, debug=debug, fresh=True)
+ self.logger.info(f'Installed spack packages for {self.spack_config.env.env_name}')
+ self.build_cache.upload(self.spack_config.buildcache_dir)
+ self.logger.info(f'Pushed spack packages for {self.spack_config.env.env_name}')
diff --git a/dedal/spack_factory/SpackOperationCreator.py b/dedal/spack_factory/SpackOperationCreator.py
index 54517a845bad14629c6019416f0e19581472991e..fdc929d34866dcb254a775e9e8f8a24ee893029d 100644
--- a/dedal/spack_factory/SpackOperationCreator.py
+++ b/dedal/spack_factory/SpackOperationCreator.py
@@ -1,14 +1,19 @@
from dedal.configuration.SpackConfig import SpackConfig
from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
class SpackOperationCreator:
@staticmethod
- def get_spack_operator(spack_config: SpackConfig = None):
+ def get_spack_operator(spack_config: SpackConfig = None, use_cache: bool = False) -> SpackOperation:
if spack_config is None:
- return SpackOperation(SpackConfig())
+ return SpackOperation()
elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
return SpackOperation(spack_config)
- else:
+ elif (spack_config.concretization_dir and spack_config.buildcache_dir) and not use_cache:
+ return SpackOperationCreateCache(spack_config)
+ elif (spack_config.concretization_dir and spack_config.buildcache_dir) and use_cache:
return SpackOperationUseCache(spack_config)
+ else:
+ return SpackOperation(SpackConfig())
diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
index 41a9094ca550f1682082dbee4eced5b8183a9f7a..cb2b3ac82c6e202ade309ec277d2fab76ac89b2d 100644
--- a/dedal/spack_factory/SpackOperationUseCache.py
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -1,8 +1,14 @@
import os
+import subprocess
+from pathlib import Path
+
from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.error_handling.exceptions import SpackInstallPackagesException
from dedal.logger.logger_builder import get_logger
from dedal.spack_factory.SpackOperation import SpackOperation
from dedal.configuration.SpackConfig import SpackConfig
+from dedal.utils.utils import file_exists_and_not_empty, run_command, log_command, copy_to_tmp, copy_file
+from dedal.wrapper.spack_wrapper import check_spack_env
class SpackOperationUseCache(SpackOperation):
@@ -10,23 +16,52 @@ class SpackOperationUseCache(SpackOperation):
This class uses caching for the concretization step and for the installation step.
"""
- def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='v1',
- cache_version_build='v1'):
+ def __init__(self, spack_config: SpackConfig = SpackConfig()):
super().__init__(spack_config, logger=get_logger(__name__))
self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
os.environ.get('CONCRETIZE_OCI_PROJECT'),
os.environ.get('CONCRETIZE_OCI_USERNAME'),
os.environ.get('CONCRETIZE_OCI_PASSWORD'),
- cache_version=cache_version_concretize)
+ cache_version=spack_config.cache_version_concretize)
self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
os.environ.get('BUILDCACHE_OCI_PROJECT'),
os.environ.get('BUILDCACHE_OCI_USERNAME'),
os.environ.get('BUILDCACHE_OCI_PASSWORD'),
- cache_version=cache_version_build)
+ cache_version=spack_config.cache_version_build)
def setup_spack_env(self):
super().setup_spack_env()
# todo add buildcache to the spack environment
- def concretize_spack_env(self, force=True):
- pass
+ @check_spack_env
+ def concretize_spack_env(self):
+ concretization_redo = False
+ self.cache_dependency.download(self.spack_config.concretization_dir)
+ if file_exists_and_not_empty(self.spack_config.concretization_dir / 'spack.lock'):
+ concretization_file_path = self.env_path / 'spack.lock'
+ copy_file(self.spack_config.concretization_dir / 'spack.lock', self.env_path)
+ # redo the concretization step if spack.lock file was not downloaded from the cache
+ if not file_exists_and_not_empty(concretization_file_path):
+ super().concretize_spack_env(force=True)
+ concretization_redo = True
+ else:
+ # redo the concretization step if spack.lock file was not downloaded from the cache
+ super().concretize_spack_env(force=True)
+ concretization_redo = True
+ return concretization_redo
+
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, debug=False):
+ signed = '' if signed else '--no-check-signature'
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack {debug} install -v --reuse {signed} -j {jobs}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
+ exception=SpackInstallPackagesException)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
+ return install_result
diff --git a/dedal/tests/integration_tests/spack_create_cache_test.py b/dedal/tests/integration_tests/spack_create_cache_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcef47a89af9f0ae7bdb53828c5e6682dd3f094e
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_create_cache_test.py
@@ -0,0 +1,58 @@
+from pathlib import Path
+
+import pytest
+
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
+
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
+
+"""
+Before running those tests, the repositories where the caching is stored must be cleared after each run.
+Ebrains Harbour does not support deletion via API, so the clean up must be done manually
+"""
+
+
+@pytest.mark.skip(
+ reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.")
+def test_spack_create_cache_concretization(tmp_path):
+ install_dir = tmp_path
+ concretization_dir = install_dir / 'concretization'
+ buildcache_dir = install_dir / 'buildcache'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ gpg = GpgConfig(gpg_name='test-gpg', gpg_mail='test@test.com')
+ config = SpackConfig(env=env, install_dir=install_dir, concretization_dir=concretization_dir,
+ buildcache_dir=buildcache_dir, gpg=gpg)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperationCreateCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env()
+ assert len(spack_operation.cache_dependency.list_tags()) > 0
+
+
+@pytest.mark.skip(
+ reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.")
+def test_spack_create_cache_installation(tmp_path):
+ install_dir = tmp_path
+ concretization_dir = install_dir / 'concretization'
+ buildcache_dir = install_dir / 'buildcache'
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ gpg = GpgConfig(gpg_name='test-gpg', gpg_mail='test@test.com')
+ config = SpackConfig(env=env, install_dir=install_dir, concretization_dir=concretization_dir,
+ buildcache_dir=buildcache_dir, gpg=gpg)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperationCreateCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env()
+ assert len(spack_operation.cache_dependency.list_tags()) > 0
+ spack_operation.install_packages()
+ assert len(spack_operation.build_cache.list_tags()) > 0
diff --git a/dedal/tests/integration_tests/spack_from_cache_test.py b/dedal/tests/integration_tests/spack_from_cache_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..33f4483314b04e894c0bca3c75f259d58031d4f4
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_from_cache_test.py
@@ -0,0 +1,42 @@
+import pytest
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.utils.utils import file_exists_and_not_empty
+from dedal.utils.variables import test_spack_env_git, ebrains_spack_builds_git
+
+
+def test_spack_from_cache_concretize(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir / 'concretize',
+ buildcache_dir=install_dir / 'buildcache')
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.concretize_spack_env() == False
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+@pytest.mark.skip(reason="Skipping test::test_spack_from_cache_install until all the functionalities in SpackOperationUseCache")
+def test_spack_from_cache_install(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir / 'concretize',
+ buildcache_dir=install_dir / 'buildcache')
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack()
+ spack_operation.setup_spack_env()
+ assert spack_operation.concretize_spack_env() == False
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+ install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False)
+ assert install_result.returncode == 0
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
index d080bc7b144c4530f375b1a265cba41ce936fc3d..794caef115dbe088224f94264ffc690030f7d7f4 100644
--- a/dedal/tests/integration_tests/spack_from_scratch_test.py
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -6,6 +6,7 @@ from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
from dedal.utils.utils import file_exists_and_not_empty
+from dedal.spack_factory.SpackOperation import SpackOperation
def test_spack_repo_exists_1(tmp_path):
@@ -23,8 +24,8 @@ def test_spack_repo_exists_2(tmp_path):
env = SpackDescriptor('ebrains-spack-builds', install_dir)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
- print(spack_operation.get_spack_installed_version())
spack_operation.setup_spack_env()
assert spack_operation.spack_repo_exists(env.env_name) == False
@@ -34,6 +35,7 @@ def test_spack_from_scratch_setup_1(tmp_path):
env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
assert spack_operation.spack_repo_exists(env.env_name) == False
@@ -47,6 +49,7 @@ def test_spack_from_scratch_setup_2(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
assert spack_operation.spack_repo_exists(env.env_name) == True
@@ -60,6 +63,7 @@ def test_spack_from_scratch_setup_3(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
with pytest.raises(BashCommandException):
spack_operation.setup_spack_env()
@@ -70,6 +74,7 @@ def test_spack_from_scratch_setup_4(tmp_path):
env = SpackDescriptor('new_env2', install_dir)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
assert spack_operation.spack_env_exists() == True
@@ -81,6 +86,7 @@ def test_spack_not_a_valid_repo():
config = SpackConfig(env=env, system_name='ebrainslab')
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
with pytest.raises(BashCommandException):
spack_operation.add_spack_repo(repo.path, repo.env_name)
@@ -95,6 +101,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.install_spack()
spack_operation.setup_spack_env()
@@ -113,6 +120,7 @@ def test_spack_from_scratch_concretize_2(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=False)
@@ -128,6 +136,7 @@ def test_spack_from_scratch_concretize_3(tmp_path):
config.add_repo(repo)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
concretization_file_path = spack_operation.env_path / 'spack.lock'
@@ -139,6 +148,7 @@ def test_spack_from_scratch_concretize_4(tmp_path):
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=False)
@@ -151,6 +161,7 @@ def test_spack_from_scratch_concretize_5(tmp_path):
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
config = SpackConfig(env=env, install_dir=install_dir)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=True)
@@ -165,6 +176,7 @@ def test_spack_from_scratch_concretize_6(tmp_path):
config = SpackConfig(env=env, install_dir=install_dir)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=False)
@@ -179,6 +191,7 @@ def test_spack_from_scratch_concretize_7(tmp_path):
config = SpackConfig(env=env)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=True)
@@ -193,6 +206,7 @@ def test_spack_from_scratch_install(tmp_path):
config = SpackConfig(env=env)
config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
spack_operation.install_spack()
spack_operation.setup_spack_env()
spack_operation.concretize_spack_env(force=True)
diff --git a/dedal/tests/integration_tests/spack_install_test.py b/dedal/tests/integration_tests/spack_install_test.py
index 564d5c6aa2138e815cd7d092215a4f2eee8816f6..0c6cf1273cf57b5ccb82e4c4cd79fc913aebeaaf 100644
--- a/dedal/tests/integration_tests/spack_install_test.py
+++ b/dedal/tests/integration_tests/spack_install_test.py
@@ -1,12 +1,12 @@
-import pytest
+from dedal.configuration.SpackConfig import SpackConfig
from dedal.spack_factory.SpackOperation import SpackOperation
from dedal.tests.testing_variables import SPACK_VERSION
-# run this test first so that spack is installed only once for all the tests
-@pytest.mark.run(order=1)
-def test_spack_install_scratch():
- spack_operation = SpackOperation()
+def test_spack_install_scratch(tmp_path):
+ install_dir = tmp_path
+ spack_config = SpackConfig(install_dir=install_dir)
+ spack_operation = SpackOperation(spack_config)
spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
installed_spack_version = spack_operation.get_spack_installed_version()
assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/integration_tests/spack_operation_creator_test.py b/dedal/tests/integration_tests/spack_operation_creator_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..226184b00a5c7136c97f5ef12761ac44c71286a1
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_operation_creator_test.py
@@ -0,0 +1,50 @@
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
+
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.tests.testing_variables import ebrains_spack_builds_git, test_spack_env_git
+
+
+def test_spack_creator_scratch_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperation)
+
+
+def test_spack_creator_scratch_2(tmp_path):
+ spack_config = None
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperation)
+
+
+def test_spack_creator_scratch_3():
+ spack_config = SpackConfig()
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperation)
+
+
+def test_spack_creator_create_cache(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir, buildcache_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config)
+ assert isinstance(spack_operation, SpackOperationCreateCache)
+
+
+def test_spack_creator_use_cache(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir, buildcache_dir=install_dir)
+ spack_config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+ assert isinstance(spack_operation, SpackOperationUseCache)
diff --git a/dedal/tests/unit_tests/spack_manager_api_test.py b/dedal/tests/unit_tests/spack_manager_api_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..4184a864c08019ae6ec6e52fdbab4849b4c6211c
--- /dev/null
+++ b/dedal/tests/unit_tests/spack_manager_api_test.py
@@ -0,0 +1,181 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from click.testing import CliRunner
+from dedal.cli.spack_manager_api import show_config, clear_config, install_spack, add_spack_repo, install_packages, \
+ setup_spack_env, concretize, set_config
+from dedal.model.SpackDescriptor import SpackDescriptor
+
+
+@pytest.fixture
+def runner():
+ return CliRunner()
+
+
+@pytest.fixture
+def mocked_session_path():
+ return '/mocked/tmp/session.json'
+
+
+@pytest.fixture
+def mock_spack_manager():
+ mock_spack_manager = MagicMock()
+ mock_spack_manager.install_spack = MagicMock()
+ mock_spack_manager.add_spack_repo = MagicMock()
+ mock_spack_manager.setup_spack_env = MagicMock()
+ mock_spack_manager.concretize_spack_env = MagicMock()
+ mock_spack_manager.install_packages = MagicMock()
+ return mock_spack_manager
+
+
+@pytest.fixture
+def mock_load_config():
+ with patch('dedal.cli.spack_manager_api.load_config') as mock_load:
+ yield mock_load
+
+
+@pytest.fixture
+def mock_save_config():
+ with patch('dedal.cli.spack_manager_api.save_config') as mock_save:
+ yield mock_save
+
+
+@pytest.fixture
+def mock_clear_config():
+ with patch('dedal.cli.spack_manager_api.clear_config') as mock_clear:
+ yield mock_clear
+
+
+def test_show_config_no_config(runner, mock_load_config):
+ mock_load_config.return_value = None
+ result = runner.invoke(show_config)
+ assert 'No configuration set. Use `set-config` first.' in result.output
+
+
+def test_show_config_with_config(runner, mock_load_config):
+ """Test the show_config command when config is present."""
+ mock_load_config.return_value = {"key": "value"}
+ result = runner.invoke(show_config)
+ assert result.exit_code == 0
+ assert '"key": "value"' in result.output
+
+
+def test_clear_config(runner, mock_clear_config):
+ """Test the clear_config command."""
+ with patch('os.path.exists', return_value=True), patch('os.remove') as mock_remove:
+ result = runner.invoke(clear_config)
+ assert 'Configuration cleared!' in result.output
+ mock_remove.assert_called_once()
+
+
+def test_install_spack_no_context_1(runner, mock_spack_manager):
+ """Test install_spack with no context, using SpackManager."""
+ with patch('dedal.cli.spack_manager_api.SpackManager', return_value=mock_spack_manager):
+ result = runner.invoke(install_spack, ['--spack_version', '0.24.0'])
+ mock_spack_manager.install_spack.assert_called_once_with('0.24.0', '~/.bashrc')
+ assert result.exit_code == 0
+
+
+def test_install_spack_no_context_2(runner, mock_spack_manager):
+ """Test install_spack with no context, using SpackManager and the default value for spack_version."""
+ with patch('dedal.cli.spack_manager_api.SpackManager', return_value=mock_spack_manager):
+ result = runner.invoke(install_spack)
+ mock_spack_manager.install_spack.assert_called_once_with('0.23.0', '~/.bashrc')
+ assert result.exit_code == 0
+
+
+def test_install_spack_with_mocked_context_1(runner, mock_spack_manager):
+ """Test install_spack with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(install_spack, ['--spack_version', '0.24.0', '--bashrc_path', '/home/.bahsrc'], obj=mock_spack_manager)
+ mock_spack_manager.install_spack.assert_called_once_with('0.24.0', '/home/.bahsrc')
+ assert result.exit_code == 0
+
+
+def test_install_spack_with_mocked_context_2(runner, mock_spack_manager):
+ """Test install_spack with a mocked context, using ctx.obj as SpackManager and the default value for spack_version."""
+ result = runner.invoke(install_spack, obj=mock_spack_manager)
+ mock_spack_manager.install_spack.assert_called_once_with('0.23.0', '~/.bashrc')
+ assert result.exit_code == 0
+
+
+def test_setup_spack_env(runner, mock_spack_manager):
+ """Test setup_spack_env with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(setup_spack_env, obj=mock_spack_manager)
+ mock_spack_manager.setup_spack_env.assert_called_once_with()
+ assert result.exit_code == 0
+
+
+def test_concretize(runner, mock_spack_manager):
+ """Test install_spack with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(concretize, obj=mock_spack_manager)
+ mock_spack_manager.concretize_spack_env.assert_called_once_with()
+ assert result.exit_code == 0
+
+
+def test_install_packages_1(runner, mock_spack_manager):
+ """Test install_packages with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(install_packages, obj=mock_spack_manager)
+ mock_spack_manager.install_packages.assert_called_once_with(jobs=2)
+ assert result.exit_code == 0
+
+
+def test_install_packages(runner, mock_spack_manager):
+ """Test install_packages with a mocked context, using ctx.obj as SpackManager."""
+ result = runner.invoke(install_packages, ['--jobs', 3], obj=mock_spack_manager)
+ mock_spack_manager.install_packages.assert_called_once_with(jobs=3)
+ assert result.exit_code == 0
+
+
+@patch('dedal.cli.spack_manager_api.resolve_path')
+@patch('dedal.cli.spack_manager_api.SpackDescriptor')
+def test_add_spack_repo(mock_spack_descriptor, mock_resolve_path, mock_load_config, mock_save_config,
+ mocked_session_path, runner):
+ """Test adding a spack repository with mocks."""
+ expected_config = {'repos': [SpackDescriptor(env_name='test-repo')]}
+ repo_name = 'test-repo'
+ path = '/path'
+ git_path = 'https://example.com/repo.git'
+ mock_resolve_path.return_value = '/resolved/path'
+ mock_load_config.return_value = expected_config
+ mock_repo_instance = MagicMock()
+ mock_spack_descriptor.return_value = mock_repo_instance
+
+ with patch('dedal.cli.spack_manager_api.SESSION_CONFIG_PATH', mocked_session_path):
+ result = runner.invoke(add_spack_repo, ['--repo_name', repo_name, '--path', path, '--git_path', git_path])
+
+ assert result.exit_code == 0
+ assert 'dedal setup_spack_env must be reran after each repo is added' in result.output
+ mock_resolve_path.assert_called_once_with(path)
+ mock_spack_descriptor.assert_called_once_with(repo_name, '/resolved/path', git_path)
+ assert mock_repo_instance in expected_config['repos']
+ mock_save_config.assert_called_once_with(expected_config, mocked_session_path)
+
+
+def test_set_config(runner, mock_save_config, mocked_session_path):
+ """Test set_config."""
+ with patch('dedal.cli.spack_manager_api.SESSION_CONFIG_PATH', mocked_session_path):
+ result = runner.invoke(set_config, ['--env_name', 'test', '--system_name', 'sys'])
+
+ expected_config = {
+ 'use_cache': False,
+ 'env_name': 'test',
+ 'env_path': None,
+ 'env_git_path': None,
+ 'install_dir': None,
+ 'upstream_instance': None,
+ 'system_name': 'sys',
+ 'concretization_dir': None,
+ 'buildcache_dir': None,
+ 'gpg_name': None,
+ 'gpg_mail': None,
+ 'use_spack_global': False,
+ 'repos': [],
+ 'cache_version_concretize': 'v1',
+ 'cache_version_build': 'v1',
+ }
+
+ mock_save_config.assert_called_once()
+ saved_config, saved_path = mock_save_config.call_args[0]
+ assert saved_path == mocked_session_path
+ assert saved_config == expected_config
+ assert result.exit_code == 0
+ assert 'Configuration saved.' in result.output
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 9fc82ad520b4a22715c819d3ca11042b88b87c7f..2f7c48479fb3c0825b06f80e3cc2c70046e4affc 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -2,6 +2,7 @@ import logging
import os
import shutil
import subprocess
+import tempfile
from pathlib import Path
from dedal.error_handling.exceptions import BashCommandException
@@ -78,6 +79,21 @@ def log_command(results, log_file: str):
log_file.write(results.stderr)
+def copy_to_tmp(file_path: Path) -> Path:
+ """
+ Creates a temporary directory and copies the given file into it.
+
+ :param file_path: Path to the file that needs to be copied.
+ :return: Path to the copied file inside the temporary directory.
+ """
+ if not file_path.is_file():
+ raise FileNotFoundError(f"File not found: {file_path}")
+ tmp_dir = Path(tempfile.mkdtemp())
+ tmp_file_path = tmp_dir / file_path.name
+ shutil.copy(file_path, tmp_file_path)
+ return tmp_file_path
+
+
def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"),
logger: logging = logging.getLogger(__name__)):
"""Update or add an environment variable in ~/.bashrc."""
@@ -98,3 +114,43 @@ def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.ex
logger.info(f"Updated {bashrc_path} with: export {var_name}={value}")
with open(bashrc_path, "w") as file:
file.writelines(lines)
+
+
+def copy_file(src: Path, dst: Path, logger: logging = logging.getLogger(__name__)) -> None:
+ """
+ Copy a file from src to dest.
+ """
+ if not os.path.exists(src):
+ raise FileNotFoundError(f"Source file '{src}' does not exist.")
+ src.resolve().as_posix()
+ dst.resolve().as_posix()
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ shutil.copy2(src, dst)
+ logger.debug(f"File copied from '{src}' to '{dst}'")
+
+
+def delete_file(file_path: str, logger: logging = logging.getLogger(__name__)) -> bool:
+ """
+ Deletes a file at the given path. Returns True if successful, False if the file doesn't exist.
+ """
+ try:
+ os.remove(file_path)
+ logger.debug(f"File '{file_path}' deleted.")
+ return True
+ except FileNotFoundError:
+ logger.error(f"File not found: {file_path}")
+ return False
+ except PermissionError:
+ logger.error(f"Permission denied: {file_path}")
+ return False
+ except Exception as e:
+ logger.error(f"Error deleting file {file_path}: {e}")
+ return False
+
+
+def resolve_path(path: str):
+ if path is None:
+ path = Path(os.getcwd()).resolve()
+ else:
+ path = Path(path).resolve()
+ return path
diff --git a/dedal/utils/variables.py b/dedal/utils/variables.py
new file mode 100644
index 0000000000000000000000000000000000000000..553ccf97992ca2dcd06970f82fdbbb26f5f1db23
--- /dev/null
+++ b/dedal/utils/variables.py
@@ -0,0 +1,5 @@
+import os
+
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
diff --git a/pyproject.toml b/pyproject.toml
index b6679ca1f6946fde61d1ec5d288a8b85ec92d7a3..aad18fa55c29b67f30ba7464f2b180a8256d479d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,7 +19,12 @@ dependencies = [
"pytest",
"pytest-mock",
"pytest-ordering",
+ "click",
+ "jsonpickle",
]
+[project.scripts]
+dedal = "dedal.cli.spack_manager_api:cli"
+
[tool.setuptools.data-files]
"dedal" = ["dedal/logger/logging.conf"]
\ No newline at end of file