.env

@@ -2,8 +2,7 @@ BUILDCACHE_OCI_HOST=""
 BUILDCACHE_OCI_PASSWORD=""
 BUILDCACHE_OCI_PROJECT=""
 BUILDCACHE_OCI_USERNAME=""
-
 CONCRETIZE_OCI_HOST=""
 CONCRETIZE_OCI_PASSWORD=""
 CONCRETIZE_OCI_PROJECT=""
-CONCRETIZE_OCI_USERNAME""
+CONCRETIZE_OCI_USERNAME=""

.gitlab-ci.yml

@@ -39,5 +39,6 @@ testing-pytest:
     paths:
       - test-results.xml
       - .dedal.log
+      - .generate_cache.log
     expire_in: 1 week
 

README.md

 # Dedal
 
-This repository provides functionalities to easily ```managed spack environments``` and ```helpers for the container image build flow```.
-
-This library runs only on different kinds of linux distribution operating systems.
-
-The lowest ```spack version``` compatible with this library is ```v0.23.0```.
-
-
-        This repository also provied CLI interface. For more informations, after installing this library, call dedal --help.
-
+This repository provides functionalities to easily ```managed spack environments``` and
+```helpers for the container image build flow```.
 
 **Setting up the needed environment variables**
-    The ````<checkout path>\dedal\.env```` file contains the environment variables required for OCI registry used for caching.
-    Ensure that you edit the ````<checkout path>\dedal\.env```` file to match your environment.
-    The following provides an explanation of the various environment variables:
-
+The ````<checkout path>\dedal\.env```` file contains the environment variables required for OCI registry used for
+caching.
+Ensure that you edit the ````<checkout path>\dedal\.env```` file to match your environment.
+The following provides an explanation of the various environment variables:
 
        # OCI Registry Configuration Sample for concretization caches
        # =============================
@@ -49,5 +42,101 @@ The lowest ```spack version``` compatible with this library is ```v0.23.0```.
        # The password used for authentication with the Docker registry.
        BUILDCACHE_OCI_HOST="###ACCESS_TOKEN###"
 
-For both concretization and binary caches, the cache version can be changed via the attributes ```cache_version_concretize``` and ```cache_version_build```. 
+For both concretization and binary caches, the cache version can be changed via the attributes
+```cache_version_concretize``` and ```cache_version_build```.
 The default values are ```v1```.
+
+Before using this library, the following tool must be installed on Linux distribution:
+
+````
+    apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+````
+
+````
+    python3 -m pip install --upgrade pip setuptools wheel
+````
+
+# Dedal library installation
+
+```sh
+  pip install dedal
+```
+
+# Dedal CLI Commands
+
+The following commands are available in this CLI tool. You can view detailed explanations by using the `--help` option
+with any command.
+
+### 1. `dedal install-spack`
+
+Install spack in the install_dir folder.
+
+**Options:**
+
+- `--spack_version <TEXT>` : Specifies the Spack version to be installed (default: v0.23.0).
+- `--bashrc_path <TEXT>` : Defines the path to .bashrc.
+
+### 2. `dedal set-config`
+
+Sets configuration parameters for the session.
+
+**Options:**
+
+- `--use_cache`                     Enables cashing
+- `--use_spack_global`              Uses spack installed globally on the os
+- `--env_name <TEXT>`                 Environment name
+- `--env_path <TEXT>`                 Environment path to download locally
+- `--env_git_path <TEXT>`             Git path to download the environment
+- `--install_dir <TEXT>`              Install directory for installing spack;
+  spack environments and repositories are
+  stored here
+- `--upstream_instance <TEXT>`        Upstream instance for spack environment
+- `--system_name <TEXT>`              System name; it is used inside the spack
+  environment
+- `--concretization_dir <TEXT>`       Directory where the concretization caching
+  (spack.lock) will be downloaded
+- `--buildcache_dir <TEXT>`           Directory where the binary caching is
+  downloaded for the spack packages
+- `--gpg_name <TEXT>`                 Gpg name
+- `--gpg_mail <TEXT>`                 Gpg mail contact address
+- `--cache_version_concretize <TEXT>`
+  Cache version for concretizaion data
+- `--cache_version_build <TEXT>`      Cache version for binary caches data
+
+### 3. `dedal show-config`
+
+Show the current configuration.
+
+### 4. `dedal clear-config`
+
+Clears stored configuration
+
+### 5. `dedal add-spack-repo`
+
+Adds a spack repository to the spack environments.
+
+**Options:**
+
+- `--repo_name <TEXT>`  Repository name  [required]
+- `--path <TEXT>`       Repository path to download locally  [required]
+- `--git_path <TEXT>`   Git path to download the repository  [required]
+
+### 6. `dedal setup-spack-env`
+
+Setups a spack environment according to the given configuration.
+
+### 7. `dedal concretize`
+
+Spack concretization step.
+
+### 9. `dedal install-packages`
+
+Installs spack packages present in the spack environment defined in configuration.
+
+**Options:**
+
+- `--jobs <INTEGER>`  Number of parallel jobs for spack installation
+
+# Dedal's UML diagram
+
+![screenshot](dedal/docs/resources/dedal_UML.png)
\ No newline at end of file

dedal/bll/SpackManager.py

+import os
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.configuration.SpackConfig import SpackConfig
+
+
+class SpackManager:
+    """
+    This class defines the logic used by the CLI
+    """
+
+    def __init__(self, spack_config: SpackConfig = None, use_cache=False):
+        self._spack_config = spack_config
+        self._use_cache = use_cache
+
+    def _get_spack_operation(self):
+        return SpackOperationCreator.get_spack_operator(self._spack_config, self._use_cache)
+
+    def install_spack(self, version: str, bashrc_path=os.path.expanduser("~/.bashrc")):
+        self._get_spack_operation().install_spack(spack_version=f'v{version}', bashrc_path=bashrc_path)
+
+    def add_spack_repo(self, repo: SpackDescriptor):
+        """
+        After additional repo was added, setup_spack_env must be invoked
+        """
+        self._spack_config.add_repo(repo)
+
+    def setup_spack_env(self):
+        self._get_spack_operation().setup_spack_env()
+
+    def concretize_spack_env(self):
+        self._get_spack_operation().concretize_spack_env()
+
+    def install_packages(self, jobs: int):
+        self._get_spack_operation().install_packages(jobs=jobs)

dedal/bll/cli_utils.py

+import jsonpickle
+import os
+
+
+def save_config(spack_config_data, config_path: str):
+    """Save config to JSON file."""
+    with open(config_path, "w") as data_file:
+        data_file.write(jsonpickle.encode(spack_config_data))
+
+
+def load_config(config_path: str):
+    """Load config from JSON file."""
+    if os.path.exists(config_path):
+        with open(config_path, "r") as data_file:
+            data = jsonpickle.decode(data_file.read())
+            return data
+    return {}
+
+
+def clear_config(config_path: str):
+    """Delete the JSON config file."""
+    if os.path.exists(config_path):
+        os.remove(config_path)

dedal/cli/spack_manager_api.py

+import os
+from pathlib import Path
+import click
+import jsonpickle
+
+from dedal.bll.SpackManager import SpackManager
+from dedal.bll.cli_utils import save_config, load_config
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.utils.utils import resolve_path
+
+SESSION_CONFIG_PATH = os.path.expanduser(f'~/tmp/dedal/dedal_session.json')
+os.makedirs(os.path.dirname(SESSION_CONFIG_PATH), exist_ok=True)
+
+
+@click.group()
+@click.pass_context
+def cli(ctx: click.Context):
+    config = load_config(SESSION_CONFIG_PATH)
+    if ctx.invoked_subcommand not in ['set-config', 'install-spack'] and not config:
+        click.echo('No configuration set. Use `set-config` first.')
+        ctx.exit(1)
+    if config:
+        config['env_path'] = resolve_path(config['env_path'])
+        env = SpackDescriptor(config['env_name'], config['env_path'], config['env_git_path'])
+        gpg = GpgConfig(config['gpg_name'], config['gpg_mail']) if config['gpg_name'] and config['gpg_mail'] else None
+        spack_config = SpackConfig(env=env, repos=None, install_dir=config['install_dir'],
+                                   upstream_instance=config['upstream_instance'],
+                                   concretization_dir=config['concretization_dir'],
+                                   buildcache_dir=config['buildcache_dir'],
+                                   system_name=config['system_name'], gpg=gpg,
+                                   use_spack_global=config['use_spack_global'])
+        ctx.obj = SpackManager(spack_config, use_cache=config['use_cache'])
+
+
+@cli.command()
+@click.option('--use_cache', is_flag=True, default=False, help='Enables cashing')
+@click.option('--use_spack_global', is_flag=True, default=False, help='Uses spack installed globally on the os')
+@click.option('--env_name', type=str, default=None, help='Environment name')
+@click.option('--env_path', type=str, default=None, help='Environment path to download locally')
+@click.option('--env_git_path', type=str, default=None, help='Git path to download the environment')
+@click.option('--install_dir', type=str,
+              help='Install directory for installing spack; spack environments and repositories are stored here')
+@click.option('--upstream_instance', type=str, default=None, help='Upstream instance for spack environment')
+@click.option('--system_name', type=str, default=None, help='System name; it is used inside the spack environment')
+@click.option('--concretization_dir', type=str, default=None,
+              help='Directory where the concretization caching (spack.lock) will be downloaded')
+@click.option('--buildcache_dir', type=str, default=None,
+              help='Directory where the binary caching is downloaded for the spack packages')
+@click.option('--gpg_name', type=str, default=None, help='Gpg name')
+@click.option('--gpg_mail', type=str, default=None, help='Gpg mail contact address')
+@click.option('--cache_version_concretize', type=str, default='v1', help='Cache version for concretizaion data')
+@click.option('--cache_version_build', type=str, default='v1', help='Cache version for binary caches data')
+def set_config(use_cache, env_name, env_path, env_git_path, install_dir, upstream_instance, system_name,
+               concretization_dir,
+               buildcache_dir, gpg_name, gpg_mail, use_spack_global, cache_version_concretize, cache_version_build):
+    """Sets configuration parameters for the session."""
+    spack_config_data = {
+        'use_cache': use_cache,
+        'env_name': env_name,
+        'env_path': env_path,
+        'env_git_path': env_git_path,
+        'install_dir': install_dir,
+        'upstream_instance': upstream_instance,
+        'system_name': system_name,
+        'concretization_dir': Path(concretization_dir) if concretization_dir else None,
+        'buildcache_dir': Path(buildcache_dir) if buildcache_dir else None,
+        'gpg_name': gpg_name,
+        'gpg_mail': gpg_mail,
+        'use_spack_global': use_spack_global,
+        'repos': [],
+        'cache_version_concretize': cache_version_concretize,
+        'cache_version_build': cache_version_build,
+    }
+    save_config(spack_config_data, SESSION_CONFIG_PATH)
+    click.echo('Configuration saved.')
+
+
+@click.command()
+def show_config():
+    """Show the current configuration."""
+    config = load_config(SESSION_CONFIG_PATH)
+    if config:
+        click.echo(jsonpickle.encode(config, indent=2))
+    else:
+        click.echo('No configuration set. Use `set-config` first.')
+
+
+@cli.command()
+@click.option('--spack_version', type=str, default='0.23.0', help='Specifies the Spack version to be installed (default: v0.23.0).')
+@click.option('--bashrc_path', type=str, default="~/.bashrc", help='Defines the path to .bashrc.')
+@click.pass_context
+def install_spack(ctx: click.Context, spack_version: str, bashrc_path: str):
+    """Install spack in the install_dir folder"""
+    bashrc_path = os.path.expanduser(bashrc_path)
+    if ctx.obj is None:
+        SpackManager().install_spack(spack_version, bashrc_path)
+    else:
+        ctx.obj.install_spack(spack_version, bashrc_path)
+
+
+@cli.command()
+@click.option('--repo_name', type=str, required=True, default=None, help='Repository name')
+@click.option('--path', type=str, required=True, default=None, help='Repository path to download locally')
+@click.option('--git_path', type=str, required=True, default=None, help='Git path to download the repository')
+def add_spack_repo(repo_name: str, path: str, git_path: str = None):
+    """Adds a spack repository to the spack environments. The setup command must be rerun."""
+    path = resolve_path(path)
+    repo = SpackDescriptor(repo_name, path, git_path)
+    config = load_config(SESSION_CONFIG_PATH)
+    config['repos'].append(repo)
+    save_config(config, SESSION_CONFIG_PATH)
+    click.echo('dedal setup_spack_env must be reran after each repo is added for the environment.')
+
+
+@cli.command()
+@click.pass_context
+def setup_spack_env(ctx: click.Context):
+    """Setups a spack environment according to the given configuration."""
+    ctx.obj.setup_spack_env()
+
+
+@cli.command()
+@click.pass_context
+def concretize(ctx: click.Context):
+    """Spack concretization step."""
+    ctx.obj.concretize_spack_env()
+
+
+@cli.command()
+@click.option('--jobs', type=int, default=2, help='Number of parallel jobs for spack installation')
+@click.pass_context
+def install_packages(ctx: click.Context, jobs):
+    """Installs spack packages present in the spack environment defined in configuration."""
+    ctx.obj.install_packages(jobs=jobs)
+
+
+@click.command()
+def clear_config():
+    """Clears stored configuration."""
+    if os.path.exists(SESSION_CONFIG_PATH):
+        os.remove(SESSION_CONFIG_PATH)
+        click.echo('Configuration cleared!')
+    else:
+        click.echo('No configuration to clear.')
+
+
+cli.add_command(show_config)
+cli.add_command(clear_config)
+
+if __name__ == '__main__':
+    cli()

dedal/configuration/SpackConfig.py

 import os
 from pathlib import Path
-
 from dedal.configuration.GpgConfig import GpgConfig
 from dedal.model import SpackDescriptor
+from dedal.utils.utils import resolve_path
 
 
 class SpackConfig:
     def __init__(self, env: SpackDescriptor = None, repos: list[SpackDescriptor] = None,
                  install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
-                 concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None):
+                 concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None,
+                 use_spack_global=False, cache_version_concretize='v1',
+                 cache_version_build='v1'):
         self.env = env
         if repos is None:
             self.repos = []
         else:
             self.repos = repos
-        self.install_dir = install_dir
-        if self.install_dir:
-            os.makedirs(self.install_dir, exist_ok=True)
         self.upstream_instance = upstream_instance
         self.system_name = system_name
-        self.concretization_dir = concretization_dir
-        if self.concretization_dir:
-            os.makedirs(self.concretization_dir, exist_ok=True)
-        self.buildcache_dir = buildcache_dir
-        if self.buildcache_dir:
-            os.makedirs(self.buildcache_dir, exist_ok=True)
+        self.concretization_dir = concretization_dir if concretization_dir is None else resolve_path(concretization_dir)
+        self.buildcache_dir = buildcache_dir if buildcache_dir is None else resolve_path(buildcache_dir)
+        self.install_dir = resolve_path(install_dir)
         self.gpg = gpg
+        self.use_spack_global = use_spack_global
+        self.cache_version_concretize = cache_version_concretize
+        self.cache_version_build = cache_version_build
 
     def add_repo(self, repo: SpackDescriptor):
         if self.repos is None:

dedal/error_handling/exceptions.py

@@ -39,3 +39,8 @@ class SpackGpgException(BashCommandException):
     """
     To be thrown when the spack fails to create gpg keys
     """
+
+class SpackRepoException(BashCommandException):
+    """
+    To be thrown when the spack fails to add a repo
+    """
\ No newline at end of file

dedal/logger/logger_config.py

-import logging
-
-
-class LoggerConfig:
-    """
-        This class sets up logging with a file handler
-        and a stream handler, ensuring consistent
-        and formatted log messages.
-    """
-    def __init__(self, log_file):
-        self.log_file = log_file
-        self._configure_logger()
-
-    def _configure_logger(self):
-        formatter = logging.Formatter(
-            fmt='%(asctime)s - %(levelname)s - %(message)s',
-            datefmt='%Y-%m-%d %H:%M:%S'
-        )
-
-        file_handler = logging.FileHandler(self.log_file)
-        file_handler.setFormatter(formatter)
-
-        stream_handler = logging.StreamHandler()
-        stream_handler.setFormatter(formatter)
-
-        self.logger = logging.getLogger(__name__)
-        self.logger.setLevel(logging.DEBUG)
-
-        self.logger.addHandler(file_handler)
-        self.logger.addHandler(stream_handler)
-
-    def get_logger(self):
-        return self.logger

dedal/model/SpackDescriptor.py

@@ -7,7 +7,7 @@ class SpackDescriptor:
     Provides details about the spack environment
     """
 
-    def __init__(self, env_name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
-        self.env_name = env_name
-        self.path = path
+    def __init__(self, name: str, path: Path = Path(os.getcwd()).resolve(), git_path: str = None):
+        self.name = name
+        self.path = path.resolve() if isinstance(path, Path) else Path(path).resolve()
         self.git_path = git_path

dedal/spack_factory/SpackOperation.py

@@ -3,7 +3,7 @@ import re
 import subprocess
 from pathlib import Path
 from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
-    SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException
+    SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException, SpackRepoException
 from dedal.logger.logger_builder import get_logger
 from dedal.configuration.SpackConfig import SpackConfig
 from dedal.tests.testing_variables import SPACK_VERSION
@@ -27,28 +27,39 @@ class SpackOperation:
 
     def __init__(self, spack_config: SpackConfig = SpackConfig(), logger=get_logger(__name__)):
         self.spack_config = spack_config
-        self.spack_config.install_dir.mkdir(parents=True, exist_ok=True)
+        self.spack_config.install_dir = spack_config.install_dir
+        os.makedirs(self.spack_config.install_dir, exist_ok=True)
         self.spack_dir = self.spack_config.install_dir / 'spack'
-        self.spack_setup_script = self.spack_dir / 'share' / 'spack' / 'setup-env.sh'
+
+        self.spack_setup_script = "" if self.spack_config.use_spack_global else f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'} &&"
         self.logger = logger
+        self.spack_config.concretization_dir = spack_config.concretization_dir
+        if self.spack_config.concretization_dir:
+            os.makedirs(self.spack_config.concretization_dir, exist_ok=True)
+        self.spack_config.buildcache_dir = spack_config.buildcache_dir
+        if self.spack_config.buildcache_dir:
+            os.makedirs(self.spack_config.buildcache_dir, exist_ok=True)
+        if self.spack_config.env and spack_config.env.name:
+            self.env_path: Path = spack_config.env.path / spack_config.env.name
+            self.spack_command_on_env = f'{self.spack_setup_script} spack env activate -p {self.env_path}'
+        else:
+            self.spack_command_on_env = self.spack_setup_script
         if self.spack_config.env and spack_config.env.path:
-            self.spack_config.env.path = spack_config.env.path.resolve()
+            self.spack_config.env.path = spack_config.env.path
             self.spack_config.env.path.mkdir(parents=True, exist_ok=True)
-            self.env_path = spack_config.env.path / spack_config.env.env_name
-            self.spack_command_on_env = f'source {self.spack_setup_script} && spack env activate -p {self.env_path}'
 
     def create_fetch_spack_environment(self):
         if self.spack_config.env.git_path:
-            git_clone_repo(self.spack_config.env.env_name, self.spack_config.env.path / self.spack_config.env.env_name,
+            git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name,
                            self.spack_config.env.git_path,
                            logger=self.logger)
         else:
-            os.makedirs(self.spack_config.env.path / self.spack_config.env.env_name, exist_ok=True)
+            os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True)
             run_command("bash", "-c",
-                        f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+                        f'{self.spack_setup_script} spack env create -d {self.env_path}',
                         check=True, logger=self.logger,
-                        info_msg=f"Created {self.spack_config.env.env_name} spack environment",
-                        exception_msg=f"Failed to create {self.spack_config.env.env_name} spack environment",
+                        info_msg=f"Created {self.spack_config.env.name} spack environment",
+                        exception_msg=f"Failed to create {self.spack_config.env.name} spack environment",
                         exception=BashCommandException)
 
     def setup_spack_env(self):
@@ -71,19 +82,19 @@ class SpackOperation:
         self.create_fetch_spack_environment()
         if self.spack_config.install_dir.exists():
             for repo in self.spack_config.repos:
-                repo_dir = self.spack_config.install_dir / repo.path / repo.env_name
-                git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
-                if not self.spack_repo_exists(repo.env_name):
-                    self.add_spack_repo(repo.path, repo.env_name)
-                    self.logger.debug(f'Added spack repository {repo.env_name}')
+                repo_dir = self.spack_config.install_dir / repo.path / repo.name
+                git_clone_repo(repo.name, repo_dir, repo.git_path, logger=self.logger)
+                if not self.spack_repo_exists(repo.name):
+                    self.add_spack_repo(repo.path, repo.name)
+                    self.logger.debug(f'Added spack repository {repo.name}')
                 else:
-                    self.logger.debug(f'Spack repository {repo.env_name} already added')
+                    self.logger.debug(f'Spack repository {repo.name} already added')
 
     def spack_repo_exists(self, repo_name: str) -> bool | None:
         """Check if the given Spack repository exists."""
         if self.spack_config.env is None:
             result = run_command("bash", "-c",
-                                 f'source {self.spack_setup_script} && spack repo list',
+                                 f'{self.spack_setup_script} spack repo list',
                                  check=True,
                                  capture_output=True, text=True, logger=self.logger,
                                  info_msg=f'Checking if {repo_name} exists')
@@ -108,7 +119,7 @@ class SpackOperation:
                              self.spack_command_on_env,
                              check=True,
                              capture_output=True, text=True, logger=self.logger,
-                             info_msg=f'Checking if environment {self.spack_config.env.env_name} exists')
+                             info_msg=f'Checking if environment {self.spack_config.env.name} exists')
         if result is None:
             return False
         return True
@@ -119,9 +130,9 @@ class SpackOperation:
         run_command("bash", "-c",
                     f'{self.spack_command_on_env} && spack repo add {repo_path}/{repo_name}',
                     check=True, logger=self.logger,
-                    info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.env_name}",
-                    exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.env_name}",
-                    exception=BashCommandException)
+                    info_msg=f"Added {repo_name} to spack environment {self.spack_config.env.name}",
+                    exception_msg=f"Failed to add {repo_name} to spack environment {self.spack_config.env.name}",
+                    exception=SpackRepoException)
 
     @check_spack_env
     def get_compiler_version(self):
@@ -129,22 +140,22 @@ class SpackOperation:
                              f'{self.spack_command_on_env} && spack compiler list',
                              check=True, logger=self.logger,
                              capture_output=True, text=True,
-                             info_msg=f"Checking spack environment compiler version for {self.spack_config.env.env_name}",
-                             exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.env_name}",
+                             info_msg=f"Checking spack environment compiler version for {self.spack_config.env.name}",
+                             exception_msg=f"Failed to checking spack environment compiler version for {self.spack_config.env.name}",
                              exception=BashCommandException)
         # todo add error handling and tests
         if result.stdout is None:
-            self.logger.debug('No gcc found for {self.env.env_name}')
+            self.logger.debug(f'No gcc found for {self.spack_config.env.name}')
             return None
 
         # Find the first occurrence of a GCC compiler using regex
         match = re.search(r"gcc@([\d\.]+)", result.stdout)
         gcc_version = match.group(1)
-        self.logger.debug(f'Found gcc for {self.spack_config.env.env_name}: {gcc_version}')
+        self.logger.debug(f'Found gcc for {self.spack_config.env.name}: {gcc_version}')
         return gcc_version
 
     def get_spack_installed_version(self):
-        spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+        spack_version = run_command("bash", "-c", f'{self.spack_setup_script} spack --version',
                                     capture_output=True, text=True, check=True,
                                     logger=self.logger,
                                     info_msg=f"Getting spack version",
@@ -159,19 +170,19 @@ class SpackOperation:
         run_command("bash", "-c",
                     f'{self.spack_command_on_env} && spack concretize {force}',
                     check=True,
-                     logger=self.logger,
-                    info_msg=f'Concertization step for {self.spack_config.env.env_name}',
-                    exception_msg=f'Failed the concertization step for {self.spack_config.env.env_name}',
+                    logger=self.logger,
+                    info_msg=f'Concertization step for {self.spack_config.env.name}',
+                    exception_msg=f'Failed the concertization step for {self.spack_config.env.name}',
                     exception=SpackConcertizeException)
 
     def create_gpg_keys(self):
         if self.spack_config.gpg:
             run_command("bash", "-c",
-                        f'source {self.spack_setup_script} && spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
+                        f'{self.spack_setup_script} spack gpg init && spack gpg create {self.spack_config.gpg.name} {self.spack_config.gpg.mail}',
                         check=True,
                         logger=self.logger,
-                        info_msg=f'Created pgp keys for {self.spack_config.env.env_name}',
-                        exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.env_name}',
+                        info_msg=f'Created pgp keys for {self.spack_config.env.name}',
+                        exception_msg=f'Failed to create pgp keys mirror {self.spack_config.env.name}',
                         exception=SpackGpgException)
         else:
             raise SpackGpgException('No GPG configuration was defined is spack configuration')
@@ -181,7 +192,7 @@ class SpackOperation:
         signed = '--signed' if signed else ''
         if global_mirror:
             run_command("bash", "-c",
-                        f'source {self.spack_setup_script} && spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
+                        f'{self.spack_setup_script} spack mirror add {autopush} {signed} {mirror_name} {mirror_path}',
                         check=True,
                         logger=self.logger,
                         info_msg=f'Added mirror {mirror_name}',
@@ -199,7 +210,7 @@ class SpackOperation:
 
     def remove_mirror(self, mirror_name: str):
         run_command("bash", "-c",
-                    f'source {self.spack_setup_script} && spack mirror rm {mirror_name}',
+                    f'{self.spack_setup_script} spack mirror rm {mirror_name}',
                     check=True,
                     logger=self.logger,
                     info_msg=f'Removing mirror {mirror_name}',
@@ -212,18 +223,19 @@ class SpackOperation:
         fresh = '--fresh' if fresh else ''
         debug = '--debug' if debug else ''
         install_result = run_command("bash", "-c",
-                                     f'{self.spack_command_on_env} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+                                     f'{self.spack_command_on_env} && spack {debug} install -v {signed} -j {jobs} {fresh}',
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE,
                                      text=True,
                                      logger=self.logger,
-                                     info_msg=f"Installing spack packages for {self.spack_config.env.env_name}",
-                                     exception_msg=f"Error installing spack packages for {self.spack_config.env.env_name}",
+                                     info_msg=f"Installing spack packages for {self.spack_config.env.name}",
+                                     exception_msg=f"Error installing spack packages for {self.spack_config.env.name}",
                                      exception=SpackInstallPackagesException)
         log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
         return install_result
 
-    def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack'):
+    def install_spack(self, spack_version=f'v{SPACK_VERSION}', spack_repo='https://github.com/spack/spack',
+                      bashrc_path=os.path.expanduser("~/.bashrc")):
         try:
             user = os.getlogin()
         except OSError:
@@ -241,23 +253,24 @@ class SpackOperation:
         else:
             self.logger.debug("Spack already cloned.")
 
-        bashrc_path = os.path.expanduser("~/.bashrc")
         # ensure the file exists before opening it
         if not os.path.exists(bashrc_path):
             open(bashrc_path, "w").close()
         # add spack setup commands to .bashrc
         with open(bashrc_path, "a") as bashrc:
             bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
-            bashrc.write(f"source {self.spack_setup_script}\n")
+            spack_setup_script = f"source {self.spack_dir / 'share' / 'spack' / 'setup-env.sh'}"
+            bashrc.write(f"{spack_setup_script}\n")
         self.logger.info("Added Spack PATH to .bashrc")
         if user:
             run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
                         info_msg='Adding permissions to the logged in user')
-        run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
         self.logger.info("Spack install completed")
-        # Restart Bash after the installation ends
-        os.system("exec bash")
-
+        if self.spack_config.use_spack_global is True:
+            # Restart the bash only of the spack is used globally
+            self.logger.info('Restarting bash')
+            run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, info_msg='Restart bash')
+            os.system("exec bash")
         # Configure upstream Spack instance if specified
         if self.spack_config.upstream_instance:
             upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")

dedal/spack_factory/SpackOperationCreateCache.py

+import os
+
+from dedal.error_handling.exceptions import NoSpackEnvironmentException
+
+from dedal.utils.utils import copy_to_tmp, copy_file
+from dedal.wrapper.spack_wrapper import check_spack_env
+from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.logger.logger_builder import get_logger
+from dedal.spack_factory.SpackOperation import SpackOperation
+
+
+class SpackOperationCreateCache(SpackOperation):
+    """
+    This class creates caching for the concretization step and for the installation step.
+    """
+
+    def __init__(self, spack_config: SpackConfig = SpackConfig()):
+        super().__init__(spack_config, logger=get_logger(__name__))
+        self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
+                                                  os.environ.get('CONCRETIZE_OCI_PROJECT'),
+                                                  os.environ.get('CONCRETIZE_OCI_USERNAME'),
+                                                  os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+                                                  cache_version=spack_config.cache_version_concretize)
+        self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
+                                             os.environ.get('BUILDCACHE_OCI_PROJECT'),
+                                             os.environ.get('BUILDCACHE_OCI_USERNAME'),
+                                             os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+                                             cache_version=spack_config.cache_version_build)
+
+    @check_spack_env
+    def concretize_spack_env(self):
+        super().concretize_spack_env(force=True)
+        dependency_path = self.spack_config.env.path / self.spack_config.env.name / 'spack.lock'
+        copy_file(dependency_path, self.spack_config.concretization_dir, logger=self.logger)
+        self.cache_dependency.upload(self.spack_config.concretization_dir)
+        self.logger.info(f'Created new spack concretization for create cache: {self.spack_config.env.name}')
+
+    @check_spack_env
+    def install_packages(self, jobs: int = 2, debug=False):
+        signed = False
+        if self.spack_config.gpg:
+            signed = True
+            self.create_gpg_keys()
+        self.add_mirror('local_cache', self.spack_config.buildcache_dir, signed=signed, autopush=signed,
+                        global_mirror=False)
+        self.logger.info(f'Added mirror for {self.spack_config.env.name}')
+        super().install_packages(jobs=jobs, signed=signed, debug=debug, fresh=True)
+        self.logger.info(f'Installed spack packages for {self.spack_config.env.name}')
+        self.build_cache.upload(self.spack_config.buildcache_dir)
+        self.logger.info(f'Pushed spack packages for {self.spack_config.env.name}')

dedal/spack_factory/SpackOperationCreator.py

 from dedal.configuration.SpackConfig import SpackConfig
 from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
 from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
 
 
 class SpackOperationCreator:
     @staticmethod
-    def get_spack_operator(spack_config: SpackConfig = None):
+    def get_spack_operator(spack_config: SpackConfig = None, use_cache: bool = False) -> SpackOperation:
         if spack_config is None:
-            return SpackOperation(SpackConfig())
+            return SpackOperation()
         elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None:
             return SpackOperation(spack_config)
-        else:
+        elif (spack_config.concretization_dir and spack_config.buildcache_dir) and not use_cache:
+            return SpackOperationCreateCache(spack_config)
+        elif (spack_config.concretization_dir and spack_config.buildcache_dir) and use_cache:
             return SpackOperationUseCache(spack_config)
+        else:
+            return SpackOperation(SpackConfig())

dedal/spack_factory/SpackOperationUseCache.py

 import os
+import subprocess
+from pathlib import Path
+
 from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.error_handling.exceptions import SpackInstallPackagesException
 from dedal.logger.logger_builder import get_logger
 from dedal.spack_factory.SpackOperation import SpackOperation
 from dedal.configuration.SpackConfig import SpackConfig
+from dedal.utils.utils import file_exists_and_not_empty, run_command, log_command, copy_to_tmp, copy_file
+from dedal.wrapper.spack_wrapper import check_spack_env
 
 
 class SpackOperationUseCache(SpackOperation):
@@ -10,23 +16,52 @@ class SpackOperationUseCache(SpackOperation):
     This class uses caching for the concretization step and for the installation step.
     """
 
-    def __init__(self, spack_config: SpackConfig = SpackConfig(), cache_version_concretize='v1',
-                 cache_version_build='v1'):
+    def __init__(self, spack_config: SpackConfig = SpackConfig()):
         super().__init__(spack_config, logger=get_logger(__name__))
         self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
                                                   os.environ.get('CONCRETIZE_OCI_PROJECT'),
                                                   os.environ.get('CONCRETIZE_OCI_USERNAME'),
                                                   os.environ.get('CONCRETIZE_OCI_PASSWORD'),
-                                                  cache_version=cache_version_concretize)
+                                                  cache_version=spack_config.cache_version_concretize)
         self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
                                              os.environ.get('BUILDCACHE_OCI_PROJECT'),
                                              os.environ.get('BUILDCACHE_OCI_USERNAME'),
                                              os.environ.get('BUILDCACHE_OCI_PASSWORD'),
-                                             cache_version=cache_version_build)
+                                             cache_version=spack_config.cache_version_build)
 
     def setup_spack_env(self):
         super().setup_spack_env()
         # todo add buildcache to the spack environment
 
-    def concretize_spack_env(self, force=True):
-        pass
+    @check_spack_env
+    def concretize_spack_env(self):
+        concretization_redo = False
+        self.cache_dependency.download(self.spack_config.concretization_dir)
+        if file_exists_and_not_empty(self.spack_config.concretization_dir / 'spack.lock'):
+            concretization_file_path = self.env_path / 'spack.lock'
+            copy_file(self.spack_config.concretization_dir / 'spack.lock', self.env_path)
+            # redo the concretization step if spack.lock file was not downloaded from the cache
+            if not file_exists_and_not_empty(concretization_file_path):
+                super().concretize_spack_env(force=True)
+                concretization_redo = True
+        else:
+            # redo the concretization step if spack.lock file was not downloaded from the cache
+            super().concretize_spack_env(force=True)
+            concretization_redo = True
+        return concretization_redo
+
+    @check_spack_env
+    def install_packages(self, jobs: int, signed=True, debug=False):
+        signed = '' if signed else '--no-check-signature'
+        debug = '--debug' if debug else ''
+        install_result = run_command("bash", "-c",
+                                     f'{self.spack_command_on_env} && spack {debug} install -v --reuse {signed} -j {jobs}',
+                                     stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     text=True,
+                                     logger=self.logger,
+                                     info_msg=f"Installing spack packages for {self.spack_config.env.name}",
+                                     exception_msg=f"Error installing spack packages for {self.spack_config.env.name}",
+                                     exception=SpackInstallPackagesException)
+        log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
+        return install_result

dedal/tests/integration_tests/spack_create_cache_test.py

+from pathlib import Path
+
+import pytest
+
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
+
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
+
+"""
+Before running those tests, the repositories where the caching is stored must be cleared after each run. 
+Ebrains Harbour does not support deletion via API, so the clean up must be done manually
+"""
+
+
+@pytest.mark.skip(
+    reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.")
+def test_spack_create_cache_concretization(tmp_path):
+    install_dir = tmp_path
+    concretization_dir = install_dir / 'concretization'
+    buildcache_dir = install_dir / 'buildcache'
+    env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+    repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+    gpg = GpgConfig(gpg_name='test-gpg', gpg_mail='test@test.com')
+    config = SpackConfig(env=env, install_dir=install_dir, concretization_dir=concretization_dir,
+                         buildcache_dir=buildcache_dir, gpg=gpg)
+    config.add_repo(repo)
+    spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperationCreateCache)
+    spack_operation.install_spack()
+    spack_operation.setup_spack_env()
+    spack_operation.concretize_spack_env()
+    assert len(spack_operation.cache_dependency.list_tags()) > 0
+
+
+@pytest.mark.skip(
+    reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.")
+def test_spack_create_cache_installation(tmp_path):
+    install_dir = tmp_path
+    concretization_dir = install_dir / 'concretization'
+    buildcache_dir = install_dir / 'buildcache'
+    env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+    repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+    gpg = GpgConfig(gpg_name='test-gpg', gpg_mail='test@test.com')
+    config = SpackConfig(env=env, install_dir=install_dir, concretization_dir=concretization_dir,
+                         buildcache_dir=buildcache_dir, gpg=gpg)
+    config.add_repo(repo)
+    spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperationCreateCache)
+    spack_operation.install_spack()
+    spack_operation.setup_spack_env()
+    spack_operation.concretize_spack_env()
+    assert len(spack_operation.cache_dependency.list_tags()) > 0
+    spack_operation.install_packages()
+    assert len(spack_operation.build_cache.list_tags()) > 0

dedal/tests/integration_tests/spack_from_cache_test.py

+import pytest
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.utils.utils import file_exists_and_not_empty
+from dedal.utils.variables import test_spack_env_git, ebrains_spack_builds_git
+
+
+def test_spack_from_cache_concretize(tmp_path):
+    install_dir = tmp_path
+    env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+    repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+    spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir / 'concretize',
+                               buildcache_dir=install_dir / 'buildcache')
+    spack_config.add_repo(repo)
+    spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+    assert isinstance(spack_operation, SpackOperationUseCache)
+    spack_operation.install_spack()
+    spack_operation.setup_spack_env()
+    assert spack_operation.concretize_spack_env() == False
+    concretization_file_path = spack_operation.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+@pytest.mark.skip(reason="Skipping test::test_spack_from_cache_install until all the functionalities in SpackOperationUseCache")
+def test_spack_from_cache_install(tmp_path):
+    install_dir = tmp_path
+    env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+    repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+    spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir / 'concretize',
+                               buildcache_dir=install_dir / 'buildcache')
+    spack_config.add_repo(repo)
+    spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+    assert isinstance(spack_operation, SpackOperationUseCache)
+    spack_operation.install_spack()
+    spack_operation.setup_spack_env()
+    assert spack_operation.concretize_spack_env() == False
+    concretization_file_path = spack_operation.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+    install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False)
+    assert install_result.returncode == 0

dedal/tests/integration_tests/spack_from_scratch_test.py

@@ -6,33 +6,28 @@ from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
 from dedal.model.SpackDescriptor import SpackDescriptor
 from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
 from dedal.utils.utils import file_exists_and_not_empty
+from dedal.spack_factory.SpackOperation import SpackOperation
 
 
-def test_spack_repo_exists_1():
-    spack_operation = SpackOperationCreator.get_spack_operator()
-    spack_operation.install_spack()
-    assert spack_operation.spack_repo_exists('ebrains-spack-builds') == False
-
-
-def test_spack_repo_exists_2(tmp_path):
+def test_spack_repo_exists_1(tmp_path):
     install_dir = tmp_path
     env = SpackDescriptor('ebrains-spack-builds', install_dir)
     config = SpackConfig(env=env, install_dir=install_dir)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
     spack_operation.install_spack()
     with pytest.raises(NoSpackEnvironmentException):
-        spack_operation.spack_repo_exists(env.env_name)
+        spack_operation.spack_repo_exists(env.name)
 
 
-def test_spack_repo_exists_3(tmp_path):
+def test_spack_repo_exists_2(tmp_path):
     install_dir = tmp_path
     env = SpackDescriptor('ebrains-spack-builds', install_dir)
     config = SpackConfig(env=env, install_dir=install_dir)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
-    print(spack_operation.get_spack_installed_version())
     spack_operation.setup_spack_env()
-    assert spack_operation.spack_repo_exists(env.env_name) == False
+    assert spack_operation.spack_repo_exists(env.name) == False
 
 
 def test_spack_from_scratch_setup_1(tmp_path):
@@ -40,9 +35,10 @@ def test_spack_from_scratch_setup_1(tmp_path):
     env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
     config = SpackConfig(env=env, system_name='ebrainslab', install_dir=install_dir)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
-    assert spack_operation.spack_repo_exists(env.env_name) == False
+    assert spack_operation.spack_repo_exists(env.name) == False
 
 
 def test_spack_from_scratch_setup_2(tmp_path):
@@ -53,9 +49,10 @@ def test_spack_from_scratch_setup_2(tmp_path):
     config.add_repo(repo)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
-    assert spack_operation.spack_repo_exists(env.env_name) == True
+    assert spack_operation.spack_repo_exists(env.name) == True
 
 
 def test_spack_from_scratch_setup_3(tmp_path):
@@ -66,6 +63,7 @@ def test_spack_from_scratch_setup_3(tmp_path):
     config.add_repo(repo)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     with pytest.raises(BashCommandException):
         spack_operation.setup_spack_env()
@@ -76,6 +74,7 @@ def test_spack_from_scratch_setup_4(tmp_path):
     env = SpackDescriptor('new_env2', install_dir)
     config = SpackConfig(env=env, install_dir=install_dir)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     assert spack_operation.spack_env_exists() == True
@@ -87,10 +86,13 @@ def test_spack_not_a_valid_repo():
     config = SpackConfig(env=env, system_name='ebrainslab')
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     with pytest.raises(BashCommandException):
-        spack_operation.add_spack_repo(repo.path, repo.env_name)
+        spack_operation.add_spack_repo(repo.path, repo.name)
 
 
+@pytest.mark.skip(
+    reason="Skipping the concretization step because it may freeze when numerous Spack packages are added to the environment.")
 def test_spack_from_scratch_concretize_1(tmp_path):
     install_dir = tmp_path
     env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
@@ -99,6 +101,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
     config.add_repo(repo)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
@@ -107,6 +110,8 @@ def test_spack_from_scratch_concretize_1(tmp_path):
     assert file_exists_and_not_empty(concretization_file_path) == True
 
 
+@pytest.mark.skip(
+    reason="Skipping the concretization step because it may freeze when numerous Spack packages are added to the environment.")
 def test_spack_from_scratch_concretize_2(tmp_path):
     install_dir = tmp_path
     env = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
@@ -115,6 +120,7 @@ def test_spack_from_scratch_concretize_2(tmp_path):
     config.add_repo(repo)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     spack_operation.concretize_spack_env(force=False)
@@ -130,6 +136,7 @@ def test_spack_from_scratch_concretize_3(tmp_path):
     config.add_repo(repo)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     concretization_file_path = spack_operation.env_path / 'spack.lock'
@@ -141,6 +148,7 @@ def test_spack_from_scratch_concretize_4(tmp_path):
     env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
     config = SpackConfig(env=env, install_dir=install_dir)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     spack_operation.concretize_spack_env(force=False)
@@ -153,6 +161,7 @@ def test_spack_from_scratch_concretize_5(tmp_path):
     env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
     config = SpackConfig(env=env, install_dir=install_dir)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     spack_operation.concretize_spack_env(force=True)
@@ -167,6 +176,7 @@ def test_spack_from_scratch_concretize_6(tmp_path):
     config = SpackConfig(env=env, install_dir=install_dir)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     spack_operation.concretize_spack_env(force=False)
@@ -181,6 +191,7 @@ def test_spack_from_scratch_concretize_7(tmp_path):
     config = SpackConfig(env=env)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     spack_operation.concretize_spack_env(force=True)
@@ -195,6 +206,7 @@ def test_spack_from_scratch_install(tmp_path):
     config = SpackConfig(env=env)
     config.add_repo(repo)
     spack_operation = SpackOperationCreator.get_spack_operator(config)
+    assert isinstance(spack_operation, SpackOperation)
     spack_operation.install_spack()
     spack_operation.setup_spack_env()
     spack_operation.concretize_spack_env(force=True)

dedal/tests/integration_tests/spack_install_test.py

-import pytest
+from dedal.configuration.SpackConfig import SpackConfig
 from dedal.spack_factory.SpackOperation import SpackOperation
 from dedal.tests.testing_variables import SPACK_VERSION
 
 
-# run this test first so that spack is installed only once for all the tests
-@pytest.mark.run(order=1)
-def test_spack_install_scratch():
-    spack_operation = SpackOperation()
+def test_spack_install_scratch(tmp_path):
+    install_dir = tmp_path
+    spack_config = SpackConfig(install_dir=install_dir)
+    spack_operation = SpackOperation(spack_config)
     spack_operation.install_spack(spack_version=f'v{SPACK_VERSION}')
     installed_spack_version = spack_operation.get_spack_installed_version()
     assert SPACK_VERSION == installed_spack_version