esd-spack-installation: setup; tests; custom exceptions

.gitlab-ci.yml

 stages:
-  - build
   - test
+  - build
 
 variables:
   BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
 
+
 build-wheel:
   stage: build
   tags:
@@ -21,18 +22,23 @@ build-wheel:
     expire_in: 1 week
 
 
-testing:
+testing-pytest:
   stage: test
   tags:
     - docker-runner
-  image: python:latest
+  image: ubuntu:22.04
   script:
-    - pip install -e .
-    - pytest ./dedal/tests/ --junitxml=test-results.xml
+    - chmod +x dedal/utils/bootstrap.sh
+    - ./dedal/utils/bootstrap.sh
+    - pip install .
+    - pytest ./dedal/tests/ -s --junitxml=test-results.xml
   artifacts:
     when: always
     reports:
       junit: test-results.xml
     paths:
       - test-results.xml
-    expire_in: 1 week
\ No newline at end of file
+      - .dedal.log
+      - .generate_cache.log
+    expire_in: 1 week
+

dedal/build_cache/BuildCacheManager.py

@@ -40,8 +40,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
 
         for sub_path in build_cache_path.rglob("*"):
             if sub_path.is_file():
-                rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
-                target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
+                rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
+                target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
                 try:
                     self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
                     self.client.push(
@@ -51,7 +51,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
                         manifest_annotations={"path": rel_path},
                         disable_path_validation=True,
                     )
-                    self.logger.info(f"Successfully pushed {sub_path.name}")
+                    self.logger.info(f"Successfully pushed {sub_path.env_name}")
                 except Exception as e:
                     self.logger.error(
                         f"An error occurred while pushing: {e}")

dedal/specfile_storage_path_source.py

@@ -49,14 +49,14 @@ for rspec in data:
 
         format_string = "{name}-{version}"
         pretty_name = pkg.spec.format_path(format_string)
-        cosmetic_path = os.path.join(pkg.name, pretty_name)
+        cosmetic_path = os.path.join(pkg.env_name, pretty_name)
         to_be_fetched.add(str(spack.mirror.mirror_archive_paths(pkg.fetcher, cosmetic_path).storage_path))
         for resource in pkg._get_needed_resources():
-            pretty_resource_name = fsys.polite_filename(f"{resource.name}-{pkg.version}")
+            pretty_resource_name = fsys.polite_filename(f"{resource.env_name}-{pkg.version}")
             to_be_fetched.add(str(spack.mirror.mirror_archive_paths(resource.fetcher, pretty_resource_name).storage_path))
         for patch in ss.patches:
             if isinstance(patch, spack.patch.UrlPatch):
-                to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.name).storage_path))
+                to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.env_name).storage_path))
 
 for elem in to_be_fetched:
     print(elem)

dedal/tests/spack_from_scratch_test.py

+from pathlib import Path
+
+import pytest
+
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+def test_spack_repo_exists_1():
+    spack_manager = SpackManagerScratch()
+    assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
+
+
+def test_spack_repo_exists_2():
+    install_dir = Path('./install').resolve()
+    env = SpackModel('ebrains-spack-builds', install_dir)
+    spack_manager = SpackManagerScratch(env=env)
+    with pytest.raises(NoSpackEnvironmentException):
+        spack_manager.spack_repo_exists(env.env_name)
+
+
+# def test_spack_repo_exists_3():
+#     install_dir = Path('./install').resolve()
+#     env = SpackModel('ebrains-spack-builds', install_dir)
+#     spack_manager = SpackManagerScratch(env=env)
+#     spack_manager.setup_spack_env()
+#     assert spack_manager.spack_repo_exists(env.env_name) == False
+
+
+def test_spack_from_scratch_setup_1():
+    install_dir = Path('./install').resolve()
+    env = SpackModel('ebrains-spack-builds', install_dir,
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+    spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
+    spack_manager.setup_spack_env()
+    assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_2():
+    install_dir = Path('./install').resolve()
+    env = SpackModel('ebrains-spack-builds', install_dir,
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+    repo = env
+    spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+    spack_manager.setup_spack_env()
+    assert spack_manager.spack_repo_exists(env.env_name) == True
+
+
+def test_spack_from_scratch_setup_3():
+    install_dir = Path('./install').resolve()
+    env = SpackModel('new_env1', install_dir)
+    repo = env
+    spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+    with pytest.raises(BashCommandException):
+        spack_manager.setup_spack_env()
+
+
+def test_spack_from_scratch_setup_4():
+    install_dir = Path('./install').resolve()
+    env = SpackModel('new_env2', install_dir)
+    spack_manager = SpackManagerScratch(env=env)
+    spack_manager.setup_spack_env()
+    assert spack_manager.spack_env_exists() == True
+
+
+def test_spack_not_a_valid_repo():
+    env = SpackModel('ebrains-spack-builds', Path(), None)
+    repo = env
+    spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
+    with pytest.raises(NoSpackEnvironmentException):
+        spack_manager.add_spack_repo(repo.path, repo.env_name)

dedal/tests/spack_install_test.py

+import pytest
+
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+# we need this test to run first so that spack is installed only once
+@pytest.mark.run(order=1)
+def test_spack_install_scratch():
+    spack_manager = SpackManagerScratch()
+    spack_manager.install_spack(spack_version="v0.21.1")
+    installed_spack_version = spack_manager.get_spack_installed_version()
+    required_version = "0.21.1"
+    assert required_version == installed_spack_version
+
+
+def test_spack_install_buildcache():
+    spack_manager = SpackManagerBuildCache()
+    installed_spack_version = spack_manager.get_spack_installed_version()
+    required_version = "0.21.1"
+    assert required_version == installed_spack_version

dedal/utils/bootstrap.sh

+# Minimal prerequisites for installing the esd_library
+# pip must be installed on the OS
+echo "Bootstrapping..."
+apt update
+apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd
+python3 -m pip install --upgrade pip setuptools wheel

dedal/utils/utils.py

+import logging
 import shutil
+import subprocess
 from pathlib import Path
 
+from esd.error_handling.exceptions import BashCommandException
+
 
 def clean_up(dirs: list[str], logging, ignore_errors=True):
     """
@@ -18,3 +22,32 @@ def clean_up(dirs: list[str], logging, ignore_errors=True):
                     raise e
         else:
             logging.info(f"{cleanup_dir} does not exist")
+
+
+def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
+                exception=None, **kwargs):
+    try:
+        logger.debug(f'{debug_msg}: args: {args}')
+        return subprocess.run(args, **kwargs)
+    except subprocess.CalledProcessError as e:
+        if exception_msg is not None:
+            logger.error(f"{exception_msg}: {e}")
+        if exception is not None:
+            raise exception(f'{exception_msg} : {e}')
+        else:
+            return None
+
+
+def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
+    if not dir.exists():
+        run_command(
+            "git", "clone", "--depth", "1",
+            "-c", "advice.detachedHead=false",
+            "-c", "feature.manyFiles=true",
+            git_path, dir
+            , check=True, logger=logger,
+            debug_msg=f'Cloned repository {repo_name}',
+            exception_msg=f'Failed to clone repository: {repo_name}',
+            exception=BashCommandException)
+    else:
+        logger.debug(f'Repository {repo_name} already cloned.')

esd/error_handling/exceptions.py

+class SpackException(Exception):
+
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = str(message)
+
+    def __str__(self):
+        return self.message
+
+class BashCommandException(SpackException):
+    """
+    To be thrown when an invalid input is received.
+    """
+
+class NoSpackEnvironmentException(SpackException):
+    """
+    To be thrown when an invalid input is received.
+    """
\ No newline at end of file

esd/model/SpackModel.py

+from pathlib import Path
+
+
+class SpackModel:
+    """"
+    Provides details about the spack environment
+    """
+
+    def __init__(self, env_name: str, path: Path, git_path: str = None):
+        self.env_name = env_name
+        self.path = path
+        self.git_path = git_path

esd/spack_manager/SpackManager.py

+import os
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.logger.logger_builder import get_logger
+from esd.model.SpackModel import SpackModel
+from esd.utils.utils import run_command, git_clone_repo
+
+
+class SpackManager(ABC):
+    """
+    This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages.
+    Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache.
+
+    Attributes:
+    -----------
+    env : SpackModel
+        spack environment details
+    repos : list[SpackModel]
+    upstream_instance : str
+        path to Spack instance to use as upstream (optional)
+    """
+
+    def __init__(self, env: SpackModel = None, repos=None,
+                 upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
+        if repos is None:
+            repos = []
+        self.repos = repos
+        self.env = env
+        self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
+        self.install_dir.mkdir(parents=True, exist_ok=True)
+        self.env_path = None
+        if self.env and self.env.path:
+            self.env.path = self.env.path.resolve()
+            self.env.path.mkdir(parents=True, exist_ok=True)
+            self.env_path = self.env.path / self.env.env_name
+        self.upstream_instance = upstream_instance
+        self.spack_dir = self.install_dir / "spack"
+        self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
+        self.logger = logger
+        self.system_name = system_name
+
+    @abstractmethod
+    def concretize_spack_env(self, force=True):
+        pass
+
+    @abstractmethod
+    def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+        pass
+
+    def create_fetch_spack_environment(self):
+
+        if self.env.git_path:
+            git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
+        else:
+            os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
+            run_command("bash", "-c",
+                        f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
+                        check=True, logger=self.logger,
+                        debug_msg=f"Created {self.env.env_name} spack environment",
+                        exception_msg=f"Failed to create {self.env.env_name} spack environment",
+                        exception=BashCommandException)
+
+    def setup_spack_env(self):
+        """
+        This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos
+        """
+        bashrc_path = os.path.expanduser("~/.bashrc")
+        if self.system_name:
+            with open(bashrc_path, "a") as bashrc:
+                bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n')
+                os.environ['SYSTEMNAME'] = self.system_name
+        if self.spack_dir.exists() and self.spack_dir.is_dir():
+            with open(bashrc_path, "a") as bashrc:
+                bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n')
+                bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n')
+            self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH')
+        else:
+            self.logger.error(f'Invalid installation path: {self.spack_dir}')
+        # Restart the bash after adding environment variables
+        self.create_fetch_spack_environment()
+        if self.install_dir.exists():
+            for repo in self.repos:
+                repo_dir = self.install_dir / repo.path / repo.env_name
+                git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger)
+                if not self.spack_repo_exists(repo.env_name):
+                    self.add_spack_repo(repo.path, repo.env_name)
+                    self.logger.debug(f'Added spack repository {repo.env_name}')
+                else:
+                    self.logger.debug(f'Spack repository {repo.env_name} already added')
+
+    def spack_repo_exists(self, repo_name: str) -> bool | None:
+        """Check if the given Spack repository exists."""
+        if self.env is None:
+            result = run_command("bash", "-c",
+                                 f'source {self.spack_setup_script} && spack repo list',
+                                 check=True,
+                                 capture_output=True, text=True, logger=self.logger,
+                                 debug_msg=f'Checking if {repo_name} exists')
+            if result is None:
+                return False
+        else:
+            if self.spack_env_exists():
+                result = run_command("bash", "-c",
+                                 f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+                                 check=True,
+                                 capture_output=True, text=True, logger=self.logger,
+                                 debug_msg=f'Checking if repository {repo_name} was added')
+            else:
+                self.logger.debug('No spack environment defined')
+                raise NoSpackEnvironmentException('No spack environment defined')
+            if result is None:
+                return False
+        return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
+
+    def spack_env_exists(self):
+        result = run_command("bash", "-c",
+                             f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
+                             check=True,
+                             capture_output=True, text=True, logger=self.logger,
+                             debug_msg=f'Checking if environment {self.env.env_name} exists')
+        if result is None:
+            return False
+        return True
+
+    def add_spack_repo(self, repo_path: Path, repo_name: str):
+        """Add the Spack repository if it does not exist."""
+        if self.spack_env_exists():
+            run_command("bash", "-c",
+                        f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+                        check=True, logger=self.logger,
+                        debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+                        exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+                        exception=BashCommandException)
+        else:
+            self.logger.debug('No spack environment defined')
+            raise NoSpackEnvironmentException('No spack environment defined')
+
+    def get_spack_installed_version(self):
+        spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
+                                    capture_output=True, text=True, check=True,
+                                    logger=self.logger,
+                                    debug_msg=f"Getting spack version",
+                                    exception_msg=f"Error retrieving Spack version")
+        if spack_version:
+            return spack_version.stdout.strip().split()[0]
+        return None
+
+    def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+        try:
+            user = os.getlogin()
+        except OSError:
+            user = None
+
+        self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}")
+        if not self.spack_dir.exists():
+            run_command(
+                "git", "clone", "--depth", "1",
+                "-c", "advice.detachedHead=false",
+                "-c", "feature.manyFiles=true",
+                "--branch", spack_version, spack_repo, self.spack_dir
+                , check=True, logger=self.logger)
+            self.logger.debug("Cloned spack")
+        else:
+            self.logger.debug("Spack already cloned.")
+
+        bashrc_path = os.path.expanduser("~/.bashrc")
+        # ensure the file exists before opening it
+        if not os.path.exists(bashrc_path):
+            open(bashrc_path, "w").close()
+        # add spack setup commands to .bashrc
+        with open(bashrc_path, "a") as bashrc:
+            bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n')
+            bashrc.write(f"source {self.spack_setup_script}\n")
+        self.logger.info("Added Spack PATH to .bashrc")
+        if user:
+            run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
+                        debug_msg='Adding permissions to the logged in user')
+        run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
+        self.logger.info("Spack install completed")
+        # Restart Bash after the installation ends
+        os.system("exec bash")
+
+        # Configure upstream Spack instance if specified
+        if self.upstream_instance:
+            upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml")
+            with open(upstreams_yaml_path, "w") as file:
+                file.write(f"""upstreams:
+                                  upstream-spack-instance:
+                                    install_tree: {self.upstream_instance}/spack/opt/spack
+                                """)
+            self.logger.info("Added upstream spack instance")

esd/spack_manager/enums/SpackManagerEnum.py

+from enum import Enum
+
+
+class SpackManagerEnum(Enum):
+    FROM_SCRATCH = "from_scratch",
+    FROM_BUILDCACHE = "from_buildcache",

esd/spack_manager/factory/SpackManagerBuildCache.py

+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerBuildCache(SpackManager):
+    def __init__(self, env: SpackModel = None, repos=None,
+                 upstream_instance=None, system_name: str = None):
+        super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+    def setup_spack_env(self):
+        super().setup_spack_env()
+        # todo add buildcache to the spack environment
+
+    def concretize_spack_env(self, force=True):
+        pass
+
+    def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+        pass

esd/spack_manager/factory/SpackManagerCreator.py

+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
+from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+
+
+class SpackManagerCreator:
+    @staticmethod
+    def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
+                         upstream_instance: str):
+        if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
+            return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+        elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
+            return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
+

esd/spack_manager/factory/SpackManagerScratch.py

+from esd.model.SpackModel import SpackModel
+from esd.spack_manager.SpackManager import SpackManager
+from esd.logger.logger_builder import get_logger
+
+
+class SpackManagerScratch(SpackManager):
+    def __init__(self, env: SpackModel = None, repos=None,
+                 upstream_instance=None, system_name: str = None):
+        super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
+
+    def concretize_spack_env(self, force=True):
+        pass
+
+    def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
+        pass

pyproject.toml

 [build-system]
-requires = ["setuptools", "setuptools-scm"]
+requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "dedal"
+version = "0.1.0"
 authors = [
     {name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
     {name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"},
 ]
 description = "This package provides all the necessary tools to create an Ebrains Software Distribution environment"
-version = "0.1.0"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
@@ -18,6 +18,7 @@ dependencies = [
     "ruamel.yaml",
     "pytest",
     "pytest-mock",
+    "pytest-ordering",
 ]
 
 [tool.setuptools.data-files]