esd-spack-installation: additional methods

.gitlab-ci.yml

@@ -5,6 +5,7 @@ stages:
 variables:
   BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
 
+
 build-wheel:
   stage: build
   tags:

esd/error_handling/exceptions.py

@@ -14,13 +14,18 @@ class BashCommandException(SpackException):
     """
 
 
-class NoSpackEnvironmentException(SpackException):
+class NoSpackEnvironmentException(BashCommandException):
     """
     To be thrown when an operation on a spack environment is executed without the environment being activated or existent
     """
 
 
-class SpackConcertizeException(SpackException):
+class SpackConcertizeException(BashCommandException):
     """
     To be thrown when the spack concretization step fails
     """
+
+class SpackInstallPackagesException(BashCommandException):
+    """
+    To be thrown when the spack fails to install spack packages
+    """

esd/spack_manager/SpackManager.py

 import os
+import re
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
+    SpackInstallPackagesException
 from esd.logger.logger_builder import get_logger
 from esd.model.SpackModel import SpackModel
+from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
 from esd.utils.utils import run_command, git_clone_repo
 
 
@@ -103,10 +106,10 @@ class SpackManager(ABC):
         else:
             if self.spack_env_exists():
                 result = run_command("bash", "-c",
-                                 f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
-                                 check=True,
-                                 capture_output=True, text=True, logger=self.logger,
-                                 debug_msg=f'Checking if repository {repo_name} was added')
+                                     f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+                                     check=True,
+                                     capture_output=True, text=True, logger=self.logger,
+                                     debug_msg=f'Checking if repository {repo_name} was added')
             else:
                 self.logger.debug('No spack environment defined')
                 raise NoSpackEnvironmentException('No spack environment defined')
@@ -124,18 +127,35 @@ class SpackManager(ABC):
             return False
         return True
 
+    @no_spack_env
     def add_spack_repo(self, repo_path: Path, repo_name: str):
         """Add the Spack repository if it does not exist."""
-        if self.spack_env_exists():
-            run_command("bash", "-c",
-                        f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
-                        check=True, logger=self.logger,
-                        debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
-                        exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
-                        exception=BashCommandException)
-        else:
-            self.logger.debug('No spack environment defined')
-            raise NoSpackEnvironmentException('No spack environment defined')
+        run_command("bash", "-c",
+                    f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+                    check=True, logger=self.logger,
+                    debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+                    exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+                    exception=BashCommandException)
+
+    @no_spack_env
+    def get_compiler_version(self):
+        result = run_command("bash", "-c",
+                             f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+                             check=True, logger=self.logger,
+                             capture_output=True, text=True,
+                             debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
+                             exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+                             exception=BashCommandException)
+        # todo add error handling and tests
+        if result.stdout is None:
+            self.logger.debug('No gcc found for {self.env.env_name}')
+            return None
+
+        # Find the first occurrence of a GCC compiler using regex
+        match = re.search(r"gcc@([\d\.]+)", result.stdout)
+        gcc_version = match.group(1)
+        self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+        return gcc_version
 
     def get_spack_installed_version(self):
         spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
@@ -147,6 +167,7 @@ class SpackManager(ABC):
             return spack_version.stdout.strip().split()[0]
         return None
 
+
     def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
         try:
             user = os.getlogin()

esd/spack_manager/wrapper/spack_wrapper.py

+import functools
+
+from esd.error_handling.exceptions import NoSpackEnvironmentException
+
+
+def no_spack_env(method):
+    @functools.wraps(method)
+    def wrapper(self, *args, **kwargs):
+        if self.spack_env_exists():
+            return method(self, *args, **kwargs)  # Call the method with 'self'
+        else:
+            self.logger.debug('No spack environment defined')
+            raise NoSpackEnvironmentException('No spack environment defined')
+
+    return wrapper

esd/tests/spack_from_scratch_test.py

+import os
 from pathlib import Path
-
 import pytest
-
 from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
 from esd.model.SpackModel import SpackModel
 from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
 from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
 from esd.utils.utils import file_exists_and_not_empty
 
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
 
 def test_spack_repo_exists_1():
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
@@ -34,9 +33,9 @@ def test_spack_repo_exists_3(tmp_path):
 def test_spack_from_scratch_setup_1(tmp_path):
     install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
-                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
-                                                                              system_name='ebrainslab')
+                                                         system_name='ebrainslab')
     spack_manager.setup_spack_env()
     assert spack_manager.spack_repo_exists(env.env_name) == False
 
@@ -44,11 +43,11 @@ def test_spack_from_scratch_setup_1(tmp_path):
 def test_spack_from_scratch_setup_2(tmp_path):
     install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
-                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
     repo = env
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
-                                                                              repos=[repo, repo],
-                                                                              system_name='ebrainslab')
+                                                         repos=[repo, repo],
+                                                         system_name='ebrainslab')
     spack_manager.setup_spack_env()
     assert spack_manager.spack_repo_exists(env.env_name) == True
 
@@ -58,8 +57,8 @@ def test_spack_from_scratch_setup_3(tmp_path):
     env = SpackModel('new_env1', install_dir)
     repo = env
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
-                                                                              repos=[repo, repo],
-                                                                              system_name='ebrainslab')
+                                                         repos=[repo, repo],
+                                                         system_name='ebrainslab')
     with pytest.raises(BashCommandException):
         spack_manager.setup_spack_env()
 
@@ -76,8 +75,8 @@ def test_spack_not_a_valid_repo():
     env = SpackModel('ebrains-spack-builds', Path(), None)
     repo = env
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
-                                                                              repos=[repo],
-                                                                              system_name='ebrainslab')
+                                                         repos=[repo],
+                                                         system_name='ebrainslab')
     with pytest.raises(NoSpackEnvironmentException):
         spack_manager.add_spack_repo(repo.path, repo.env_name)
 
@@ -85,7 +84,7 @@ def test_spack_not_a_valid_repo():
 def test_spack_from_scratch_concretize_1(tmp_path):
     install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
-                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
     repo = env
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
                                                          system_name='ebrainslab')
@@ -98,7 +97,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
 def test_spack_from_scratch_concretize_2(tmp_path):
     install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
-                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
     repo = env
     spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
                                                          system_name='ebrainslab')
@@ -111,11 +110,59 @@ def test_spack_from_scratch_concretize_2(tmp_path):
 def test_spack_from_scratch_concretize_3(tmp_path):
     install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
-                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
     repo = env
-    spack_manager  = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
-                                                                              repos=[repo, repo],
-                                                                              system_name='ebrainslab')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+                                                         repos=[repo, repo],
+                                                         system_name='ebrainslab')
     spack_manager.setup_spack_env()
     concretization_file_path = spack_manager.env_path / 'spack.lock'
     assert file_exists_and_not_empty(concretization_file_path) == False
+
+
+def test_spack_from_scratch_concretize_4(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('test-spack-env', install_dir,
+                     f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+    spack_manager.setup_spack_env()
+    spack_manager.concretize_spack_env(force=False)
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_5(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('test-spack-env', install_dir,
+                     f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+    spack_manager.setup_spack_env()
+    spack_manager.concretize_spack_env(force=True)
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_6(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('test-spack-env', install_dir,
+                     f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+    repo = SpackModel('ebrains-spack-builds', install_dir,
+                      'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+    spack_manager.setup_spack_env()
+    spack_manager.concretize_spack_env(force=False)
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_7(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('test-spack-env', install_dir,
+                     'https://gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+    repo = SpackModel('ebrains-spack-builds', install_dir,
+                      'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+    spack_manager.setup_spack_env()
+    spack_manager.concretize_spack_env(force=True)
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True