esd-spack-installation: added concretization step and tests

README.md

 # ~~Yashchiki~~Koutakia
 
 For now, this repository provides helpers for the EBRAINS container image build flow.
+The lowest spack version which should be used with this library is v0.22.0

dedal/tests/spack_from_scratch_test.py

@@ -4,62 +4,70 @@ import pytest
 
 from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
 from esd.model.SpackModel import SpackModel
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
 from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+from esd.utils.utils import file_exists_and_not_empty
 
 
 def test_spack_repo_exists_1():
-    spack_manager = SpackManagerScratch()
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
     assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
 
 
-def test_spack_repo_exists_2():
-    install_dir = Path('./install').resolve()
+def test_spack_repo_exists_2(tmp_path):
+    install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir)
-    spack_manager = SpackManagerScratch(env=env)
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
     with pytest.raises(NoSpackEnvironmentException):
         spack_manager.spack_repo_exists(env.env_name)
 
 
-# def test_spack_repo_exists_3():
-#     install_dir = Path('./install').resolve()
-#     env = SpackModel('ebrains-spack-builds', install_dir)
-#     spack_manager = SpackManagerScratch(env=env)
-#     spack_manager.setup_spack_env()
-#     assert spack_manager.spack_repo_exists(env.env_name) == False
+def test_spack_repo_exists_3(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('ebrains-spack-builds', install_dir)
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+    spack_manager.setup_spack_env()
+    assert spack_manager.spack_repo_exists(env.env_name) == False
 
 
-def test_spack_from_scratch_setup_1():
-    install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_1(tmp_path):
+    install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
                      'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
-    spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+                                                                              system_name='ebrainslab')
     spack_manager.setup_spack_env()
-    assert spack_manager.spack_repo_exists(env.env_name) == True
+    assert spack_manager.spack_repo_exists(env.env_name) == False
 
 
-def test_spack_from_scratch_setup_2():
-    install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_2(tmp_path):
+    install_dir = tmp_path
     env = SpackModel('ebrains-spack-builds', install_dir,
                      'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
     repo = env
-    spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+                                                                              repos=[repo, repo],
+                                                                              system_name='ebrainslab')
     spack_manager.setup_spack_env()
     assert spack_manager.spack_repo_exists(env.env_name) == True
 
 
-def test_spack_from_scratch_setup_3():
-    install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_3(tmp_path):
+    install_dir = tmp_path
     env = SpackModel('new_env1', install_dir)
     repo = env
-    spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+                                                                              repos=[repo, repo],
+                                                                              system_name='ebrainslab')
     with pytest.raises(BashCommandException):
         spack_manager.setup_spack_env()
 
 
-def test_spack_from_scratch_setup_4():
-    install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_4(tmp_path):
+    install_dir = tmp_path
     env = SpackModel('new_env2', install_dir)
-    spack_manager = SpackManagerScratch(env=env)
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
     spack_manager.setup_spack_env()
     assert spack_manager.spack_env_exists() == True
 
@@ -67,6 +75,47 @@ def test_spack_from_scratch_setup_4():
 def test_spack_not_a_valid_repo():
     env = SpackModel('ebrains-spack-builds', Path(), None)
     repo = env
-    spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+                                                                              repos=[repo],
+                                                                              system_name='ebrainslab')
     with pytest.raises(NoSpackEnvironmentException):
         spack_manager.add_spack_repo(repo.path, repo.env_name)
+
+
+def test_spack_from_scratch_concretize_1(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('ebrains-spack-builds', install_dir,
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+    repo = env
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+                                                         system_name='ebrainslab')
+    spack_manager.setup_spack_env()
+    spack_manager.concretize_spack_env(force=True)
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_2(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('ebrains-spack-builds', install_dir,
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+    repo = env
+    spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+                                                         system_name='ebrainslab')
+    spack_manager.setup_spack_env()
+    spack_manager.concretize_spack_env(force=False)
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_3(tmp_path):
+    install_dir = tmp_path
+    env = SpackModel('ebrains-spack-builds', install_dir,
+                     'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+    repo = env
+    spack_manager  = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+                                                                              repos=[repo, repo],
+                                                                              system_name='ebrainslab')
+    spack_manager.setup_spack_env()
+    concretization_file_path = spack_manager.env_path / 'spack.lock'
+    assert file_exists_and_not_empty(concretization_file_path) == False

dedal/tests/spack_install_test.py

@@ -4,18 +4,18 @@ from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCa
 from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
 
 
-# we need this test to run first so that spack is installed only once
+SPACK_VERSION = "0.22.0"
+
+# we need this test to run first so that spack is installed only once for all the tests
 @pytest.mark.run(order=1)
 def test_spack_install_scratch():
     spack_manager = SpackManagerScratch()
-    spack_manager.install_spack(spack_version="v0.21.1")
+    spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
     installed_spack_version = spack_manager.get_spack_installed_version()
-    required_version = "0.21.1"
-    assert required_version == installed_spack_version
+    assert SPACK_VERSION == installed_spack_version
 
 
 def test_spack_install_buildcache():
     spack_manager = SpackManagerBuildCache()
     installed_spack_version = spack_manager.get_spack_installed_version()
-    required_version = "0.21.1"
-    assert required_version == installed_spack_version
+    assert SPACK_VERSION == installed_spack_version

dedal/tests/utils_test.py

@@ -61,3 +61,21 @@ def test_clean_up_nonexistent_dirs(mocker):
 
     for dir_path in nonexistent_dirs:
         mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
+
+
+def test_file_does_not_exist(tmp_path: Path):
+    non_existent_file = tmp_path / "non_existent.txt"
+    assert not file_exists_and_not_empty(non_existent_file)
+
+
+def test_file_exists_but_empty(tmp_path: Path):
+    empty_file = tmp_path / "empty.txt"
+    # Create an empty file
+    empty_file.touch()
+    assert not file_exists_and_not_empty(empty_file)
+
+
+def test_file_exists_and_not_empty(tmp_path: Path):
+    non_empty_file = tmp_path / "non_empty.txt"
+    non_empty_file.write_text("Some content")
+    assert file_exists_and_not_empty(non_empty_file)

dedal/utils/utils.py

@@ -51,3 +51,7 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
             exception=BashCommandException)
     else:
         logger.debug(f'Repository {repo_name} already cloned.')
+
+
+def file_exists_and_not_empty(file: Path) -> bool:
+    return file.is_file() and file.stat().st_size > 0

esd/error_handling/exceptions.py

@@ -7,12 +7,20 @@ class SpackException(Exception):
     def __str__(self):
         return self.message
 
+
 class BashCommandException(SpackException):
     """
-    To be thrown when an invalid input is received.
+    To be thrown when a bash command has failed
     """
 
+
 class NoSpackEnvironmentException(SpackException):
     """
-    To be thrown when an invalid input is received.
-    """
\ No newline at end of file
+    To be thrown when an operation on a spack environment is executed without the environment being activated or existent
+    """
+
+
+class SpackConcertizeException(SpackException):
+    """
+    To be thrown when the spack concretization step fails
+    """

esd/spack_manager/factory/SpackManagerCreator.py

+from esd.model.SpackModel import SpackModel
 from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
 from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
 from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
@@ -5,10 +6,9 @@ from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
 
 class SpackManagerCreator:
     @staticmethod
-    def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
-                         upstream_instance: str):
+    def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
+                         upstream_instance=None, system_name: str = None):
         if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
-            return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+            return SpackManagerScratch(env, repos, upstream_instance, system_name)
         elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
-            return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
-
+            return SpackManagerBuildCache(env, repos, upstream_instance, system_name)

esd/spack_manager/factory/SpackManagerScratch.py

+from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
 from esd.model.SpackModel import SpackModel
 from esd.spack_manager.SpackManager import SpackManager
 from esd.logger.logger_builder import get_logger
+from esd.utils.utils import run_command
 
 
 class SpackManagerScratch(SpackManager):
@@ -9,7 +11,18 @@ class SpackManagerScratch(SpackManager):
         super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
 
     def concretize_spack_env(self, force=True):
-        pass
+        force = '--force' if force else ''
+        if self.spack_env_exists():
+            run_command("bash", "-c",
+                        f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+                        check=True,
+                        capture_output=True, text=True, logger=self.logger,
+                        debug_msg=f'Concertization step for {self.env.env_name}',
+                        exception_msg=f'Failed the concertization step for {self.env.env_name}',
+                        exception=SpackConcertizeException)
+        else:
+            self.logger.debug('No spack environment defined')
+            raise NoSpackEnvironmentException('No spack environment defined')
 
     def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
         pass