diff --git a/README.md b/README.md
index 62e00c689da73463a22bb85cc9671f8634b25c06..86ab9c2faabf361a78652d0621fae36e165c8a6c 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,4 @@
# ~~Yashchiki~~Koutakia
For now, this repository provides helpers for the EBRAINS container image build flow.
+The lowest spack version which should be used with this library is v0.22.0
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index cca0d2e9e957b0cd271c47729c9b75ff29198df3..d1aca83cc8f83f4cc7e2a32725b87ee9d8fb1e53 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -4,62 +4,70 @@ import pytest
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+from esd.utils.utils import file_exists_and_not_empty
def test_spack_repo_exists_1():
- spack_manager = SpackManagerScratch()
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
-def test_spack_repo_exists_2():
- install_dir = Path('./install').resolve()
+def test_spack_repo_exists_2(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerScratch(env=env)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
with pytest.raises(NoSpackEnvironmentException):
spack_manager.spack_repo_exists(env.env_name)
-# def test_spack_repo_exists_3():
-# install_dir = Path('./install').resolve()
-# env = SpackModel('ebrains-spack-builds', install_dir)
-# spack_manager = SpackManagerScratch(env=env)
-# spack_manager.setup_spack_env()
-# assert spack_manager.spack_repo_exists(env.env_name) == False
+def test_spack_repo_exists_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == False
-def test_spack_from_scratch_setup_1():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_1(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
- spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == True
+ assert spack_manager.spack_repo_exists(env.env_name) == False
-def test_spack_from_scratch_setup_2():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_2(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True
-def test_spack_from_scratch_setup_3():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_3(tmp_path):
+ install_dir = tmp_path
env = SpackModel('new_env1', install_dir)
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
with pytest.raises(BashCommandException):
spack_manager.setup_spack_env()
-def test_spack_from_scratch_setup_4():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_4(tmp_path):
+ install_dir = tmp_path
env = SpackModel('new_env2', install_dir)
- spack_manager = SpackManagerScratch(env=env)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
assert spack_manager.spack_env_exists() == True
@@ -67,6 +75,47 @@ def test_spack_from_scratch_setup_4():
def test_spack_not_a_valid_repo():
env = SpackModel('ebrains-spack-builds', Path(), None)
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo],
+ system_name='ebrainslab')
with pytest.raises(NoSpackEnvironmentException):
spack_manager.add_spack_repo(repo.path, repo.env_name)
+
+
+def test_spack_from_scratch_concretize_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_2(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == False
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py
index 34f683234b064d0f800bfa9376fcf6fec592d7ef..9a32d5c77ea97070b8eb1e1876cadfa4c19957bf 100644
--- a/dedal/tests/spack_install_test.py
+++ b/dedal/tests/spack_install_test.py
@@ -4,18 +4,18 @@ from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCa
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-# we need this test to run first so that spack is installed only once
+SPACK_VERSION = "0.22.0"
+
+# we need this test to run first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version="v0.21.1")
+ spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
def test_spack_install_buildcache():
spack_manager = SpackManagerBuildCache()
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/utils_test.py b/dedal/tests/utils_test.py
index 14795726ce81896eb03d9862a9c096d78123815c..256b82184ab25f5ee1d1a0c2551974c1f4cf1b0a 100644
--- a/dedal/tests/utils_test.py
+++ b/dedal/tests/utils_test.py
@@ -61,3 +61,21 @@ def test_clean_up_nonexistent_dirs(mocker):
for dir_path in nonexistent_dirs:
mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
+
+
+def test_file_does_not_exist(tmp_path: Path):
+ non_existent_file = tmp_path / "non_existent.txt"
+ assert not file_exists_and_not_empty(non_existent_file)
+
+
+def test_file_exists_but_empty(tmp_path: Path):
+ empty_file = tmp_path / "empty.txt"
+ # Create an empty file
+ empty_file.touch()
+ assert not file_exists_and_not_empty(empty_file)
+
+
+def test_file_exists_and_not_empty(tmp_path: Path):
+ non_empty_file = tmp_path / "non_empty.txt"
+ non_empty_file.write_text("Some content")
+ assert file_exists_and_not_empty(non_empty_file)
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 48c500c3a5f652dad4e5c9a3aec06120305f8cd9..173347d06b08f6092a06b4590fd969d0fb66535e 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -51,3 +51,7 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
exception=BashCommandException)
else:
logger.debug(f'Repository {repo_name} already cloned.')
+
+
+def file_exists_and_not_empty(file: Path) -> bool:
+ return file.is_file() and file.stat().st_size > 0
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 79f8051f5019992ab29baa74ffd7180985f7a108..d6de666bd472555ff75b87166d532cfd4fcc4446 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -7,12 +7,20 @@ class SpackException(Exception):
def __str__(self):
return self.message
+
class BashCommandException(SpackException):
"""
- To be thrown when an invalid input is received.
+ To be thrown when a bash command has failed
"""
+
class NoSpackEnvironmentException(SpackException):
"""
- To be thrown when an invalid input is received.
- """
\ No newline at end of file
+ To be thrown when an operation on a spack environment is executed without the environment being activated or existent
+ """
+
+
+class SpackConcertizeException(SpackException):
+ """
+ To be thrown when the spack concretization step fails
+ """
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
index 9728467f39c2753c9f12ef959c80def1150f0314..6eb26a04cfbb4fc7be66e1a2900d9698bda91b5b 100644
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -1,3 +1,4 @@
+from esd.model.SpackModel import SpackModel
from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
@@ -5,10 +6,9 @@ from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
class SpackManagerCreator:
@staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
- upstream_instance: str):
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ return SpackManagerScratch(env, repos, upstream_instance, system_name)
elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
-
+ return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 5a79797ce924af10d1eda003fd55cf78661314a0..2ec22705240ab549ba59229a503f00be8091b02a 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -1,6 +1,8 @@
+from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
from esd.spack_manager.SpackManager import SpackManager
from esd.logger.logger_builder import get_logger
+from esd.utils.utils import run_command
class SpackManagerScratch(SpackManager):
@@ -9,7 +11,18 @@ class SpackManagerScratch(SpackManager):
super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
def concretize_spack_env(self, force=True):
- pass
+ force = '--force' if force else ''
+ if self.spack_env_exists():
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Concertization step for {self.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.env.env_name}',
+ exception=SpackConcertizeException)
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
pass