From 93d07dbfff3742fcd5fe8f8a7949e288c3a4056c Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 5 Feb 2025 21:32:38 +0200
Subject: [PATCH] esd-spack-installation: added concretization step and tests
---
README.md | 1 +
dedal/tests/spack_from_scratch_test.py | 97 ++++++++++++++-----
dedal/tests/spack_install_test.py | 12 +--
dedal/tests/utils_test.py | 18 ++++
dedal/utils/utils.py | 4 +
esd/error_handling/exceptions.py | 14 ++-
.../factory/SpackManagerCreator.py | 10 +-
.../factory/SpackManagerScratch.py | 15 ++-
8 files changed, 132 insertions(+), 39 deletions(-)
diff --git a/README.md b/README.md
index 62e00c68..86ab9c2f 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,4 @@
# ~~Yashchiki~~Koutakia
For now, this repository provides helpers for the EBRAINS container image build flow.
+The lowest spack version which should be used with this library is v0.22.0
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index cca0d2e9..d1aca83c 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -4,62 +4,70 @@ import pytest
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
+from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
+from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
+from esd.utils.utils import file_exists_and_not_empty
def test_spack_repo_exists_1():
- spack_manager = SpackManagerScratch()
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False
-def test_spack_repo_exists_2():
- install_dir = Path('./install').resolve()
+def test_spack_repo_exists_2(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir)
- spack_manager = SpackManagerScratch(env=env)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
with pytest.raises(NoSpackEnvironmentException):
spack_manager.spack_repo_exists(env.env_name)
-# def test_spack_repo_exists_3():
-# install_dir = Path('./install').resolve()
-# env = SpackModel('ebrains-spack-builds', install_dir)
-# spack_manager = SpackManagerScratch(env=env)
-# spack_manager.setup_spack_env()
-# assert spack_manager.spack_repo_exists(env.env_name) == False
+def test_spack_repo_exists_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ assert spack_manager.spack_repo_exists(env.env_name) == False
-def test_spack_from_scratch_setup_1():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_1(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
- spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
- assert spack_manager.spack_repo_exists(env.env_name) == True
+ assert spack_manager.spack_repo_exists(env.env_name) == False
-def test_spack_from_scratch_setup_2():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_2(tmp_path):
+ install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True
-def test_spack_from_scratch_setup_3():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_3(tmp_path):
+ install_dir = tmp_path
env = SpackModel('new_env1', install_dir)
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
with pytest.raises(BashCommandException):
spack_manager.setup_spack_env()
-def test_spack_from_scratch_setup_4():
- install_dir = Path('./install').resolve()
+def test_spack_from_scratch_setup_4(tmp_path):
+ install_dir = tmp_path
env = SpackModel('new_env2', install_dir)
- spack_manager = SpackManagerScratch(env=env)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
assert spack_manager.spack_env_exists() == True
@@ -67,6 +75,47 @@ def test_spack_from_scratch_setup_4():
def test_spack_not_a_valid_repo():
env = SpackModel('ebrains-spack-builds', Path(), None)
repo = env
- spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo],
+ system_name='ebrainslab')
with pytest.raises(NoSpackEnvironmentException):
spack_manager.add_spack_repo(repo.path, repo.env_name)
+
+
+def test_spack_from_scratch_concretize_1(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_2(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_3(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ repo = env
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
+ spack_manager.setup_spack_env()
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == False
diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py
index 34f68323..9a32d5c7 100644
--- a/dedal/tests/spack_install_test.py
+++ b/dedal/tests/spack_install_test.py
@@ -4,18 +4,18 @@ from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCa
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
-# we need this test to run first so that spack is installed only once
+SPACK_VERSION = "0.22.0"
+
+# we need this test to run first so that spack is installed only once for all the tests
@pytest.mark.run(order=1)
def test_spack_install_scratch():
spack_manager = SpackManagerScratch()
- spack_manager.install_spack(spack_version="v0.21.1")
+ spack_manager.install_spack(spack_version=f'v{SPACK_VERSION}')
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
def test_spack_install_buildcache():
spack_manager = SpackManagerBuildCache()
installed_spack_version = spack_manager.get_spack_installed_version()
- required_version = "0.21.1"
- assert required_version == installed_spack_version
+ assert SPACK_VERSION == installed_spack_version
diff --git a/dedal/tests/utils_test.py b/dedal/tests/utils_test.py
index 14795726..256b8218 100644
--- a/dedal/tests/utils_test.py
+++ b/dedal/tests/utils_test.py
@@ -61,3 +61,21 @@ def test_clean_up_nonexistent_dirs(mocker):
for dir_path in nonexistent_dirs:
mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
+
+
+def test_file_does_not_exist(tmp_path: Path):
+ non_existent_file = tmp_path / "non_existent.txt"
+ assert not file_exists_and_not_empty(non_existent_file)
+
+
+def test_file_exists_but_empty(tmp_path: Path):
+ empty_file = tmp_path / "empty.txt"
+ # Create an empty file
+ empty_file.touch()
+ assert not file_exists_and_not_empty(empty_file)
+
+
+def test_file_exists_and_not_empty(tmp_path: Path):
+ non_empty_file = tmp_path / "non_empty.txt"
+ non_empty_file.write_text("Some content")
+ assert file_exists_and_not_empty(non_empty_file)
diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py
index 48c500c3..173347d0 100644
--- a/dedal/utils/utils.py
+++ b/dedal/utils/utils.py
@@ -51,3 +51,7 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
exception=BashCommandException)
else:
logger.debug(f'Repository {repo_name} already cloned.')
+
+
+def file_exists_and_not_empty(file: Path) -> bool:
+ return file.is_file() and file.stat().st_size > 0
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index 79f8051f..d6de666b 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -7,12 +7,20 @@ class SpackException(Exception):
def __str__(self):
return self.message
+
class BashCommandException(SpackException):
"""
- To be thrown when an invalid input is received.
+ To be thrown when a bash command has failed
"""
+
class NoSpackEnvironmentException(SpackException):
"""
- To be thrown when an invalid input is received.
- """
\ No newline at end of file
+ To be thrown when an operation on a spack environment is executed without the environment being activated or existent
+ """
+
+
+class SpackConcertizeException(SpackException):
+ """
+ To be thrown when the spack concretization step fails
+ """
diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py
index 9728467f..6eb26a04 100644
--- a/esd/spack_manager/factory/SpackManagerCreator.py
+++ b/esd/spack_manager/factory/SpackManagerCreator.py
@@ -1,3 +1,4 @@
+from esd.model.SpackModel import SpackModel
from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache
from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
@@ -5,10 +6,9 @@ from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
class SpackManagerCreator:
@staticmethod
- def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str,
- upstream_instance: str):
+ def get_spack_manger(spack_manager_type: SpackManagerEnum, env: SpackModel = None, repos=None,
+ upstream_instance=None, system_name: str = None):
if spack_manager_type == SpackManagerEnum.FROM_SCRATCH:
- return SpackManagerScratch(env_name, repo, repo_name, upstream_instance)
+ return SpackManagerScratch(env, repos, upstream_instance, system_name)
elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE:
- return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance)
-
+ return SpackManagerBuildCache(env, repos, upstream_instance, system_name)
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 5a79797c..2ec22705 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -1,6 +1,8 @@
+from esd.error_handling.exceptions import SpackConcertizeException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
from esd.spack_manager.SpackManager import SpackManager
from esd.logger.logger_builder import get_logger
+from esd.utils.utils import run_command
class SpackManagerScratch(SpackManager):
@@ -9,7 +11,18 @@ class SpackManagerScratch(SpackManager):
super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__))
def concretize_spack_env(self, force=True):
- pass
+ force = '--force' if force else ''
+ if self.spack_env_exists():
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack concretize {force}',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Concertization step for {self.env.env_name}',
+ exception_msg=f'Failed the concertization step for {self.env.env_name}',
+ exception=SpackConcertizeException)
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
pass
--
GitLab