diff --git a/dedal/docs/resources/dedal_UML.png b/dedal/docs/resources/dedal_UML.png
index 6e970d08690dc6f53f95c5088ad3933b9bae6f15..65d9f15828d9c6e19f8bffee41d5db3e02307b3f 100644
Binary files a/dedal/docs/resources/dedal_UML.png and b/dedal/docs/resources/dedal_UML.png differ
diff --git a/dedal/error_handling/exceptions.py b/dedal/error_handling/exceptions.py
index 85398df060dc8248372d273c2f6f35d65cf16696..055768f82555a38b16ce87f96d5d441b11b6be45 100644
--- a/dedal/error_handling/exceptions.py
+++ b/dedal/error_handling/exceptions.py
@@ -66,3 +66,8 @@ class SpackConfigException(BashCommandException):
"""
To be thrown when the spack config command fails
"""
+
+class SpackFindException(BashCommandException):
+ """
+ To be thrown when the spack find command fails
+ """
\ No newline at end of file
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
index 8eaf3ef54a584147a81377ac5d0f20f9aedf97e0..e190b6ef6e6742a4cdca237cfff887bec7031e30 100644
--- a/dedal/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -6,7 +6,7 @@ from dedal.configuration.SpackConfig import SpackConfig
from dedal.enum.SpackConfigCommand import SpackConfigCommand
from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
SpackInstallPackagesException, SpackConcertizeException, SpackMirrorException, SpackGpgException, \
- SpackRepoException, SpackReindexException, SpackSpecException, SpackConfigException
+ SpackRepoException, SpackReindexException, SpackSpecException, SpackConfigException, SpackFindException
from dedal.logger.logger_builder import get_logger
from dedal.tests.testing_variables import SPACK_VERSION
from dedal.utils.utils import run_command, git_clone_repo, log_command, set_bashrc_variable, get_first_word
@@ -363,6 +363,33 @@ class SpackOperation:
log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
+ @check_spack_env
+ def find_packages(self):
+ """Returns a dictionary of installed Spack packages in the current environment.
+ Each key is the name of a Spack package, and the corresponding value is a list of
+ installed versions for that package.
+ Raises:
+ NoSpackEnvironmentException: If the spack environment is not set up.
+ """
+ packages = run_command("bash", "-c",
+ f'{self.spack_command_on_env} && spack find -c',
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ info_msg=f'Listing installed packages.',
+ exception_msg=f'Failed to list installed packages',
+ exception=SpackFindException).stdout
+ dict_packages = {}
+ for package in packages.strip().splitlines():
+ if package.startswith('[+]'):
+ package = package.replace('@', ' ').split()
+ if len(package) == 3:
+ _, name, version = package
+ dict_packages.setdefault(name, []).append(version)
+ return dict_packages
+
def install_spack(self, spack_version=f'{SPACK_VERSION}', spack_repo='https://github.com/spack/spack',
bashrc_path=os.path.expanduser("~/.bashrc")):
"""Install spack.
diff --git a/dedal/tests/integration_tests/helper.py b/dedal/tests/integration_tests/helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..58f9e0b1968e7f83afb289aa6bc3df050a809e97
--- /dev/null
+++ b/dedal/tests/integration_tests/helper.py
@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from dedal.spack_factory.SpackOperation import SpackOperation
+from dedal.utils.spack_utils import extract_spack_packages
+from dedal.utils.variables import test_spack_env_name
+
+
+def check_installed_spack_packages(spack_operation: SpackOperation, install_dir: Path):
+ to_install_spack_packages = extract_spack_packages(str(install_dir / test_spack_env_name / 'spack.yaml'))
+ installed_spack_packages = list(spack_operation.find_packages().keys())
+ for spack_pacakge in to_install_spack_packages:
+ assert spack_pacakge in installed_spack_packages
diff --git a/dedal/tests/integration_tests/spack_create_cache_test.py b/dedal/tests/integration_tests/spack_create_cache_test.py
index 2ae70502e0db3548ba08ded83645bec7957ea773..1da51128a873222796753bfe2d81cbd5506777f6 100644
--- a/dedal/tests/integration_tests/spack_create_cache_test.py
+++ b/dedal/tests/integration_tests/spack_create_cache_test.py
@@ -8,6 +8,7 @@ from dedal.configuration.SpackConfig import SpackConfig
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.tests.integration_tests.helper import check_installed_spack_packages
from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
"""
@@ -43,3 +44,4 @@ def test_spack_create_cache_installation(tmp_path):
spack_operation = test_spack_create_cache_concretization(tmp_path)
spack_operation.install_packages()
assert len(spack_operation.build_cache.list_tags()) > 0
+ check_installed_spack_packages(spack_operation, tmp_path)
diff --git a/dedal/tests/integration_tests/spack_from_cache_test.py b/dedal/tests/integration_tests/spack_from_cache_test.py
index 42ecf3b7bf52c478299e41f270fa0d58000f5ec6..206fdbfeaf536bf95802c1c5864883219b634899 100644
--- a/dedal/tests/integration_tests/spack_from_cache_test.py
+++ b/dedal/tests/integration_tests/spack_from_cache_test.py
@@ -4,12 +4,13 @@ from dedal.configuration.SpackConfig import SpackConfig
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache
+from dedal.tests.integration_tests.helper import check_installed_spack_packages
from dedal.utils.utils import file_exists_and_not_empty, count_files_in_folder
from dedal.utils.variables import test_spack_env_git, ebrains_spack_builds_git
-def test_spack_from_cache_setup(tmp_path):
- install_dir = tmp_path
+def setup(path):
+ install_dir = path
env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
concretization_dir = install_dir / 'concretize'
@@ -18,6 +19,11 @@ def test_spack_from_cache_setup(tmp_path):
buildcache_dir=buildcache_dir)
spack_config.add_repo(repo)
spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+ return spack_operation, concretization_dir, buildcache_dir
+
+
+def test_spack_from_cache_setup(tmp_path):
+ spack_operation, concretization_dir, buildcache_dir = setup(tmp_path)
assert isinstance(spack_operation, SpackOperationUseCache)
spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
spack_operation.setup_spack_env()
@@ -26,24 +32,54 @@ def test_spack_from_cache_setup(tmp_path):
assert file_exists_and_not_empty(concretization_download_file_path) == True
assert count_files_in_folder(buildcache_dir) == num_tags
assert 'local_cache' in spack_operation.mirror_list()
- return spack_operation
def test_spack_from_cache_concretize(tmp_path):
- spack_operation = test_spack_from_cache_setup(tmp_path)
+ spack_operation, concretization_dir, buildcache_dir = setup(tmp_path)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
+ spack_operation.setup_spack_env()
+ num_tags = len(spack_operation.build_cache.list_tags())
+ concretization_download_file_path = concretization_dir / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_download_file_path) == True
+ assert count_files_in_folder(buildcache_dir) == num_tags
+ assert 'local_cache' in spack_operation.mirror_list()
assert spack_operation.concretize_spack_env() == False
concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
- return spack_operation
def test_spack_from_cache_install_1(tmp_path):
- spack_operation = test_spack_from_cache_concretize(tmp_path)
+ spack_operation, concretization_dir, buildcache_dir = setup(tmp_path)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
+ spack_operation.setup_spack_env()
+ num_tags = len(spack_operation.build_cache.list_tags())
+ concretization_download_file_path = concretization_dir / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_download_file_path) == True
+ assert count_files_in_folder(buildcache_dir) == num_tags
+ assert 'local_cache' in spack_operation.mirror_list()
+ assert spack_operation.concretize_spack_env() == False
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False)
assert install_result.returncode == 0
+ check_installed_spack_packages(spack_operation, tmp_path)
def test_spack_from_cache_install_2(tmp_path):
- spack_operation = test_spack_from_cache_concretize(tmp_path)
+ spack_operation, concretization_dir, buildcache_dir = setup(tmp_path)
+ assert isinstance(spack_operation, SpackOperationUseCache)
+ spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
+ spack_operation.setup_spack_env()
+ num_tags = len(spack_operation.build_cache.list_tags())
+ concretization_download_file_path = concretization_dir / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_download_file_path) == True
+ assert count_files_in_folder(buildcache_dir) == num_tags
+ assert 'local_cache' in spack_operation.mirror_list()
+ assert spack_operation.concretize_spack_env() == False
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False, test='root')
assert install_result.returncode == 0
+ check_installed_spack_packages(spack_operation, tmp_path)
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
index 50b19faab603232f0ebc9d19503b5303e16604ff..bed34df1d7cce9121e9263b92f22f06d6294fe81 100644
--- a/dedal/tests/integration_tests/spack_from_scratch_test.py
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -4,6 +4,7 @@ from dedal.configuration.SpackConfig import SpackConfig
from dedal.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.tests.integration_tests.helper import check_installed_spack_packages
from dedal.tests.testing_variables import test_spack_env_git, ebrains_spack_builds_git
from dedal.utils.utils import file_exists_and_not_empty
from dedal.spack_factory.SpackOperation import SpackOperation
@@ -49,6 +50,7 @@ def test_spack_reindex(tmp_path):
spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
spack_operation.reindex()
+
@pytest.mark.skip(reason="It does nopt work on bare metal operating systems")
def test_spack_spec(tmp_path):
install_dir = tmp_path
@@ -216,19 +218,20 @@ def test_spack_from_scratch_concretize_7(tmp_path):
concretization_file_path = spack_operation.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
- def test_spack_from_scratch_concretize_8(tmp_path):
- install_dir = tmp_path
- env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
- repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
- config = SpackConfig(env=env, install_dir=install_dir)
- config.add_repo(repo)
- spack_operation = SpackOperationCreator.get_spack_operator(config)
- assert isinstance(spack_operation, SpackOperation)
- spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
- spack_operation.setup_spack_env()
- spack_operation.concretize_spack_env(force=True, test='root')
- concretization_file_path = spack_operation.env_path / 'spack.lock'
- assert file_exists_and_not_empty(concretization_file_path) == True
+
+def test_spack_from_scratch_concretize_8(tmp_path):
+ install_dir = tmp_path
+ env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ config = SpackConfig(env=env, install_dir=install_dir)
+ config.add_repo(repo)
+ spack_operation = SpackOperationCreator.get_spack_operator(config)
+ assert isinstance(spack_operation, SpackOperation)
+ spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc')))
+ spack_operation.setup_spack_env()
+ spack_operation.concretize_spack_env(force=True, test='root')
+ concretization_file_path = spack_operation.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
def test_spack_from_scratch_install(tmp_path):
@@ -246,6 +249,7 @@ def test_spack_from_scratch_install(tmp_path):
assert file_exists_and_not_empty(concretization_file_path) == True
install_result = spack_operation.install_packages(jobs=2, signed=False, fresh=True, debug=False)
assert install_result.returncode == 0
+ check_installed_spack_packages(spack_operation, install_dir)
def test_spack_from_scratch_install_2(tmp_path):
@@ -263,6 +267,7 @@ def test_spack_from_scratch_install_2(tmp_path):
assert file_exists_and_not_empty(concretization_file_path) == True
install_result = spack_operation.install_packages(jobs=2, signed=False, fresh=True, debug=False, test='root')
assert install_result.returncode == 0
+ check_installed_spack_packages(spack_operation, install_dir)
def test_spack_mirror_env(tmp_path):
diff --git a/dedal/tests/integration_tests/spack_utils_test.py b/dedal/tests/integration_tests/spack_utils_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0fb09a76f33476b000897f238eacef73a6a8ce7
--- /dev/null
+++ b/dedal/tests/integration_tests/spack_utils_test.py
@@ -0,0 +1,10 @@
+from pathlib import Path
+
+from dedal.utils.spack_utils import extract_spack_packages
+
+
+def test_extract_spack_packages():
+ test_file_path = Path(__file__).parent
+ assert sorted(extract_spack_packages(test_file_path.parent / 'spack_files' / 'spack1.yaml')) == sorted(
+ ['python', 'numpy',
+ 'py-scipy', 'cmake'])
diff --git a/dedal/tests/spack_files/__init__.py b/dedal/tests/spack_files/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dedal/tests/spack_files/spack1.yaml b/dedal/tests/spack_files/spack1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c8a0bbe798063370c45c919bbe044ce5edb87b99
--- /dev/null
+++ b/dedal/tests/spack_files/spack1.yaml
@@ -0,0 +1,11 @@
+spack:
+ view: false
+ specs:
+ - python@3.10.4+optimizations%gcc@11.2.0
+ - numpy+blas
+ - py-scipy@1.9.3 ^openblas threads=openmp
+ - cmake
+ concretizer:
+ unify: true
+ repos: [ ]
+ mirrors: { }
diff --git a/dedal/utils/spack_utils.py b/dedal/utils/spack_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..90566f219b99e2353947fcbd1eedf25a1eae9810
--- /dev/null
+++ b/dedal/utils/spack_utils.py
@@ -0,0 +1,24 @@
+import yaml
+import re
+
+
+def extract_spack_packages(spack_yaml_path: str):
+ """
+ Extracts only the package names from a spack.yaml file, ignoring versions, variants, compilers, and dependencies.
+ Args:
+ spack_yaml_path (str): Path to the spack.yaml file.
+ Returns:
+ List[str]: Clean list of package names.
+ """
+ with open(spack_yaml_path, 'r') as f:
+ data = yaml.safe_load(f)
+ specs = data.get('spack', {}).get('specs', [])
+ package_names = []
+ for spec in specs:
+ if isinstance(spec, str):
+ # Match the first word-like token before any of: @ + % ^ whitespace
+ match = re.match(r'^([\w-]+)', spec.strip())
+ if match:
+ package_names.append(match.group(1))
+
+ return package_names
diff --git a/dedal/utils/variables.py b/dedal/utils/variables.py
index 553ccf97992ca2dcd06970f82fdbbb26f5f1db23..63dd6d637c2d263e82890623e39e5bc8e565eb39 100644
--- a/dedal/utils/variables.py
+++ b/dedal/utils/variables.py
@@ -2,4 +2,5 @@ import os
SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
+test_spack_env_name = 'test-spack-env'
ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
diff --git a/pyproject.toml b/pyproject.toml
index 7b82909769b161d8777cadd560899ba827cea941..1885e60e88421cd8572defa4f20ef062c88f49bc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,6 +19,7 @@ dependencies = [
"ruamel.yaml",
"click",
"jsonpickle",
+ "pyyaml",
]
[project.scripts]