diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1affd3bd6b6dcb16a69afcf042712679581da1b2..65062ec73e35ad1daa1f5e88cae077a86e7c9cc0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,5 +40,6 @@ testing-pytest:
paths:
- test-results.xml
- .esd.log
+ - .generate_cache.log
expire_in: 1 week
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index 5d14d1e0a79bacd9a4d72213077130df13a66f2d..65a21d6e224cb7b3c0d4150de1fefa7adbe97b4f 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -1,13 +1,15 @@
import os
import re
+import subprocess
from abc import ABC, abstractmethod
from pathlib import Path
+from tabnanny import check
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
-from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
+from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
from esd.utils.utils import run_command, git_clone_repo
@@ -48,12 +50,7 @@ class SpackManager(ABC):
def concretize_spack_env(self, force=True):
pass
- @abstractmethod
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
-
def create_fetch_spack_environment(self):
-
if self.env.git_path:
git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
else:
@@ -127,7 +124,7 @@ class SpackManager(ABC):
return False
return True
- @no_spack_env
+ @check_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
@@ -137,7 +134,7 @@ class SpackManager(ABC):
exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
exception=BashCommandException)
- @no_spack_env
+ @check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
@@ -167,18 +164,27 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
- @no_spack_env
- def install_packages(self, jobs: int):
- # spack install -v --j "$cpu_count" --fresh
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack install --env {self.env.env_name} -v --j {jobs} --fresh',
- capture_output=True, text=True, check=True,
- logger=self.logger,
- debug_msg=f"Installing spack packages for {self.env.env_name}",
- exception_msg=f"Error installing spack packages for {self.env.env_name}",
- exception=SpackInstallPackagesException)
-
- def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
+ @check_spack_env
+ def install_packages(self, jobs: int, signed=True, fresh=False, debug=False):
+ signed = '' if signed else '--no-check-signature'
+ fresh = '--fresh' if fresh else ''
+ debug = '--debug' if debug else ''
+ install_result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack {debug} install -v {signed} --j {jobs} {fresh}',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ logger=self.logger,
+ debug_msg=f"Installing spack packages for {self.env.env_name}",
+ exception_msg=f"Error installing spack packages for {self.env.env_name}",
+ exception=SpackInstallPackagesException)
+ with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
+ log_file.write(install_result.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(install_result.stderr)
+ return install_result
+
+ def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py
index 38151c6d0741d16c7a06e64f4a2ee5bc9db2e376..5b66f5c51bb6bb3377749998c1e9377fdcd70f94 100644
--- a/esd/spack_manager/factory/SpackManagerBuildCache.py
+++ b/esd/spack_manager/factory/SpackManagerBuildCache.py
@@ -14,6 +14,3 @@ class SpackManagerBuildCache(SpackManager):
def concretize_spack_env(self, force=True):
pass
-
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py
index 2ec22705240ab549ba59229a503f00be8091b02a..3dbc25f68e4bb64204b6901a356b6feaca054c9a 100644
--- a/esd/spack_manager/factory/SpackManagerScratch.py
+++ b/esd/spack_manager/factory/SpackManagerScratch.py
@@ -23,6 +23,3 @@ class SpackManagerScratch(SpackManager):
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
-
- def install_spack_packages(self, jobs: 3, verbose=False, debug=False):
- pass
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/spack_manager/wrapper/spack_wrapper.py
index d075a317289669d00aa07b086ed78a41922f26fd..c2f9c116abb782eae1c22ebad4b8dae073c5acac 100644
--- a/esd/spack_manager/wrapper/spack_wrapper.py
+++ b/esd/spack_manager/wrapper/spack_wrapper.py
@@ -3,7 +3,7 @@ import functools
from esd.error_handling.exceptions import NoSpackEnvironmentException
-def no_spack_env(method):
+def check_spack_env(method):
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
if self.spack_env_exists():
diff --git a/esd/tests/spack_from_scratch_test.py b/esd/tests/spack_from_scratch_test.py
index 984556e5e0b3ea0bca70ecab401f38885a900653..782d6be8f7504ad2116b761b15504290de0663e9 100644
--- a/esd/tests/spack_from_scratch_test.py
+++ b/esd/tests/spack_from_scratch_test.py
@@ -8,6 +8,9 @@ from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
from esd.utils.utils import file_exists_and_not_empty
SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
+test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
+ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
+
def test_spack_repo_exists_1():
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
@@ -32,8 +35,7 @@ def test_spack_repo_exists_3(tmp_path):
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
system_name='ebrainslab')
spack_manager.setup_spack_env()
@@ -42,8 +44,7 @@ def test_spack_from_scratch_setup_1(tmp_path):
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
repos=[repo, repo],
@@ -83,8 +84,7 @@ def test_spack_not_a_valid_repo():
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -96,8 +96,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -109,8 +108,7 @@ def test_spack_from_scratch_concretize_2(tmp_path):
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
- env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
repos=[repo, repo],
@@ -122,8 +120,7 @@ def test_spack_from_scratch_concretize_3(tmp_path):
def test_spack_from_scratch_concretize_4(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=False)
@@ -133,8 +130,7 @@ def test_spack_from_scratch_concretize_4(tmp_path):
def test_spack_from_scratch_concretize_5(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=True)
@@ -144,10 +140,8 @@ def test_spack_from_scratch_concretize_5(tmp_path):
def test_spack_from_scratch_concretize_6(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:"$SPACK_ENV_ACCESS_TOKEN"@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
- repo = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=False)
@@ -157,12 +151,23 @@ def test_spack_from_scratch_concretize_6(tmp_path):
def test_spack_from_scratch_concretize_7(tmp_path):
install_dir = tmp_path
- env = SpackModel('test-spack-env', install_dir,
- f'https://oauth2:"$SPACK_ENV_ACCESS_TOKEN"gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
- repo = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_install(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
+ repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=True)
concretization_file_path = spack_manager.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == True
+ install_result = spack_manager.install_packages(jobs=2, signed=False, fresh=True, debug=False)
+ assert install_result.returncode == 0