From d0a57d8fffcb8286c5f4eb220a9590b55002f8b5 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Thu, 6 Feb 2025 10:10:33 +0200
Subject: [PATCH] esd-spack-installation: additional methods
---
.gitlab-ci.yml | 1 +
dedal/tests/spack_from_scratch_test.py | 83 +++++++++++++++++-----
esd/error_handling/exceptions.py | 9 ++-
esd/spack_manager/SpackManager.py | 51 +++++++++----
esd/spack_manager/wrapper/__init__.py | 0
esd/spack_manager/wrapper/spack_wrapper.py | 15 ++++
6 files changed, 124 insertions(+), 35 deletions(-)
create mode 100644 esd/spack_manager/wrapper/__init__.py
create mode 100644 esd/spack_manager/wrapper/spack_wrapper.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4f15b9ab..bd5669bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,6 +6,7 @@ variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
+
build-wheel:
stage: build
tags:
diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py
index d1aca83c..2131e7df 100644
--- a/dedal/tests/spack_from_scratch_test.py
+++ b/dedal/tests/spack_from_scratch_test.py
@@ -1,14 +1,13 @@
+import os
from pathlib import Path
-
import pytest
-
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
from esd.model.SpackModel import SpackModel
from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum
from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
-from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch
from esd.utils.utils import file_exists_and_not_empty
+SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
def test_spack_repo_exists_1():
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
@@ -34,9 +33,9 @@ def test_spack_repo_exists_3(tmp_path):
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- system_name='ebrainslab')
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == False
@@ -44,11 +43,11 @@ def test_spack_from_scratch_setup_1(tmp_path):
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True
@@ -58,8 +57,8 @@ def test_spack_from_scratch_setup_3(tmp_path):
env = SpackModel('new_env1', install_dir)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ repos=[repo, repo],
+ system_name='ebrainslab')
with pytest.raises(BashCommandException):
spack_manager.setup_spack_env()
@@ -76,8 +75,8 @@ def test_spack_not_a_valid_repo():
env = SpackModel('ebrains-spack-builds', Path(), None)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo],
- system_name='ebrainslab')
+ repos=[repo],
+ system_name='ebrainslab')
with pytest.raises(NoSpackEnvironmentException):
spack_manager.add_spack_repo(repo.path, repo.env_name)
@@ -85,7 +84,7 @@ def test_spack_not_a_valid_repo():
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -98,7 +97,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
@@ -111,11 +110,59 @@ def test_spack_from_scratch_concretize_2(tmp_path):
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
- 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
repo = env
- spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
- repos=[repo, repo],
- system_name='ebrainslab')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
+ repos=[repo, repo],
+ system_name='ebrainslab')
spack_manager.setup_spack_env()
concretization_file_path = spack_manager.env_path / 'spack.lock'
assert file_exists_and_not_empty(concretization_file_path) == False
+
+
+def test_spack_from_scratch_concretize_4(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_5(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_6(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ repo = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=False)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
+
+
+def test_spack_from_scratch_concretize_7(tmp_path):
+ install_dir = tmp_path
+ env = SpackModel('test-spack-env', install_dir,
+ 'https://gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
+ repo = SpackModel('ebrains-spack-builds', install_dir,
+ 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
+ spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
+ spack_manager.setup_spack_env()
+ spack_manager.concretize_spack_env(force=True)
+ concretization_file_path = spack_manager.env_path / 'spack.lock'
+ assert file_exists_and_not_empty(concretization_file_path) == True
diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py
index d6de666b..9d11b5fa 100644
--- a/esd/error_handling/exceptions.py
+++ b/esd/error_handling/exceptions.py
@@ -14,13 +14,18 @@ class BashCommandException(SpackException):
"""
-class NoSpackEnvironmentException(SpackException):
+class NoSpackEnvironmentException(BashCommandException):
"""
To be thrown when an operation on a spack environment is executed without the environment being activated or existent
"""
-class SpackConcertizeException(SpackException):
+class SpackConcertizeException(BashCommandException):
"""
To be thrown when the spack concretization step fails
"""
+
+class SpackInstallPackagesException(BashCommandException):
+ """
+ To be thrown when the spack fails to install spack packages
+ """
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index 340b1b95..a7f46c27 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -1,10 +1,13 @@
import os
+import re
from abc import ABC, abstractmethod
from pathlib import Path
-from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException
+from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
+ SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
+from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
from esd.utils.utils import run_command, git_clone_repo
@@ -103,10 +106,10 @@ class SpackManager(ABC):
else:
if self.spack_env_exists():
result = run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
- check=True,
- capture_output=True, text=True, logger=self.logger,
- debug_msg=f'Checking if repository {repo_name} was added')
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
+ check=True,
+ capture_output=True, text=True, logger=self.logger,
+ debug_msg=f'Checking if repository {repo_name} was added')
else:
self.logger.debug('No spack environment defined')
raise NoSpackEnvironmentException('No spack environment defined')
@@ -124,18 +127,35 @@ class SpackManager(ABC):
return False
return True
+ @no_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
- if self.spack_env_exists():
- run_command("bash", "-c",
- f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
- check=True, logger=self.logger,
- debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
- exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
- exception=BashCommandException)
- else:
- self.logger.debug('No spack environment defined')
- raise NoSpackEnvironmentException('No spack environment defined')
+ run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
+ check=True, logger=self.logger,
+ debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
+ exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
+ exception=BashCommandException)
+
+ @no_spack_env
+ def get_compiler_version(self):
+ result = run_command("bash", "-c",
+ f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
+ check=True, logger=self.logger,
+ capture_output=True, text=True,
+ debug_msg=f"Checking spack environment compiler version for {self.env.env_name}",
+ exception_msg=f"Failed to checking spack environment compiler version for {self.env.env_name}",
+ exception=BashCommandException)
+ # todo add error handling and tests
+ if result.stdout is None:
+ self.logger.debug('No gcc found for {self.env.env_name}')
+ return None
+
+ # Find the first occurrence of a GCC compiler using regex
+ match = re.search(r"gcc@([\d\.]+)", result.stdout)
+ gcc_version = match.group(1)
+ self.logger.debug(f'Found gcc for {self.env.env_name}: {gcc_version}')
+ return gcc_version
def get_spack_installed_version(self):
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
@@ -147,6 +167,7 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
+
def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
diff --git a/esd/spack_manager/wrapper/__init__.py b/esd/spack_manager/wrapper/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/spack_manager/wrapper/spack_wrapper.py b/esd/spack_manager/wrapper/spack_wrapper.py
new file mode 100644
index 00000000..d075a317
--- /dev/null
+++ b/esd/spack_manager/wrapper/spack_wrapper.py
@@ -0,0 +1,15 @@
+import functools
+
+from esd.error_handling.exceptions import NoSpackEnvironmentException
+
+
+def no_spack_env(method):
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.spack_env_exists():
+ return method(self, *args, **kwargs) # Call the method with 'self'
+ else:
+ self.logger.debug('No spack environment defined')
+ raise NoSpackEnvironmentException('No spack environment defined')
+
+ return wrapper
--
GitLab