Skip to content
Snippets Groups Projects
Commit 60faa456 authored by Adrian Ciu's avatar Adrian Ciu
Browse files

esd-spack-installation: fixed passing access token to tests; added log file for spack install step

parent b18e16ba
No related branches found
No related tags found
1 merge request!4feat(spack_operation): implement setup_spack_env functionality
Pipeline #58879 passed with stages
in 10 minutes and 10 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 1
− 0
View file @ 60faa456
......@@ -39,5 +39,6 @@ testing-pytest:
paths:
- test-results.xml
- .esd.log
- .generate_cache.log
expire_in: 1 week
esd/spack_manager/SpackManager.py
+ 19
− 6
View file @ 60faa456
......@@ -2,12 +2,13 @@ import os
import re
from abc import ABC, abstractmethod
from pathlib import Path
from tabnanny import check
from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironmentException, \
SpackInstallPackagesException
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
from esd.spack_manager.wrapper.spack_wrapper import no_spack_env
from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
from esd.utils.utils import run_command, git_clone_repo
......@@ -53,7 +54,6 @@ class SpackManager(ABC):
pass
def create_fetch_spack_environment(self):
if self.env.git_path:
git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
else:
......@@ -127,7 +127,7 @@ class SpackManager(ABC):
return False
return True
@no_spack_env
@check_spack_env
def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist."""
run_command("bash", "-c",
......@@ -137,7 +137,7 @@ class SpackManager(ABC):
exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
exception=BashCommandException)
@no_spack_env
@check_spack_env
def get_compiler_version(self):
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack compiler list',
......@@ -167,8 +167,21 @@ class SpackManager(ABC):
return spack_version.stdout.strip().split()[0]
return None
def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
@check_spack_env
def install_packages(self, jobs: int, signed=True, fresh=False):
signed = '' if signed else '--no-check-signature'
fresh = '--fresh' if fresh else ''
with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
run_command("bash", "-c",
f'source {self.spack_setup_script} && spack install --env {self.env.env_name} -v {signed} --j {jobs} {fresh}',
stdout=log_file,
capture_output=True, text=True, check=True,
logger=self.logger,
debug_msg=f"Installing spack packages for {self.env.env_name}",
exception_msg=f"Error installing spack packages for {self.env.env_name}",
exception=SpackInstallPackagesException)
def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
try:
user = os.getlogin()
except OSError:
......
esd/spack_manager/wrapper/spack_wrapper.py
+ 1
− 1
View file @ 60faa456
......@@ -3,7 +3,7 @@ import functools
from esd.error_handling.exceptions import NoSpackEnvironmentException
def no_spack_env(method):
def check_spack_env(method):
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
if self.spack_env_exists():
......
esd/tests/spack_from_scratch_test.py
+ 15
− 22
View file @ 60faa456
......@@ -8,6 +8,10 @@ from esd.spack_manager.factory.SpackManagerCreator import SpackManagerCreator
from esd.utils.utils import file_exists_and_not_empty
SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
def test_spack_repo_exists_1():
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH)
......@@ -32,8 +36,7 @@ def test_spack_repo_exists_3(tmp_path):
def test_spack_from_scratch_setup_1(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
system_name='ebrainslab')
spack_manager.setup_spack_env()
......@@ -42,8 +45,7 @@ def test_spack_from_scratch_setup_1(tmp_path):
def test_spack_from_scratch_setup_2(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
repos=[repo, repo],
......@@ -83,8 +85,7 @@ def test_spack_not_a_valid_repo():
def test_spack_from_scratch_concretize_1(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
......@@ -96,8 +97,7 @@ def test_spack_from_scratch_concretize_1(tmp_path):
def test_spack_from_scratch_concretize_2(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo, repo],
system_name='ebrainslab')
......@@ -109,8 +109,7 @@ def test_spack_from_scratch_concretize_2(tmp_path):
def test_spack_from_scratch_concretize_3(tmp_path):
install_dir = tmp_path
env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
repo = env
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env,
repos=[repo, repo],
......@@ -122,8 +121,7 @@ def test_spack_from_scratch_concretize_3(tmp_path):
def test_spack_from_scratch_concretize_4(tmp_path):
install_dir = tmp_path
env = SpackModel('test-spack-env', install_dir,
f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=False)
......@@ -133,8 +131,7 @@ def test_spack_from_scratch_concretize_4(tmp_path):
def test_spack_from_scratch_concretize_5(tmp_path):
install_dir = tmp_path
env = SpackModel('test-spack-env', install_dir,
f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env)
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=True)
......@@ -144,10 +141,8 @@ def test_spack_from_scratch_concretize_5(tmp_path):
def test_spack_from_scratch_concretize_6(tmp_path):
install_dir = tmp_path
env = SpackModel('test-spack-env', install_dir,
f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
repo = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=False)
......@@ -157,10 +152,8 @@ def test_spack_from_scratch_concretize_6(tmp_path):
def test_spack_from_scratch_concretize_7(tmp_path):
install_dir = tmp_path
env = SpackModel('test-spack-env', install_dir,
'https://gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/test-spack-env.git')
repo = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git')
env = SpackModel('test-spack-env', install_dir, test_spack_env_git)
repo = SpackModel('ebrains-spack-builds', install_dir, ebrains_spack_builds_git)
spack_manager = SpackManagerCreator.get_spack_manger(SpackManagerEnum.FROM_SCRATCH, env=env, repos=[repo])
spack_manager.setup_spack_env()
spack_manager.concretize_spack_env(force=True)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment