Skip to content
Snippets Groups Projects
Commit 81cc4cdb authored by Adrian Ciu's avatar Adrian Ciu
Browse files

esd-spack-installation: fixing tests

parent 76673b42
No related branches found
No related tags found
1 merge request!4feat(spack_operation): implement setup_spack_env functionality
import os import os
import subprocess
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from pathlib import Path from pathlib import Path
...@@ -26,11 +25,17 @@ class SpackManager(ABC): ...@@ -26,11 +25,17 @@ class SpackManager(ABC):
def __init__(self, env: SpackModel = None, repos=None, def __init__(self, env: SpackModel = None, repos=None,
upstream_instance=None, system_name: str = None, logger=get_logger(__name__)): upstream_instance=None, system_name: str = None, logger=get_logger(__name__)):
if repos is None: if repos is None:
self.repos = list() repos = []
self.repos = repos
self.env = env self.env = env
self.upstream_instance = upstream_instance self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()).resolve()
self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd())
self.install_dir.mkdir(parents=True, exist_ok=True) self.install_dir.mkdir(parents=True, exist_ok=True)
self.env_path = None
if self.env and self.env.path:
self.env.path = self.env.path.resolve()
self.env.path.mkdir(parents=True, exist_ok=True)
self.env_path = self.env.path / self.env.env_name
self.upstream_instance = upstream_instance
self.spack_dir = self.install_dir / "spack" self.spack_dir = self.install_dir / "spack"
self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh" self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh"
self.logger = logger self.logger = logger
...@@ -45,23 +50,17 @@ class SpackManager(ABC): ...@@ -45,23 +50,17 @@ class SpackManager(ABC):
pass pass
def create_fetch_spack_environment(self): def create_fetch_spack_environment(self):
env_dir = self.install_dir / self.env.path / self.env.env_name
if self.env.git_path: if self.env.git_path:
try: git_clone_repo(self.env.env_name, self.env.path / self.env.env_name, self.env.git_path, logger=self.logger)
git_clone_repo(self.env.env_name, env_dir, self.env.git_path, logger=self.logger)
except subprocess.CalledProcessError as e:
self.logger.exception(f'Failed to clone repository: {self.env.env_name}: {e}')
raise BashCommandException(f'Failed to clone repository: {self.env.env_name}: {e}')
else: else:
try: os.makedirs(self.env.path / self.env.env_name, exist_ok=True)
os.makedirs(self.env.path / self.env.env_name, exist_ok=True) run_command("bash", "-c",
run_command("bash", "-c", f'source {self.spack_setup_script} && spack env create -d {self.env_path}',
f'source {self.spack_setup_script} && spack env create -d {self.env.path}/{self.env.env_name}', check=True, logger=self.logger,
check=True, logger=self.logger) debug_msg=f"Created {self.env.env_name} spack environment",
self.logger.debug(f"Created {self.env.env_name} spack environment") exception_msg=f"Failed to create {self.env.env_name} spack environment",
except subprocess.CalledProcessError as e: exception=BashCommandException)
self.logger.error(f"Failed to create {self.env.env_name} spack environment")
raise BashCommandException(f"Failed to create {self.env.env_name} spack environment")
def setup_spack_env(self): def setup_spack_env(self):
""" """
...@@ -94,46 +93,51 @@ class SpackManager(ABC): ...@@ -94,46 +93,51 @@ class SpackManager(ABC):
def spack_repo_exists(self, repo_name: str) -> bool: def spack_repo_exists(self, repo_name: str) -> bool:
"""Check if the given Spack repository exists.""" """Check if the given Spack repository exists."""
if self.env is None: if self.env is None:
try: result = run_command("bash", "-c",
result = run_command("bash", "-c", f'source {self.spack_setup_script} && spack repo list',
f'source {self.spack_setup_script} && spack repo list', check=True,
check=True, capture_output=True, text=True, logger=self.logger,
capture_output=True, text=True, logger=self.logger) debug_msg=f'Checking if {repo_name} exists')
except subprocess.CalledProcessError: if result is None:
return False return False
else: else:
try: result = run_command("bash", "-c",
result = run_command("bash", "-c", f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo list',
f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo list', check=True,
check=True, capture_output=True, text=True, logger=self.logger,
capture_output=True, text=True, logger=self.logger) debug_msg=f'Checking if repository {repo_name} was added')
except subprocess.CalledProcessError: if result is None:
return False return False
return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines()) return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines())
def spack_env_exists(self):
result = run_command("bash", "-c",
f'source {self.spack_setup_script} && spack env activate -p {self.env_path}',
check=True,
capture_output=True, text=True, logger=self.logger,
debug_msg=f'Checking if environment {self.env.env_name} exists')
if result is None:
return False
return True
def add_spack_repo(self, repo_path: Path, repo_name: str): def add_spack_repo(self, repo_path: Path, repo_name: str):
"""Add the Spack repository if it does not exist.""" """Add the Spack repository if it does not exist."""
repo_path = repo_path.resolve().as_posix() run_command("bash", "-c",
try: f'source {self.spack_setup_script} && spack env activate -p {self.env_path} && spack repo add {repo_path}/{repo_name}',
run_command("bash", "-c", check=True, logger=self.logger,
f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo add {repo_path}/{repo_name}', debug_msg=f"Added {repo_name} to spack environment {self.env.env_name}",
check=True, logger=self.logger) exception_msg=f"Failed to add {repo_name} to spack environment {self.env.env_name}",
self.logger.debug(f"Added {repo_name} to spack environment {self.env.env_name}") exception=BashCommandException)
except subprocess.CalledProcessError as e:
self.logger.error(f"Failed to add {repo_name} to spack environment {self.env.env_name}")
raise BashCommandException(f"Failed to add {repo_name} to spack environment {self.env.env_name}: {e}")
def get_spack_installed_version(self): def get_spack_installed_version(self):
try: spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version',
spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version', capture_output=True, text=True, check=True,
capture_output=True, text=True, check=True, logger=self.logger,
logger=self.logger) debug_msg=f"Getting spack version",
spack_version = spack_version.stdout.strip().split()[0] exception_msg=f"Error retrieving Spack version")
self.logger.debug(f"Getting spack version: {spack_version}") if spack_version:
return spack_version return spack_version.stdout.strip().split()[0]
except subprocess.SubprocessError as e: return None
self.logger.error(f"Error retrieving Spack version: {e}")
return None
def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'): def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'):
try: try:
...@@ -163,8 +167,9 @@ class SpackManager(ABC): ...@@ -163,8 +167,9 @@ class SpackManager(ABC):
bashrc.write(f"source {self.spack_setup_script}\n") bashrc.write(f"source {self.spack_setup_script}\n")
self.logger.info("Added Spack PATH to .bashrc") self.logger.info("Added Spack PATH to .bashrc")
if user: if user:
run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger) run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger,
run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger) debug_msg='Adding permissions to the logged in user')
run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger, debug_msg='Restart bash')
self.logger.info("Spack install completed") self.logger.info("Spack install completed")
# Restart Bash after the installation ends # Restart Bash after the installation ends
os.system("exec bash") os.system("exec bash")
......
...@@ -23,7 +23,7 @@ def test_spack_from_scratch_setup_1(): ...@@ -23,7 +23,7 @@ def test_spack_from_scratch_setup_1():
install_dir = Path('./install').resolve() install_dir = Path('./install').resolve()
env = SpackModel('ebrains-spack-builds', install_dir, env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', ) 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
spack_manager = SpackManagerScratch(env, [env], system_name='ebrainslab') spack_manager = SpackManagerScratch(env=env, repos=[env], system_name='ebrainslab')
spack_manager.setup_spack_env() spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True assert spack_manager.spack_repo_exists(env.env_name) == True
...@@ -33,31 +33,32 @@ def test_spack_from_scratch_setup_2(): ...@@ -33,31 +33,32 @@ def test_spack_from_scratch_setup_2():
env = SpackModel('ebrains-spack-builds', install_dir, env = SpackModel('ebrains-spack-builds', install_dir,
'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', ) 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', )
repo = env repo = env
spack_manager = SpackManagerScratch(env, [repo, repo], system_name='ebrainslab') spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
spack_manager.setup_spack_env() spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True assert spack_manager.spack_repo_exists(env.env_name) == True
def test_spack_from_scratch_setup_3(): def test_spack_from_scratch_setup_3():
install_dir = Path('./install').resolve() install_dir = Path('./install').resolve()
env = SpackModel('new_environment', install_dir, ) env = SpackModel('new_env1', install_dir)
repo = env repo = env
spack_manager = SpackManagerScratch(env, [repo, repo], system_name='ebrainslab') spack_manager = SpackManagerScratch(env=env, repos=[repo, repo], system_name='ebrainslab')
with pytest.raises(BashCommandException): with pytest.raises(BashCommandException):
spack_manager.setup_spack_env() spack_manager.setup_spack_env()
def test_spack_from_scratch_setup_4(): def test_spack_from_scratch_setup_4():
install_dir = Path('./install').resolve() install_dir = Path('./install').resolve()
env = SpackModel('new_environment', install_dir, ) env = SpackModel('new_env2', install_dir)
spack_manager = SpackManagerScratch(env, system_name='ebrainslab') spack_manager = SpackManagerScratch(env=env)
spack_manager.setup_spack_env() spack_manager.setup_spack_env()
assert spack_manager.spack_repo_exists(env.env_name) == True assert spack_manager.spack_env_exists() == True
def test_spack_from_scratch_bash_error(): def test_spack_from_scratch_bash_error():
env = SpackModel('ebrains-spack-builds', Path(), None) env = SpackModel('ebrains-spack-builds', Path(), None)
repo = env repo = env
spack_manager = SpackManagerScratch(env, [repo], system_name='ebrainslab') #
spack_manager = SpackManagerScratch(env=env, repos=[repo], system_name='ebrainslab')
with pytest.raises(BashCommandException): with pytest.raises(BashCommandException):
spack_manager.add_spack_repo(repo.path, repo.env_name) spack_manager.add_spack_repo(repo.path, repo.env_name)
...@@ -3,6 +3,8 @@ import shutil ...@@ -3,6 +3,8 @@ import shutil
import subprocess import subprocess
from pathlib import Path from pathlib import Path
from esd.error_handling.exceptions import BashCommandException
def clean_up(dirs: list[str], logging, ignore_errors=True): def clean_up(dirs: list[str], logging, ignore_errors=True):
""" """
...@@ -22,21 +24,30 @@ def clean_up(dirs: list[str], logging, ignore_errors=True): ...@@ -22,21 +24,30 @@ def clean_up(dirs: list[str], logging, ignore_errors=True):
logging.info(f"{cleanup_dir} does not exist") logging.info(f"{cleanup_dir} does not exist")
def run_command(*args, logger: None, **kwargs): def run_command(*args, logger=logging.getLogger(__name__), debug_msg: str = '', exception_msg: str = None,
if logger is None: exception=None, **kwargs):
logger = logging.getLogger(__name__) try:
logger.debug(f'{args}') logger.debug(f'{debug_msg}: args: {args}')
return subprocess.run(args, **kwargs) return subprocess.run(args, **kwargs)
except subprocess.CalledProcessError as e:
if exception_msg is not None:
logger.error(f"{exception_msg}: {e}")
if exception is not None:
raise exception(f'{exception_msg} : {e}')
else:
return None
def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging): def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = logging.getLogger(__name__)):
if not dir.exists(): if not dir.exists():
run_command( run_command(
"git", "clone", "--depth", "1", "git", "clone", "--depth", "1",
"-c", "advice.detachedHead=false", "-c", "advice.detachedHead=false",
"-c", "feature.manyFiles=true", "-c", "feature.manyFiles=true",
git_path, dir git_path, dir
, check=True, logger=logger) , check=True, logger=logger,
logger.debug(f'Cloned repository {repo_name}') debug_msg=f'Cloned repository {repo_name}',
exception_msg=f'Failed to clone repository: {repo_name}',
exception=BashCommandException)
else: else:
logger.debug(f'Repository {repo_name} already cloned.') logger.debug(f'Repository {repo_name} already cloned.')
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment