diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7cdc2157784919724ef7e210b7f1bf9789334397..4f15b9ab2407e9d56f56ed2acb0abceab1dddf9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,11 @@ stages: - - build - test + - build variables: BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest + build-wheel: stage: build tags: @@ -21,18 +22,23 @@ build-wheel: expire_in: 1 week -testing: +testing-pytest: stage: test tags: - docker-runner - image: python:latest + image: ubuntu:22.04 script: - - pip install -e . - - pytest ./dedal/tests/ --junitxml=test-results.xml + - chmod +x dedal/utils/bootstrap.sh + - ./dedal/utils/bootstrap.sh + - pip install . + - pytest ./dedal/tests/ -s --junitxml=test-results.xml artifacts: when: always reports: junit: test-results.xml paths: - test-results.xml - expire_in: 1 week \ No newline at end of file + - .dedal.log + - .generate_cache.log + expire_in: 1 week + diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py index 2da39e252c8ebc3e0b9aa6d6a5612d9f1f4ee02e..dbd50bf9132d546afbfea5d3fd4f729ba267cb70 100644 --- a/dedal/build_cache/BuildCacheManager.py +++ b/dedal/build_cache/BuildCacheManager.py @@ -40,8 +40,8 @@ class BuildCacheManager(BuildCacheManagerInterface): for sub_path in build_cache_path.rglob("*"): if sub_path.is_file(): - rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "") - target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}" + rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "") + target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}" try: self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...") self.client.push( @@ -51,7 +51,7 @@ class BuildCacheManager(BuildCacheManagerInterface): manifest_annotations={"path": rel_path}, disable_path_validation=True, ) - self.logger.info(f"Successfully pushed {sub_path.name}") + self.logger.info(f"Successfully pushed {sub_path.env_name}") except Exception as e: self.logger.error( f"An error occurred while pushing: {e}") diff --git a/dedal/specfile_storage_path_source.py b/dedal/specfile_storage_path_source.py index 6e8a8889e809c0d8c5116670e19bd7fbd420904c..4d2ff65888aa6dcce2f2a9526575b85d8966d90d 100644 --- a/dedal/specfile_storage_path_source.py +++ b/dedal/specfile_storage_path_source.py @@ -49,14 +49,14 @@ for rspec in data: format_string = "{name}-{version}" pretty_name = pkg.spec.format_path(format_string) - cosmetic_path = os.path.join(pkg.name, pretty_name) + cosmetic_path = os.path.join(pkg.env_name, pretty_name) to_be_fetched.add(str(spack.mirror.mirror_archive_paths(pkg.fetcher, cosmetic_path).storage_path)) for resource in pkg._get_needed_resources(): - pretty_resource_name = fsys.polite_filename(f"{resource.name}-{pkg.version}") + pretty_resource_name = fsys.polite_filename(f"{resource.env_name}-{pkg.version}") to_be_fetched.add(str(spack.mirror.mirror_archive_paths(resource.fetcher, pretty_resource_name).storage_path)) for patch in ss.patches: if isinstance(patch, spack.patch.UrlPatch): - to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.name).storage_path)) + to_be_fetched.add(str(spack.mirror.mirror_archive_paths(patch.stage.fetcher, patch.stage.env_name).storage_path)) for elem in to_be_fetched: print(elem) diff --git a/dedal/tests/spack_from_scratch_test.py b/dedal/tests/spack_from_scratch_test.py new file mode 100644 index 0000000000000000000000000000000000000000..4981b760312ff39c81fd15b78c66b6bd76999d41 --- /dev/null +++ b/dedal/tests/spack_from_scratch_test.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import pytest + +from esd.error_handling.exceptions import BashCommandException +from esd.model.SpackModel import SpackModel +from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch + + +def test_spack_repo_exists_1(): + install_dir = Path('./install').resolve() + env = SpackModel('ebrains-spack-builds', install_dir) + spack_manager = SpackManagerScratch(env=env) + assert spack_manager.spack_repo_exists(env.env_name) == False + + +def test_spack_repo_exists_2(): + spack_manager = SpackManagerScratch() + assert spack_manager.spack_repo_exists('ebrains-spack-builds') == False + + +def test_spack_from_scratch_setup_1(): + install_dir = Path('./install').resolve() + env = SpackModel('ebrains-spack-builds', install_dir, + 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', ) + spack_manager = SpackManagerScratch(env, [env], system_name='ebrainslab') + spack_manager.setup_spack_env() + assert spack_manager.spack_repo_exists(env.env_name) == True + + +def test_spack_from_scratch_setup_2(): + install_dir = Path('./install').resolve() + env = SpackModel('ebrains-spack-builds', install_dir, + 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git', ) + repo = env + spack_manager = SpackManagerScratch(env, [repo, repo], system_name='ebrainslab') + spack_manager.setup_spack_env() + assert spack_manager.spack_repo_exists(env.env_name) == True + + +def test_spack_from_scratch_setup_3(): + install_dir = Path('./install').resolve() + env = SpackModel('new_environment', install_dir, ) + repo = env + spack_manager = SpackManagerScratch(env, [repo, repo], system_name='ebrainslab') + with pytest.raises(BashCommandException): + spack_manager.setup_spack_env() + + +def test_spack_from_scratch_setup_4(): + install_dir = Path('./install').resolve() + env = SpackModel('new_environment', install_dir, ) + spack_manager = SpackManagerScratch(env, system_name='ebrainslab') + spack_manager.setup_spack_env() + assert spack_manager.spack_repo_exists(env.env_name) == True + + +def test_spack_from_scratch_bash_error(): + env = SpackModel('ebrains-spack-builds', Path(), None) + repo = env + spack_manager = SpackManagerScratch(env, [repo], system_name='ebrainslab') + with pytest.raises(BashCommandException): + spack_manager.add_spack_repo(repo.path, repo.env_name) diff --git a/dedal/tests/spack_install_test.py b/dedal/tests/spack_install_test.py new file mode 100644 index 0000000000000000000000000000000000000000..34f683234b064d0f800bfa9376fcf6fec592d7ef --- /dev/null +++ b/dedal/tests/spack_install_test.py @@ -0,0 +1,21 @@ +import pytest + +from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache +from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch + + +# we need this test to run first so that spack is installed only once +@pytest.mark.run(order=1) +def test_spack_install_scratch(): + spack_manager = SpackManagerScratch() + spack_manager.install_spack(spack_version="v0.21.1") + installed_spack_version = spack_manager.get_spack_installed_version() + required_version = "0.21.1" + assert required_version == installed_spack_version + + +def test_spack_install_buildcache(): + spack_manager = SpackManagerBuildCache() + installed_spack_version = spack_manager.get_spack_installed_version() + required_version = "0.21.1" + assert required_version == installed_spack_version diff --git a/dedal/utils/bootstrap.sh b/dedal/utils/bootstrap.sh new file mode 100644 index 0000000000000000000000000000000000000000..9b7d0131e95a3be9b0f2cfc4dc82492517fb22dc --- /dev/null +++ b/dedal/utils/bootstrap.sh @@ -0,0 +1,6 @@ +# Minimal prerequisites for installing the esd_library +# pip must be installed on the OS +echo "Bootstrapping..." +apt update +apt install -y bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 python3-pip tar unzip xz-utils zstd +python3 -m pip install --upgrade pip setuptools wheel diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py index 811d258e7e5856f4b666bc3196996f3b24571112..d229e3453cb3ba4b29319ec84eeeb68998851447 100644 --- a/dedal/utils/utils.py +++ b/dedal/utils/utils.py @@ -1,4 +1,6 @@ +import logging import shutil +import subprocess from pathlib import Path @@ -18,3 +20,23 @@ def clean_up(dirs: list[str], logging, ignore_errors=True): raise e else: logging.info(f"{cleanup_dir} does not exist") + + +def run_command(*args, logger: None, **kwargs): + if logger is None: + logger = logging.getLogger(__name__) + logger.debug(f'{args}') + return subprocess.run(args, **kwargs) + + +def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging): + if not dir.exists(): + run_command( + "git", "clone", "--depth", "1", + "-c", "advice.detachedHead=false", + "-c", "feature.manyFiles=true", + git_path, dir + , check=True, logger=logger) + logger.debug(f'Cloned repository {repo_name}') + else: + logger.debug(f'Repository {repo_name} already cloned.') diff --git a/esd/error_handling/exceptions.py b/esd/error_handling/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..2acca54ea6decc66da1bd0cc827ea5b890b2780d --- /dev/null +++ b/esd/error_handling/exceptions.py @@ -0,0 +1,13 @@ +class SpackException(Exception): + + def __init__(self, message): + super().__init__(message) + self.message = str(message) + + def __str__(self): + return self.message + +class BashCommandException(SpackException): + """ + To be thrown when an invalid input is received. + """ diff --git a/esd/model/SpackModel.py b/esd/model/SpackModel.py new file mode 100644 index 0000000000000000000000000000000000000000..4b065dba06f558ce21a0354257d77aa595bcaeb1 --- /dev/null +++ b/esd/model/SpackModel.py @@ -0,0 +1,12 @@ +from pathlib import Path + + +class SpackModel: + """" + Provides details about the spack environment + """ + + def __init__(self, env_name: str, path: Path, git_path: str = None): + self.env_name = env_name + self.path = path + self.git_path = git_path diff --git a/esd/model/__init__.py b/esd/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py new file mode 100644 index 0000000000000000000000000000000000000000..cdf4a1d36d249017636798b05349a88b5e67f97e --- /dev/null +++ b/esd/spack_manager/SpackManager.py @@ -0,0 +1,180 @@ +import os +import subprocess +from abc import ABC, abstractmethod +from pathlib import Path + +from esd.error_handling.exceptions import BashCommandException +from esd.logger.logger_builder import get_logger +from esd.model.SpackModel import SpackModel +from esd.utils.utils import run_command, git_clone_repo + + +class SpackManager(ABC): + """ + This class should implement the methods necessary for installing spack, set up an environment, concretize and install packages. + Factory design pattern is used because there are 2 cases: creating an environment from scratch or creating an environment from the buildcache. + + Attributes: + ----------- + env : SpackModel + spack environment details + repos : list[SpackModel] + upstream_instance : str + path to Spack instance to use as upstream (optional) + """ + + def __init__(self, env: SpackModel = None, repos=None, + upstream_instance=None, system_name: str = None, logger=get_logger(__name__)): + if repos is None: + self.repos = list() + self.env = env + self.upstream_instance = upstream_instance + self.install_dir = Path(os.environ.get("INSTALLATION_ROOT") or os.getcwd()) + self.install_dir.mkdir(parents=True, exist_ok=True) + self.spack_dir = self.install_dir / "spack" + self.spack_setup_script = self.spack_dir / "share" / "spack" / "setup-env.sh" + self.logger = logger + self.system_name = system_name + + @abstractmethod + def concretize_spack_env(self, force=True): + pass + + @abstractmethod + def install_spack_packages(self, jobs: 3, verbose=False, debug=False): + pass + + def create_fetch_spack_environment(self): + env_dir = self.install_dir / self.env.path / self.env.env_name + if self.env.git_path: + try: + git_clone_repo(self.env.env_name, env_dir, self.env.git_path, logger=self.logger) + except subprocess.CalledProcessError as e: + self.logger.exception(f'Failed to clone repository: {self.env.env_name}: {e}') + raise BashCommandException(f'Failed to clone repository: {self.env.env_name}: {e}') + else: + try: + os.makedirs(self.env.path / self.env.env_name, exist_ok=True) + run_command("bash", "-c", + f'source {self.spack_setup_script} && spack env create -d {self.env.path}/{self.env.env_name}', + check=True, logger=self.logger) + self.logger.debug(f"Created {self.env.env_name} spack environment") + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to create {self.env.env_name} spack environment") + raise BashCommandException(f"Failed to create {self.env.env_name} spack environment") + + def setup_spack_env(self): + """ + This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos + """ + bashrc_path = os.path.expanduser("~/.bashrc") + if self.system_name: + with open(bashrc_path, "a") as bashrc: + bashrc.write(f'export SYSTEMNAME="{self.system_name}"\n') + os.environ['SYSTEMNAME'] = self.system_name + if self.spack_dir.exists() and self.spack_dir.is_dir(): + with open(bashrc_path, "a") as bashrc: + bashrc.write(f'export SPACK_USER_CACHE_PATH="{str(self.spack_dir / ".spack")}"\n') + bashrc.write(f'export SPACK_USER_CONFIG_PATH="{str(self.spack_dir / ".spack")}"\n') + self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH') + else: + self.logger.error(f'Invalid installation path: {self.spack_dir}') + # Restart the bash after adding environment variables + self.create_fetch_spack_environment() + if self.install_dir.exists(): + for repo in self.repos: + repo_dir = self.install_dir / repo.path / repo.env_name + git_clone_repo(repo.env_name, repo_dir, repo.git_path, logger=self.logger) + if not self.spack_repo_exists(repo.env_name): + self.add_spack_repo(repo.path, repo.env_name) + self.logger.debug(f'Added spack repository {repo.env_name}') + else: + self.logger.debug(f'Spack repository {repo.env_name} already added') + + def spack_repo_exists(self, repo_name: str) -> bool: + """Check if the given Spack repository exists.""" + if self.env is None: + try: + result = run_command("bash", "-c", + f'source {self.spack_setup_script} && spack repo list', + check=True, + capture_output=True, text=True, logger=self.logger) + except subprocess.CalledProcessError: + return False + else: + try: + result = run_command("bash", "-c", + f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo list', + check=True, + capture_output=True, text=True, logger=self.logger) + except subprocess.CalledProcessError: + return False + return any(line.strip().endswith(repo_name) for line in result.stdout.splitlines()) + + def add_spack_repo(self, repo_path: Path, repo_name: str): + """Add the Spack repository if it does not exist.""" + repo_path = repo_path.resolve().as_posix() + try: + run_command("bash", "-c", + f'source {self.spack_setup_script} && spack env activate -p {self.env.path}/{self.env.env_name} && spack repo add {repo_path}/{repo_name}', + check=True, logger=self.logger) + self.logger.debug(f"Added {repo_name} to spack environment {self.env.env_name}") + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to add {repo_name} to spack environment {self.env.env_name}") + raise BashCommandException(f"Failed to add {repo_name} to spack environment {self.env.env_name}: {e}") + + def get_spack_installed_version(self): + try: + spack_version = run_command("bash", "-c", f'source {self.spack_setup_script} && spack --version', + capture_output=True, text=True, check=True, + logger=self.logger) + spack_version = spack_version.stdout.strip().split()[0] + self.logger.debug(f"Getting spack version: {spack_version}") + return spack_version + except subprocess.SubprocessError as e: + self.logger.error(f"Error retrieving Spack version: {e}") + return None + + def install_spack(self, spack_version="v0.21.1", spack_repo='https://github.com/spack/spack'): + try: + user = os.getlogin() + except OSError: + user = None + + self.logger.info(f"Starting to install Spack into {self.spack_dir} from branch {spack_version}") + if not self.spack_dir.exists(): + run_command( + "git", "clone", "--depth", "1", + "-c", "advice.detachedHead=false", + "-c", "feature.manyFiles=true", + "--branch", spack_version, spack_repo, self.spack_dir + , check=True, logger=self.logger) + self.logger.debug("Cloned spack") + else: + self.logger.debug("Spack already cloned.") + + bashrc_path = os.path.expanduser("~/.bashrc") + # ensure the file exists before opening it + if not os.path.exists(bashrc_path): + open(bashrc_path, "w").close() + # add spack setup commands to .bashrc + with open(bashrc_path, "a") as bashrc: + bashrc.write(f'export PATH="{self.spack_dir}/bin:$PATH"\n') + bashrc.write(f"source {self.spack_setup_script}\n") + self.logger.info("Added Spack PATH to .bashrc") + if user: + run_command("chown", "-R", f"{user}:{user}", self.spack_dir, check=True, logger=self.logger) + run_command("bash", "-c", f"source {bashrc_path}", check=True, logger=self.logger) + self.logger.info("Spack install completed") + # Restart Bash after the installation ends + os.system("exec bash") + + # Configure upstream Spack instance if specified + if self.upstream_instance: + upstreams_yaml_path = os.path.join(self.spack_dir, "etc/spack/defaults/upstreams.yaml") + with open(upstreams_yaml_path, "w") as file: + file.write(f"""upstreams: + upstream-spack-instance: + install_tree: {self.upstream_instance}/spack/opt/spack + """) + self.logger.info("Added upstream spack instance") diff --git a/esd/spack_manager/__init__.py b/esd/spack_manager/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esd/spack_manager/enums/SpackManagerEnum.py b/esd/spack_manager/enums/SpackManagerEnum.py new file mode 100644 index 0000000000000000000000000000000000000000..a24358394d19ee1903835f7eafea8f8e8c964fa6 --- /dev/null +++ b/esd/spack_manager/enums/SpackManagerEnum.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class SpackManagerEnum(Enum): + FROM_SCRATCH = "from_scratch", + FROM_BUILDCACHE = "from_buildcache", diff --git a/esd/spack_manager/enums/__init__.py b/esd/spack_manager/enums/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esd/spack_manager/factory/SpackManagerBuildCache.py b/esd/spack_manager/factory/SpackManagerBuildCache.py new file mode 100644 index 0000000000000000000000000000000000000000..38151c6d0741d16c7a06e64f4a2ee5bc9db2e376 --- /dev/null +++ b/esd/spack_manager/factory/SpackManagerBuildCache.py @@ -0,0 +1,19 @@ +from esd.model.SpackModel import SpackModel +from esd.spack_manager.SpackManager import SpackManager +from esd.logger.logger_builder import get_logger + + +class SpackManagerBuildCache(SpackManager): + def __init__(self, env: SpackModel = None, repos=None, + upstream_instance=None, system_name: str = None): + super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__)) + + def setup_spack_env(self): + super().setup_spack_env() + # todo add buildcache to the spack environment + + def concretize_spack_env(self, force=True): + pass + + def install_spack_packages(self, jobs: 3, verbose=False, debug=False): + pass diff --git a/esd/spack_manager/factory/SpackManagerCreator.py b/esd/spack_manager/factory/SpackManagerCreator.py new file mode 100644 index 0000000000000000000000000000000000000000..9728467f39c2753c9f12ef959c80def1150f0314 --- /dev/null +++ b/esd/spack_manager/factory/SpackManagerCreator.py @@ -0,0 +1,14 @@ +from esd.spack_manager.enums.SpackManagerEnum import SpackManagerEnum +from esd.spack_manager.factory.SpackManagerBuildCache import SpackManagerBuildCache +from esd.spack_manager.factory.SpackManagerScratch import SpackManagerScratch + + +class SpackManagerCreator: + @staticmethod + def get_spack_manger(spack_manager_type: SpackManagerEnum, env_name: str, repo: str, repo_name: str, + upstream_instance: str): + if spack_manager_type == SpackManagerEnum.FROM_SCRATCH: + return SpackManagerScratch(env_name, repo, repo_name, upstream_instance) + elif spack_manager_type == SpackManagerEnum.FROM_BUILDCACHE: + return SpackManagerBuildCache(env_name, repo, repo_name, upstream_instance) + diff --git a/esd/spack_manager/factory/SpackManagerScratch.py b/esd/spack_manager/factory/SpackManagerScratch.py new file mode 100644 index 0000000000000000000000000000000000000000..5a79797ce924af10d1eda003fd55cf78661314a0 --- /dev/null +++ b/esd/spack_manager/factory/SpackManagerScratch.py @@ -0,0 +1,15 @@ +from esd.model.SpackModel import SpackModel +from esd.spack_manager.SpackManager import SpackManager +from esd.logger.logger_builder import get_logger + + +class SpackManagerScratch(SpackManager): + def __init__(self, env: SpackModel = None, repos=None, + upstream_instance=None, system_name: str = None): + super().__init__(env, repos, upstream_instance, system_name, logger=get_logger(__name__)) + + def concretize_spack_env(self, force=True): + pass + + def install_spack_packages(self, jobs: 3, verbose=False, debug=False): + pass diff --git a/esd/spack_manager/factory/__init__.py b/esd/spack_manager/factory/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/pyproject.toml b/pyproject.toml index 757f370c12a7602c9b7011c98c8aae416087ca24..b6679ca1f6946fde61d1ec5d288a8b85ec92d7a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,15 @@ [build-system] -requires = ["setuptools", "setuptools-scm"] +requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] name = "dedal" +version = "0.1.0" authors = [ {name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"}, {name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"}, ] description = "This package provides all the necessary tools to create an Ebrains Software Distribution environment" -version = "0.1.0" readme = "README.md" requires-python = ">=3.10" dependencies = [ @@ -18,6 +18,7 @@ dependencies = [ "ruamel.yaml", "pytest", "pytest-mock", + "pytest-ordering", ] [tool.setuptools.data-files]