diff --git a/README.md b/README.md index 299377a1782b473cb83c568160bfd4c2daa077bb..d6d39beef662e06cbf8a130b28889eff294133a6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Dedal - + This repository provides functionalities to easily ```managed spack environments``` and ```helpers for the container image build flow```. @@ -104,7 +104,7 @@ Sets configuration parameters for the session. Cache version for concretizaion data - `--cache_version_build <TEXT>` Cache version for binary caches data - `--view <SpackViewEnum>` Spack environment view -- `--update_cache <bool>` Flag for overriding existing cache +- `--override_cache <bool>` Flag for overriding existing cache ### 3. `dedal show-config` diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py index 1ca1ce955904ac7764b7bc2e54d4fd1d46b241f7..747435688ad00095bb6219122d791023d4d7b80e 100644 --- a/dedal/build_cache/BuildCacheManager.py +++ b/dedal/build_cache/BuildCacheManager.py @@ -33,12 +33,12 @@ class BuildCacheManager(BuildCacheManagerInterface): self.cache_version = cache_version self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/{self.cache_version}' - def upload(self, upload_dir: Path, update_cache=True): + def upload(self, upload_dir: Path, override_cache=True): """ This method pushed all the files from the build cache folder into the OCI Registry Args: upload_dir (Path): directory with the local binary caches - update_cache (bool): Updates the cache from the OCI Registry with the same tag + override_cache (bool): Updates the cache from the OCI Registry with the same tag """ build_cache_path = upload_dir.resolve() # build cache folder must exist before pushing all the artifacts @@ -53,7 +53,7 @@ class BuildCacheManager(BuildCacheManagerInterface): rel_path = str(sub_path.relative_to(build_cache_path)).replace(tag, "") target = f"{self._registry_host}/{self._registry_project}/{self.cache_version}:{tag}" upload_file = True - if update_cache is False and tag in tags: + if override_cache is False and tag in tags: upload_file = False if upload_file: try: diff --git a/dedal/cli/spack_manager_api.py b/dedal/cli/spack_manager_api.py index d5f5a9f05e2098fb47878a360fe833edcc7fb862..38e7fead52ada1c39e30f1438b1788141e947403 100644 --- a/dedal/cli/spack_manager_api.py +++ b/dedal/cli/spack_manager_api.py @@ -33,7 +33,7 @@ def cli(ctx: click.Context): system_name=config['system_name'], gpg=gpg, use_spack_global=config['use_spack_global'], view=config['view'], - update_cache=config['update_cache']) + override_cache=config['override_cache']) ctx.obj = SpackManager(spack_config, use_cache=config['use_cache']) @@ -56,11 +56,11 @@ def cli(ctx: click.Context): @click.option('--cache_version_concretize', type=str, default='v1', help='Cache version for concretizaion data') @click.option('--cache_version_build', type=str, default='v1', help='Cache version for binary caches data') @click.option('--view', type=SpackViewEnum, default=SpackViewEnum.VIEW, help='Spack environment view') -@click.option('--update_cache', is_flag=True, default=True, help='Flag for overriding existing cache') +@click.option('--override_cache', is_flag=True, default=True, help='Flag for overriding existing cache') def set_config(use_cache, env_name, env_path, env_git_path, install_dir, upstream_instance, system_name, concretization_dir, buildcache_dir, gpg_name, gpg_mail, use_spack_global, cache_version_concretize, cache_version_build, - view, update_cache): + view, override_cache): """Sets configuration parameters for the session.""" spack_config_data = { 'use_cache': use_cache, @@ -79,7 +79,7 @@ def set_config(use_cache, env_name, env_path, env_git_path, install_dir, upstrea 'cache_version_concretize': cache_version_concretize, 'cache_version_build': cache_version_build, 'view': view, - 'update_cache': update_cache, + 'override_cache': override_cache, } save_config(spack_config_data, SESSION_CONFIG_PATH) click.echo('Configuration saved.') diff --git a/dedal/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py index 7945e848be6b6655f65118afb0e6b2bff2c5112a..aaf66271b58e3cc0951166af0283c883a2ded6e2 100644 --- a/dedal/configuration/SpackConfig.py +++ b/dedal/configuration/SpackConfig.py @@ -11,7 +11,7 @@ class SpackConfig: install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None, concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None, use_spack_global=False, cache_version_concretize='v1', - cache_version_build='v1', view=SpackViewEnum.VIEW, update_cache=True): + cache_version_build='v1', view=SpackViewEnum.VIEW, override_cache=True, bashrc_path=os.path.expanduser("~/.bashrc")): self.env = env if repos is None: self.repos = [] @@ -27,7 +27,8 @@ class SpackConfig: self.cache_version_concretize = cache_version_concretize self.cache_version_build = cache_version_build self.view = view - self.update_cache = update_cache + self.override_cache = override_cache + self.bashrc_path = bashrc_path def add_repo(self, repo: SpackDescriptor): if self.repos is None: diff --git a/dedal/docs/resources/dedal_UML.png b/dedal/docs/resources/dedal_UML.png index 65d9f15828d9c6e19f8bffee41d5db3e02307b3f..4f9df5608e58c865fe13b3f3a9b74e6436023ef9 100644 Binary files a/dedal/docs/resources/dedal_UML.png and b/dedal/docs/resources/dedal_UML.png differ diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py index e190b6ef6e6742a4cdca237cfff887bec7031e30..a916582d62e39a619ac6f1216ce18f4785b61e1e 100644 --- a/dedal/spack_factory/SpackOperation.py +++ b/dedal/spack_factory/SpackOperation.py @@ -44,11 +44,13 @@ class SpackOperation: os.makedirs(self.spack_config.buildcache_dir, exist_ok=True) if self.spack_config.env and spack_config.env.name: self.env_path: Path = spack_config.env.path / spack_config.env.name - self.spack_command_on_env = f'{self.spack_setup_script} && spack env activate -p {spack_config.view.value} {self.env_path}' + if self.spack_setup_script != "": + self.spack_command_on_env = f'{self.spack_setup_script} && spack env activate -p {spack_config.view.value} {self.env_path}' + else: + self.spack_command_on_env = f'spack env activate -p {spack_config.view.value} {self.env_path}' else: self.spack_command_on_env = self.spack_setup_script if self.spack_config.env and spack_config.env.path: - self.spack_config.env.path = spack_config.env.path self.spack_config.env.path.mkdir(parents=True, exist_ok=True) def create_fetch_spack_environment(self): @@ -70,14 +72,14 @@ class SpackOperation: """ This method prepares a spack environment by fetching/creating the spack environment and adding the necessary repos """ - bashrc_path = os.path.expanduser("~/.bashrc") if self.spack_config.system_name: - set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, bashrc_path, logger=self.logger) + set_bashrc_variable('SYSTEMNAME', self.spack_config.system_name, self.spack_config.bashrc_path, + logger=self.logger) os.environ['SYSTEMNAME'] = self.spack_config.system_name if self.spack_dir.exists() and self.spack_dir.is_dir(): - set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), bashrc_path, + set_bashrc_variable('SPACK_USER_CACHE_PATH', str(self.spack_dir / ".spack"), self.spack_config.bashrc_path, logger=self.logger) - set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), bashrc_path, + set_bashrc_variable('SPACK_USER_CONFIG_PATH', str(self.spack_dir / ".spack"), self.spack_config.bashrc_path, logger=self.logger) self.logger.debug('Added env variables SPACK_USER_CACHE_PATH and SPACK_USER_CONFIG_PATH') else: @@ -307,7 +309,7 @@ class SpackOperation: def config(self, config_type: SpackConfigCommand, config_parameter): run_command("bash", "-c", - f'{self.spack_command_on_env} && spack config {config_type.value} {config_parameter}', + f'{self.spack_command_on_env} && spack config {config_type.value} \"{config_parameter}\"', check=True, logger=self.logger, info_msg='Spack config command', @@ -341,6 +343,28 @@ class SpackOperation: exception_msg=f'Failed to remove mirror {mirror_name}', exception=SpackMirrorException) + def update_buildcache_index(self, mirror_path: str): + """Updates buildcache index""" + if not mirror_path: + raise ValueError("mirror_path is required") + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack buildcache update-index {mirror_path}', + check=True, + logger=self.logger, + info_msg=f'Updating build cache index for mirror {mirror_path}', + exception_msg=f'Failed to update build cache index for mirror {mirror_path}', + exception=SpackMirrorException) + + def install_gpg_keys(self): + """Install gpg keys""" + run_command("bash", "-c", + f'{self.spack_command_on_env} && spack buildcache keys --install --trust', + check=True, + logger=self.logger, + info_msg=f'Installing gpg keys from mirror', + exception_msg=f'Failed to install gpg keys from mirror', + exception=SpackGpgException) + @check_spack_env def install_packages(self, jobs: int, signed=True, fresh=False, debug=False, test=None): """Installs all spack packages. @@ -361,6 +385,10 @@ class SpackOperation: exception_msg=f"Error installing spack packages for {self.spack_config.env.name}", exception=SpackInstallPackagesException) log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log")) + if install_result.returncode == 0: + self.logger.info(f'Finished installation of spack packages from scratch') + else: + self.logger.error(f'Something went wrong during installation. Please check the logs.') return install_result @check_spack_env diff --git a/dedal/spack_factory/SpackOperationCreateCache.py b/dedal/spack_factory/SpackOperationCreateCache.py index 45e0e7441a874d2ed09057ea889a064aab77f087..6d365a6fe2cc64d090904d7871ec9a4b3a615c5f 100644 --- a/dedal/spack_factory/SpackOperationCreateCache.py +++ b/dedal/spack_factory/SpackOperationCreateCache.py @@ -25,6 +25,23 @@ class SpackOperationCreateCache(SpackOperation): os.environ.get('BUILDCACHE_OCI_USERNAME'), os.environ.get('BUILDCACHE_OCI_PASSWORD'), cache_version=spack_config.cache_version_build) + self.signed = False + if self.spack_config.gpg: + self.signed = True + + def setup_spack_env(self) -> None: + """Set up the spack environment for creating the cache. + Creates the caching for the concretization and install steps and then uploads them to the OCI registry. + Raises: + NoSpackEnvironmentException: If the spack environment is not set up. + """ + super().setup_spack_env() + self.add_mirror('local_cache', + str(self.spack_config.buildcache_dir), + signed=self.signed, + autopush=True, + global_mirror=False) + self.logger.info(f'Added mirror for {self.spack_config.env.name}') @check_spack_env def concretize_spack_env(self, test=None): @@ -35,7 +52,7 @@ class SpackOperationCreateCache(SpackOperation): super().concretize_spack_env(force=True, test=test) dependency_path = self.spack_config.env.path / self.spack_config.env.name / 'spack.lock' copy_file(dependency_path, self.spack_config.concretization_dir, logger=self.logger) - self.cache_dependency.upload(self.spack_config.concretization_dir, update_cache=self.spack_config.update_cache) + self.cache_dependency.upload(self.spack_config.concretization_dir, override_cache=self.spack_config.override_cache) self.logger.info(f'Created new spack concretization for create cache: {self.spack_config.env.name}') @check_spack_env @@ -44,17 +61,8 @@ class SpackOperationCreateCache(SpackOperation): Raises: NoSpackEnvironmentException: If the spack environment is not set up. """ - signed = False - if self.spack_config.gpg: - signed = True + if self.signed: self.create_gpg_keys() - self.add_mirror('local_cache', - str(self.spack_config.buildcache_dir), - signed=signed, - autopush=signed, - global_mirror=False) - self.logger.info(f'Added mirror for {self.spack_config.env.name}') - super().install_packages(jobs=jobs, signed=signed, debug=debug, fresh=True, test=test) - self.logger.info(f'Installed spack packages for {self.spack_config.env.name}') - self.build_cache.upload(self.spack_config.buildcache_dir, update_cache=self.spack_config.update_cache) - self.logger.info(f'Pushed spack packages for {self.spack_config.env.name}') + self.logger.info('Created gpg keys') + super().install_packages(jobs=jobs, signed=self.signed, debug=debug, fresh=True, test=test) + self.build_cache.upload(self.spack_config.buildcache_dir, override_cache=self.spack_config.override_cache) diff --git a/dedal/spack_factory/SpackOperationCreator.py b/dedal/spack_factory/SpackOperationCreator.py index fdc929d34866dcb254a775e9e8f8a24ee893029d..f139f51a0b0374fd0809517dbccebde2af5e1fc0 100644 --- a/dedal/spack_factory/SpackOperationCreator.py +++ b/dedal/spack_factory/SpackOperationCreator.py @@ -1,3 +1,5 @@ +import os + from dedal.configuration.SpackConfig import SpackConfig from dedal.spack_factory.SpackOperation import SpackOperation from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache @@ -7,8 +9,19 @@ from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache class SpackOperationCreator: @staticmethod def get_spack_operator(spack_config: SpackConfig = None, use_cache: bool = False) -> SpackOperation: + env_vars = [os.environ.get('CONCRETIZE_OCI_HOST'), + os.environ.get('CONCRETIZE_OCI_PROJECT'), + os.environ.get('CONCRETIZE_OCI_USERNAME'), + os.environ.get('CONCRETIZE_OCI_PASSWORD'), + os.environ.get('BUILDCACHE_OCI_HOST'), + os.environ.get('BUILDCACHE_OCI_PROJECT'), + os.environ.get('BUILDCACHE_OCI_USERNAME'), + os.environ.get('BUILDCACHE_OCI_PASSWORD') + ] if spack_config is None: return SpackOperation() + elif None in env_vars: + return SpackOperation(spack_config) elif spack_config.concretization_dir is None and spack_config.buildcache_dir is None: return SpackOperation(spack_config) elif (spack_config.concretization_dir and spack_config.buildcache_dir) and not use_cache: diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py index 45c0ab4f189f8d5d35f0e2f53f84f31f64171df5..80c500dbbe848aca1f845ce2ab7717452cd2b7ce 100644 --- a/dedal/spack_factory/SpackOperationUseCache.py +++ b/dedal/spack_factory/SpackOperationUseCache.py @@ -4,6 +4,7 @@ from pathlib import Path from dedal.build_cache.BuildCacheManager import BuildCacheManager from dedal.configuration.SpackConfig import SpackConfig +from dedal.enum.SpackConfigCommand import SpackConfigCommand from dedal.error_handling.exceptions import SpackInstallPackagesException from dedal.logger.logger_builder import get_logger from dedal.spack_factory.SpackOperation import SpackOperation @@ -49,8 +50,10 @@ class SpackOperationUseCache(SpackOperation): self.add_mirror('local_cache', str(self.spack_config.buildcache_dir), signed=signed, - autopush=True, + autopush=False, global_mirror=False) + self.update_buildcache_index(self.spack_config.buildcache_dir) + self.install_gpg_keys() @check_spack_env def concretize_spack_env(self, test=None): @@ -74,7 +77,7 @@ class SpackOperationUseCache(SpackOperation): if concretization_redo is True: self.logger.info(f'Redo of concretization step.') else: - self.logger.info(f'USed concretization from cache.') + self.logger.info(f'Used concretization from cache.') return concretization_redo @check_spack_env @@ -88,7 +91,7 @@ class SpackOperationUseCache(SpackOperation): debug = '--debug' if debug else '' test = f'--test {test}' if test else '' install_result = run_command("bash", "-c", - f'{self.spack_command_on_env} && spack {debug} install -v --reuse {signed} -j {jobs} {test}', + f'{self.spack_command_on_env} && spack {debug} install -v --cache-only {signed} -j {jobs} {test}', stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -97,5 +100,8 @@ class SpackOperationUseCache(SpackOperation): exception_msg=f"Error installing spack packages for {self.spack_config.env.name}", exception=SpackInstallPackagesException) log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log")) - self.logger.info(f'Finished installation of spack packages from cache.') + if install_result.returncode == 0: + self.logger.info(f'Finished installation of spack packages from cache.') + else: + self.logger.error(f'Something went wrong during installation from cache. Please check the logs.') return install_result diff --git a/dedal/tests/integration_tests/helper.py b/dedal/tests/integration_tests/helper.py index 58f9e0b1968e7f83afb289aa6bc3df050a809e97..11ff3615aaaf8612c90a67d88d656c35bf40b6e1 100644 --- a/dedal/tests/integration_tests/helper.py +++ b/dedal/tests/integration_tests/helper.py @@ -1,3 +1,4 @@ +import uuid from pathlib import Path from dedal.spack_factory.SpackOperation import SpackOperation @@ -10,3 +11,7 @@ def check_installed_spack_packages(spack_operation: SpackOperation, install_dir: installed_spack_packages = list(spack_operation.find_packages().keys()) for spack_pacakge in to_install_spack_packages: assert spack_pacakge in installed_spack_packages + + +def get_test_path(): + return Path(f"/tmp/test-{uuid.uuid4()}").resolve() \ No newline at end of file diff --git a/dedal/tests/integration_tests/spack_create_cache_test.py b/dedal/tests/integration_tests/spack_create_cache_test.py index 1da51128a873222796753bfe2d81cbd5506777f6..c8c5a76515c7095de02246408c753d6539b37bd5 100644 --- a/dedal/tests/integration_tests/spack_create_cache_test.py +++ b/dedal/tests/integration_tests/spack_create_cache_test.py @@ -17,10 +17,8 @@ Ebrains Harbour does not support deletion via API, so the clean up must be done """ -@pytest.mark.skip( - reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.") -def test_spack_create_cache_concretization(tmp_path): - install_dir = tmp_path +def setup(path): + install_dir = path concretization_dir = install_dir / 'concretization' buildcache_dir = install_dir / 'buildcache' env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git) @@ -30,18 +28,31 @@ def test_spack_create_cache_concretization(tmp_path): buildcache_dir=buildcache_dir, gpg=gpg) config.add_repo(repo) spack_operation = SpackOperationCreator.get_spack_operator(config) + return spack_operation + + +@pytest.mark.skip( + reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.") +def test_spack_create_cache_concretization(tmp_path): + install_dir = tmp_path + spack_operation = setup(install_dir) assert isinstance(spack_operation, SpackOperationCreateCache) spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc'))) spack_operation.setup_spack_env() spack_operation.concretize_spack_env() assert len(spack_operation.cache_dependency.list_tags()) > 0 - return spack_operation @pytest.mark.skip( reason="Skipping until an OCI registry which supports via API deletion; Clean up for OCI registry repo must be added before this test.") def test_spack_create_cache_installation(tmp_path): - spack_operation = test_spack_create_cache_concretization(tmp_path) + install_dir = tmp_path + spack_operation = setup(install_dir) + assert isinstance(spack_operation, SpackOperationCreateCache) + spack_operation.install_spack(bashrc_path=str(install_dir / Path('.bashrc'))) + spack_operation.setup_spack_env() + spack_operation.concretize_spack_env() + assert len(spack_operation.cache_dependency.list_tags()) > 0 spack_operation.install_packages() assert len(spack_operation.build_cache.list_tags()) > 0 check_installed_spack_packages(spack_operation, tmp_path) diff --git a/dedal/tests/integration_tests/spack_from_cache_test.py b/dedal/tests/integration_tests/spack_from_cache_test.py index 206fdbfeaf536bf95802c1c5864883219b634899..d0f6a5e0896d5d4d6df0262524d2090a1aea82a4 100644 --- a/dedal/tests/integration_tests/spack_from_cache_test.py +++ b/dedal/tests/integration_tests/spack_from_cache_test.py @@ -4,7 +4,7 @@ from dedal.configuration.SpackConfig import SpackConfig from dedal.model.SpackDescriptor import SpackDescriptor from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache -from dedal.tests.integration_tests.helper import check_installed_spack_packages +from dedal.tests.integration_tests.helper import check_installed_spack_packages, get_test_path from dedal.utils.utils import file_exists_and_not_empty, count_files_in_folder from dedal.utils.variables import test_spack_env_git, ebrains_spack_builds_git @@ -22,64 +22,68 @@ def setup(path): return spack_operation, concretization_dir, buildcache_dir -def test_spack_from_cache_setup(tmp_path): - spack_operation, concretization_dir, buildcache_dir = setup(tmp_path) +def test_spack_from_cache_setup(): + install_dir = get_test_path() + spack_operation, concretization_dir, buildcache_dir = setup(install_dir) assert isinstance(spack_operation, SpackOperationUseCache) - spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc'))) + spack_operation.install_spack(bashrc_path=str(install_dir / Path('.bashrc'))) spack_operation.setup_spack_env() num_tags = len(spack_operation.build_cache.list_tags()) concretization_download_file_path = concretization_dir / 'spack.lock' assert file_exists_and_not_empty(concretization_download_file_path) == True - assert count_files_in_folder(buildcache_dir) == num_tags + assert count_files_in_folder(buildcache_dir) == num_tags + 2 assert 'local_cache' in spack_operation.mirror_list() -def test_spack_from_cache_concretize(tmp_path): - spack_operation, concretization_dir, buildcache_dir = setup(tmp_path) +def test_spack_from_cache_concretize(): + install_dir = get_test_path() + spack_operation, concretization_dir, buildcache_dir = setup(install_dir) assert isinstance(spack_operation, SpackOperationUseCache) - spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc'))) + spack_operation.install_spack(bashrc_path=str(install_dir / Path('.bashrc'))) spack_operation.setup_spack_env() num_tags = len(spack_operation.build_cache.list_tags()) concretization_download_file_path = concretization_dir / 'spack.lock' assert file_exists_and_not_empty(concretization_download_file_path) == True - assert count_files_in_folder(buildcache_dir) == num_tags + assert count_files_in_folder(buildcache_dir) == num_tags + 2 assert 'local_cache' in spack_operation.mirror_list() assert spack_operation.concretize_spack_env() == False concretization_file_path = spack_operation.env_path / 'spack.lock' assert file_exists_and_not_empty(concretization_file_path) == True -def test_spack_from_cache_install_1(tmp_path): - spack_operation, concretization_dir, buildcache_dir = setup(tmp_path) +def test_spack_from_cache_install_1(): + install_dir = get_test_path() + spack_operation, concretization_dir, buildcache_dir = setup(install_dir) assert isinstance(spack_operation, SpackOperationUseCache) - spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc'))) + spack_operation.install_spack(bashrc_path=str(install_dir / Path('.bashrc'))) spack_operation.setup_spack_env() num_tags = len(spack_operation.build_cache.list_tags()) concretization_download_file_path = concretization_dir / 'spack.lock' assert file_exists_and_not_empty(concretization_download_file_path) == True - assert count_files_in_folder(buildcache_dir) == num_tags + assert count_files_in_folder(buildcache_dir) == num_tags + 2 assert 'local_cache' in spack_operation.mirror_list() assert spack_operation.concretize_spack_env() == False concretization_file_path = spack_operation.env_path / 'spack.lock' assert file_exists_and_not_empty(concretization_file_path) == True install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False) assert install_result.returncode == 0 - check_installed_spack_packages(spack_operation, tmp_path) + check_installed_spack_packages(spack_operation, install_dir) -def test_spack_from_cache_install_2(tmp_path): - spack_operation, concretization_dir, buildcache_dir = setup(tmp_path) +def test_spack_from_cache_install_2(): + install_dir = get_test_path() + spack_operation, concretization_dir, buildcache_dir = setup(install_dir) assert isinstance(spack_operation, SpackOperationUseCache) - spack_operation.install_spack(bashrc_path=str(tmp_path / Path('.bashrc'))) + spack_operation.install_spack(bashrc_path=str(install_dir / Path('.bashrc'))) spack_operation.setup_spack_env() num_tags = len(spack_operation.build_cache.list_tags()) concretization_download_file_path = concretization_dir / 'spack.lock' assert file_exists_and_not_empty(concretization_download_file_path) == True - assert count_files_in_folder(buildcache_dir) == num_tags + assert count_files_in_folder(buildcache_dir) == num_tags + 2 assert 'local_cache' in spack_operation.mirror_list() assert spack_operation.concretize_spack_env() == False concretization_file_path = spack_operation.env_path / 'spack.lock' assert file_exists_and_not_empty(concretization_file_path) == True install_result = spack_operation.install_packages(jobs=2, signed=True, debug=False, test='root') assert install_result.returncode == 0 - check_installed_spack_packages(spack_operation, tmp_path) + check_installed_spack_packages(spack_operation, install_dir) diff --git a/dedal/tests/integration_tests/spack_operation_creator_test.py b/dedal/tests/integration_tests/spack_operation_creator_test.py index 226184b00a5c7136c97f5ef12761ac44c71286a1..c93d039dc44fd9084655fb261d422862850b51f3 100644 --- a/dedal/tests/integration_tests/spack_operation_creator_test.py +++ b/dedal/tests/integration_tests/spack_operation_creator_test.py @@ -1,3 +1,5 @@ +import os + from dedal.spack_factory.SpackOperationCreateCache import SpackOperationCreateCache from dedal.configuration.SpackConfig import SpackConfig @@ -29,6 +31,18 @@ def test_spack_creator_scratch_3(): spack_operation = SpackOperationCreator.get_spack_operator(spack_config) assert isinstance(spack_operation, SpackOperation) +def test_spack_creator_scratch_4(tmp_path): + concretize_oci_host = os.environ.get('CONCRETIZE_OCI_HOST') + os.environ.pop('CONCRETIZE_OCI_HOST', None) + install_dir = tmp_path + env = SpackDescriptor('test-spack-env', install_dir, test_spack_env_git) + repo = SpackDescriptor('ebrains-spack-builds', install_dir, ebrains_spack_builds_git) + spack_config = SpackConfig(env, install_dir=install_dir, concretization_dir=install_dir, buildcache_dir=install_dir) + spack_config.add_repo(repo) + spack_operation = SpackOperationCreator.get_spack_operator(spack_config) + os.environ['CONCRETIZE_OCI_HOST'] = concretize_oci_host + assert isinstance(spack_operation, SpackOperation) + def test_spack_creator_create_cache(tmp_path): install_dir = tmp_path diff --git a/dedal/tests/unit_tests/spack_manager_api_test.py b/dedal/tests/unit_tests/spack_manager_api_test.py index 2037a502565d47ea19498aee7802e0ea6c30e73f..040ed58bd49d6f239c2bcc2d21928303835e466a 100644 --- a/dedal/tests/unit_tests/spack_manager_api_test.py +++ b/dedal/tests/unit_tests/spack_manager_api_test.py @@ -175,7 +175,7 @@ def test_set_config(runner, mock_save_config, mocked_session_path): 'cache_version_concretize': 'v1', 'cache_version_build': 'v1', 'view': SpackViewEnum.VIEW, - 'update_cache': True, + 'override_cache': True, } mock_save_config.assert_called_once() diff --git a/dedal/tests/unit_tests/spack_operation_use_cache_test.py b/dedal/tests/unit_tests/spack_operation_use_cache_test.py deleted file mode 100644 index fad549bf8c8b07543da45fba7c5b86141bcb3666..0000000000000000000000000000000000000000 --- a/dedal/tests/unit_tests/spack_operation_use_cache_test.py +++ /dev/null @@ -1,75 +0,0 @@ -from pathlib import Path - -import pytest - -from dedal.error_handling.exceptions import NoSpackEnvironmentException -from dedal.spack_factory.SpackOperationUseCache import SpackOperationUseCache - - -@pytest.fixture -def spack_operation_use_cache_mock(mocker): - super_mock = mocker.patch("dedal.spack_factory.SpackOperationUseCache.super") - super_mock.return_value.setup_spack_env = mocker.MagicMock() - mocker.patch("dedal.spack_factory.SpackOperationUseCache.BuildCacheManager") - mock_spack_operation_use_cache = SpackOperationUseCache() - mock_spack_operation_use_cache.build_cache = mocker.MagicMock() - mock_spack_operation_use_cache.spack_config = mocker.MagicMock() - mock_spack_operation_use_cache.spack_config.buildcache_dir = Path("path/to/buildcache") - mock_spack_operation_use_cache.logger = mocker.MagicMock() - return mock_spack_operation_use_cache - - -class TestSpackOperationUseCache: - - @pytest.mark.parametrize("test_id, signed, key_path", [ - ("key_path_exists", True, "path/to/key.gpg"), - ("key_path_does_not_exist", False, None)]) - def test_setup_spack_env(self, mocker, spack_operation_use_cache_mock, test_id, signed, key_path): - # Test ID: setup_spack_env_success - super_mock = mocker.patch("dedal.spack_factory.SpackOperationUseCache.super") - spack_operation_use_cache_mock.trust_gpg_key = mocker.MagicMock() - spack_operation_use_cache_mock.add_mirror = mocker.MagicMock() - - # Arrange - spack_operation_use_cache_mock.build_cache.get_public_key_from_cache.return_value = key_path - spack_operation_use_cache_mock.trust_gpg_key.return_value = signed - spack_operation_use_cache_mock.add_mirror.return_value = None - - # Act - spack_operation_use_cache_mock.setup_spack_env() - - # Assert - spack_operation_use_cache_mock.build_cache.download.assert_called_once_with( - spack_operation_use_cache_mock.spack_config.buildcache_dir) - spack_operation_use_cache_mock.build_cache.get_public_key_from_cache.assert_called_once_with( - str(spack_operation_use_cache_mock.spack_config.buildcache_dir)) - - if key_path: - spack_operation_use_cache_mock.trust_gpg_key.assert_called_once_with(key_path) - else: - spack_operation_use_cache_mock.trust_gpg_key.assert_not_called() - - spack_operation_use_cache_mock.add_mirror.assert_called_once_with( - 'local_cache', - str(spack_operation_use_cache_mock.spack_config.buildcache_dir), - signed=signed, - autopush=True, - global_mirror=False - ) - super_mock.return_value.setup_spack_env.assert_called_once() # call original method - - @pytest.mark.parametrize("exception_type", [NoSpackEnvironmentException]) - def test_setup_spack_env_exceptions(self, mocker, spack_operation_use_cache_mock, exception_type): - # Test ID: setup_spack_env_exceptions - spack_operation_use_cache_mock.trust_gpg_key = mocker.MagicMock() - spack_operation_use_cache_mock.add_mirror = mocker.MagicMock() - - # Arrange - spack_operation_use_cache_mock.build_cache.get_public_key_from_cache.return_value = "path/to/key.gpg" - spack_operation_use_cache_mock.trust_gpg_key.return_value = True - exception = exception_type("test exception") - spack_operation_use_cache_mock.add_mirror.side_effect = exception - - # Act & Assert - with pytest.raises(exception_type): - spack_operation_use_cache_mock.setup_spack_env() \ No newline at end of file diff --git a/dedal/utils/bootstrap.sh b/dedal/utils/bootstrap.sh index 9cd2e1e11b4ab82fd301cc86179de7ba373f5d03..831ee4c3ea313dea7cd2f863ea5217a91c45ea7c 100644 --- a/dedal/utils/bootstrap.sh +++ b/dedal/utils/bootstrap.sh @@ -7,5 +7,5 @@ export DEBIAN_FRONTEND=noninteractive apt update apt install -o DPkg::Options::=--force-confold -y -q --reinstall \ bzip2 ca-certificates g++ gcc make gfortran git gzip lsb-release \ - patch python3 python3-pip tar unzip xz-utils zstd gnupg2 vim curl rsync + patch python3 python3-pip tar unzip xz-utils zstd gnupg2 vim curl rsync file python3 -m pip install --upgrade pip setuptools wheel diff --git a/dedal/utils/utils.py b/dedal/utils/utils.py index 29c6a2a622b9d2e6503da697b8d91be948102aee..cbc936e924f453ace3dcd05af045ee3fb1abed0b 100644 --- a/dedal/utils/utils.py +++ b/dedal/utils/utils.py @@ -97,6 +97,8 @@ def copy_to_tmp(file_path: Path) -> Path: def set_bashrc_variable(var_name: str, value: str, bashrc_path: str = os.path.expanduser("~/.bashrc"), logger: logging = logging.getLogger(__name__), update_variable=False): """Update or add an environment variable in ~/.bashrc.""" + if bashrc_path is None or var_name is None or value is None: + return value = value.replace("$", r"\$") with open(bashrc_path, "r") as file: lines = file.readlines()