VT-109: factory design patter for build cache manager

dedal/build_cache/BuildCacheManagerCreator.py

+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+#  (c) Copyright 2025 Dedal developers
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#      http://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+
+from dedal.build_cache.BuildCacheManagerOci import BuildCacheManagerOci
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.enum.SpackCacheStorageEnum import SpackCacheStorageEnum
+
+
+class BuildCacheManagerCreator:
+    @staticmethod
+    def get_build_cache_manager_operator(spack_config: SpackConfig = SpackConfig()):
+        if spack_config.spack_cache_storage_type is None:
+            return None, None
+        elif spack_config.spack_cache_storage_type is SpackCacheStorageEnum.OCI:
+            cache_dependency = BuildCacheManagerOci(os.environ.get('CONCRETIZE_OCI_HOST'),
+                                                    os.environ.get('CONCRETIZE_OCI_PROJECT'),
+                                                    os.environ.get('CONCRETIZE_OCI_USERNAME'),
+                                                    os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+                                                    cache_version=spack_config.cache_version_concretize)
+            build_cache = BuildCacheManagerOci(os.environ.get('BUILDCACHE_OCI_HOST'),
+                                               os.environ.get('BUILDCACHE_OCI_PROJECT'),
+                                               os.environ.get('BUILDCACHE_OCI_USERNAME'),
+                                               os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+                                               cache_version=spack_config.cache_version_build)
+            return cache_dependency, build_cache

dedal/build_cache/BuildCacheManager.py → dedal/build_cache/BuildCacheManagerOci.py

@@ -27,7 +27,7 @@ from dedal.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterf
 from dedal.logger.logger_builder import get_logger
 
 
-class BuildCacheManager(BuildCacheManagerInterface):
+class BuildCacheManagerOci(BuildCacheManagerInterface):
     """
         This class aims to manage the push/pull/delete of build cache files
     """
@@ -35,7 +35,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
     def __new__(cls, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
                 auth_backend='basic', insecure=False, tls_verify=True):
         instance = super().__new__(cls)
-        instance._logger = get_logger(__name__, BuildCacheManager.__name__)
+        instance._logger = get_logger(__name__, BuildCacheManagerOci.__name__)
         instance._registry_project = registry_project
 
         instance._registry_username = registry_username

dedal/configuration/SpackConfig.py

@@ -18,6 +18,7 @@
 import os
 from pathlib import Path
 from dedal.configuration.GpgConfig import GpgConfig
+from dedal.enum.SpackCacheStorageEnum import SpackCacheStorageEnum
 from dedal.enum.SpackViewEnum import SpackViewEnum
 from dedal.model import SpackDescriptor
 from dedal.utils.utils import resolve_path
@@ -28,7 +29,8 @@ class SpackConfig:
                  install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
                  concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None,
                  use_spack_global=False, cache_version_concretize='v1',
-                 cache_version_build='v1', view=SpackViewEnum.VIEW, override_cache=True, bashrc_path=os.path.expanduser("~/.bashrc")):
+                 cache_version_build='v1', view=SpackViewEnum.VIEW, override_cache=True, bashrc_path=os.path.expanduser("~/.bashrc"),
+                 spack_cache_storage_type: SpackCacheStorageEnum = SpackCacheStorageEnum.OCI):
         self.env = env
         if repos is None:
             self.repos = []
@@ -46,6 +48,7 @@ class SpackConfig:
         self.view = view
         self.override_cache = override_cache
         self.bashrc_path = bashrc_path
+        self.spack_cache_storage_type = spack_cache_storage_type
 
     def add_repo(self, repo: SpackDescriptor):
         if self.repos is None:

dedal/enum/SpackCacheStorageEnum.py

+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+#  (c) Copyright 2025 Dedal developers
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#      http://www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from enum import Enum
+
+
+class SpackCacheStorageEnum(Enum):
+    """
+    Different types of storages used for caching
+    """
+    OCI = 'oci_cache'

dedal/spack_factory/SpackEnvOperation.py

@@ -40,14 +40,16 @@ class SpackEnvOperation:
         else:
             raise MissingAttributeException(f'Missing attribute for class {__name__}')
 
-    def create_fetch_spack_environment(self):
+    def setup_spack_environment(self):
         """Fetches a spack environment if the git path is defined, otherwise creates it."""
         if self.spack_config.env and self.spack_config.env.git_path:
-            git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name,
+            git_clone_repo(self.spack_config.env.name, self.env_path,
                            self.spack_config.env.git_path,
                            logger=self.logger)
+        elif self.env_path and self.env_path.exists():
+            return
         else:
-            os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True)
+            os.makedirs(self.env_path, exist_ok=True)
             if self.env_path:
                 run_command("bash", "-c",
                             f'{self.spack_setup_script} && spack env create -d {self.env_path}',

dedal/spack_factory/SpackOperation.py

@@ -99,7 +99,7 @@ class SpackOperation:
             self.logger.error(f'Invalid installation path: {self.spack_dir}')
         # Restart the bash after adding environment variables
         if self.spack_config.env:
-            self.spack_env_operation.create_fetch_spack_environment()
+            self.spack_env_operation.setup_spack_environment()
         if self.spack_config.install_dir.exists():
             for repo in self.spack_config.repos:
                 repo_dir = self.spack_config.install_dir / repo.path / repo.name
@@ -156,7 +156,7 @@ class SpackOperation:
         return install_result
 
     def create_fetch_spack_environment(self):
-        self.spack_env_operation.create_fetch_spack_environment()
+        self.spack_env_operation.setup_spack_environment()
 
     def add_spack_repo(self, repo_path: Path, repo_name: str):
         self.spack_env_operation.add_spack_repo(repo_path, repo_name)

dedal/spack_factory/SpackOperationCreateCache.py

@@ -17,9 +17,9 @@
 
 import os
 
+from dedal.build_cache.BuildCacheManagerCreator import BuildCacheManagerCreator
 from dedal.utils.utils import copy_file
 from dedal.wrapper.spack_wrapper import check_spack_env
-from dedal.build_cache.BuildCacheManager import BuildCacheManager
 from dedal.configuration.SpackConfig import SpackConfig
 from dedal.logger.logger_builder import get_logger
 from dedal.spack_factory.SpackOperation import SpackOperation
@@ -32,16 +32,8 @@ class SpackOperationCreateCache(SpackOperation):
 
     def __init__(self, spack_config: SpackConfig = SpackConfig()):
         super().__init__(spack_config, logger=get_logger(__name__))
-        self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
-                                                  os.environ.get('CONCRETIZE_OCI_PROJECT'),
-                                                  os.environ.get('CONCRETIZE_OCI_USERNAME'),
-                                                  os.environ.get('CONCRETIZE_OCI_PASSWORD'),
-                                                  cache_version=spack_config.cache_version_concretize)
-        self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
-                                             os.environ.get('BUILDCACHE_OCI_PROJECT'),
-                                             os.environ.get('BUILDCACHE_OCI_USERNAME'),
-                                             os.environ.get('BUILDCACHE_OCI_PASSWORD'),
-                                             cache_version=spack_config.cache_version_build)
+        self.cache_dependency, self.build_cache = BuildCacheManagerCreator.get_build_cache_manager_operator(
+            spack_config)
         self.signed = False
         if self.spack_config.gpg:
             self.signed = True

dedal/spack_factory/SpackOperationUseCache.py

@@ -19,9 +19,8 @@ import os
 import subprocess
 from pathlib import Path
 
-from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.build_cache.BuildCacheManagerCreator import BuildCacheManagerCreator
 from dedal.configuration.SpackConfig import SpackConfig
-from dedal.enum.SpackConfigCommand import SpackConfigCommand
 from dedal.error_handling.exceptions import SpackInstallPackagesException
 from dedal.logger.logger_builder import get_logger
 from dedal.spack_factory.SpackOperation import SpackOperation
@@ -36,16 +35,8 @@ class SpackOperationUseCache(SpackOperation):
 
     def __init__(self, spack_config: SpackConfig = SpackConfig()):
         super().__init__(spack_config, logger=get_logger(__name__))
-        self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
-                                                  os.environ.get('CONCRETIZE_OCI_PROJECT'),
-                                                  os.environ.get('CONCRETIZE_OCI_USERNAME'),
-                                                  os.environ.get('CONCRETIZE_OCI_PASSWORD'),
-                                                  cache_version=spack_config.cache_version_concretize)
-        self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
-                                             os.environ.get('BUILDCACHE_OCI_PROJECT'),
-                                             os.environ.get('BUILDCACHE_OCI_USERNAME'),
-                                             os.environ.get('BUILDCACHE_OCI_PASSWORD'),
-                                             cache_version=spack_config.cache_version_build)
+        self.cache_dependency, self.build_cache = BuildCacheManagerCreator.get_build_cache_manager_operator(
+            spack_config)
 
     def setup_spack_env(self) -> None:
         """Set up the spack environment for using the cache.

dedal/tests/integration_tests/spack_from_scratch_test.py

@@ -68,7 +68,7 @@ def test_spack_reindex(tmp_path):
     spack_operation.reindex()
 
 
-@pytest.mark.skip(reason="It does nopt work on bare metal operating systems")
+@pytest.mark.skip(reason="It does not work on bare metal operating systems")
 def test_spack_spec(tmp_path):
     install_dir = tmp_path
     config = SpackConfig(install_dir=install_dir)

dedal/tests/testing_variables.py

@@ -18,6 +18,6 @@
 import os
 
 ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
-SPACK_VERSION = "0.23.0"
+SPACK_VERSION = "0.23.1"
 SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
 test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'

dedal/tests/unit_tests/build_cache_manager_test.py

@@ -18,15 +18,15 @@
 import pytest
 from _pytest.fixtures import fixture
 
-from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.build_cache.BuildCacheManagerOci import BuildCacheManagerOci
 
 
-class TestBuildCacheManager:
+class TestBuildCacheManagerOci:
 
     @fixture(scope="function")
     def mock_build_cache_manager(self, mocker):
-        mocker.patch("dedal.build_cache.BuildCacheManager.get_logger")
-        return BuildCacheManager("TEST_HOST", "TEST_PROJECT", "TEST_USERNAME", "TEST_PASSWORD", "TEST_VERSION")
+        mocker.patch("dedal.build_cache.BuildCacheManagerOci.get_logger")
+        return BuildCacheManagerOci("TEST_HOST", "TEST_PROJECT", "TEST_USERNAME", "TEST_PASSWORD", "TEST_VERSION")
 
     def test_get_public_key_from_cache_success_path(self, mock_build_cache_manager, tmp_path):
 
@@ -149,7 +149,7 @@ class TestBuildCacheManager:
         warn_message = "test message"
 
         # Act
-        mock_build_cache_manager._BuildCacheManager__log_warning_if_needed(warn_message, items)
+        mock_build_cache_manager._BuildCacheManagerOci__log_warning_if_needed(warn_message, items)
 
         # Assert
         mock_build_cache_manager._logger.warning.assert_called_once_with(warn_message, items, items[0])
@@ -165,7 +165,7 @@ class TestBuildCacheManager:
         warn_message = "test message"
 
         # Act
-        mock_build_cache_manager._BuildCacheManager__log_warning_if_needed(warn_message, items)
+        mock_build_cache_manager._BuildCacheManagerOci__log_warning_if_needed(warn_message, items)
 
         # Assert
         mock_build_cache_manager._logger.warning.assert_not_called()