From f68beec3d4e30a7bd36c010e35c419297ebbf538 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 16 Apr 2025 17:23:42 +0300
Subject: [PATCH] VT-109: factory design patter for build cache manager
---
dedal/build_cache/BuildCacheManagerCreator.py | 41 +++++++++++++++++++
...acheManager.py => BuildCacheManagerOci.py} | 4 +-
dedal/configuration/SpackConfig.py | 5 ++-
dedal/enum/SpackCacheStorageEnum.py | 25 +++++++++++
dedal/spack_factory/SpackEnvOperation.py | 8 ++--
dedal/spack_factory/SpackOperation.py | 4 +-
.../SpackOperationCreateCache.py | 14 ++-----
dedal/spack_factory/SpackOperationUseCache.py | 15 ++-----
.../spack_from_scratch_test.py | 2 +-
dedal/tests/testing_variables.py | 2 +-
.../unit_tests/build_cache_manager_test.py | 12 +++---
11 files changed, 93 insertions(+), 39 deletions(-)
create mode 100644 dedal/build_cache/BuildCacheManagerCreator.py
rename dedal/build_cache/{BuildCacheManager.py => BuildCacheManagerOci.py} (99%)
create mode 100644 dedal/enum/SpackCacheStorageEnum.py
diff --git a/dedal/build_cache/BuildCacheManagerCreator.py b/dedal/build_cache/BuildCacheManagerCreator.py
new file mode 100644
index 00000000..1f6355a9
--- /dev/null
+++ b/dedal/build_cache/BuildCacheManagerCreator.py
@@ -0,0 +1,41 @@
+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+# (c) Copyright 2025 Dedal developers
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from dedal.build_cache.BuildCacheManagerOci import BuildCacheManagerOci
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.enum.SpackCacheStorageEnum import SpackCacheStorageEnum
+
+
+class BuildCacheManagerCreator:
+ @staticmethod
+ def get_build_cache_manager_operator(spack_config: SpackConfig = SpackConfig()):
+ if spack_config.spack_cache_storage_type is None:
+ return None, None
+ elif spack_config.spack_cache_storage_type is SpackCacheStorageEnum.OCI:
+ cache_dependency = BuildCacheManagerOci(os.environ.get('CONCRETIZE_OCI_HOST'),
+ os.environ.get('CONCRETIZE_OCI_PROJECT'),
+ os.environ.get('CONCRETIZE_OCI_USERNAME'),
+ os.environ.get('CONCRETIZE_OCI_PASSWORD'),
+ cache_version=spack_config.cache_version_concretize)
+ build_cache = BuildCacheManagerOci(os.environ.get('BUILDCACHE_OCI_HOST'),
+ os.environ.get('BUILDCACHE_OCI_PROJECT'),
+ os.environ.get('BUILDCACHE_OCI_USERNAME'),
+ os.environ.get('BUILDCACHE_OCI_PASSWORD'),
+ cache_version=spack_config.cache_version_build)
+ return cache_dependency, build_cache
diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManagerOci.py
similarity index 99%
rename from dedal/build_cache/BuildCacheManager.py
rename to dedal/build_cache/BuildCacheManagerOci.py
index 91604332..df283455 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManagerOci.py
@@ -27,7 +27,7 @@ from dedal.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterf
from dedal.logger.logger_builder import get_logger
-class BuildCacheManager(BuildCacheManagerInterface):
+class BuildCacheManagerOci(BuildCacheManagerInterface):
"""
This class aims to manage the push/pull/delete of build cache files
"""
@@ -35,7 +35,7 @@ class BuildCacheManager(BuildCacheManagerInterface):
def __new__(cls, registry_host, registry_project, registry_username, registry_password, cache_version='cache',
auth_backend='basic', insecure=False, tls_verify=True):
instance = super().__new__(cls)
- instance._logger = get_logger(__name__, BuildCacheManager.__name__)
+ instance._logger = get_logger(__name__, BuildCacheManagerOci.__name__)
instance._registry_project = registry_project
instance._registry_username = registry_username
diff --git a/dedal/configuration/SpackConfig.py b/dedal/configuration/SpackConfig.py
index e4a258e9..98a70a41 100644
--- a/dedal/configuration/SpackConfig.py
+++ b/dedal/configuration/SpackConfig.py
@@ -18,6 +18,7 @@
import os
from pathlib import Path
from dedal.configuration.GpgConfig import GpgConfig
+from dedal.enum.SpackCacheStorageEnum import SpackCacheStorageEnum
from dedal.enum.SpackViewEnum import SpackViewEnum
from dedal.model import SpackDescriptor
from dedal.utils.utils import resolve_path
@@ -28,7 +29,8 @@ class SpackConfig:
install_dir=Path(os.getcwd()).resolve(), upstream_instance=None, system_name=None,
concretization_dir: Path = None, buildcache_dir: Path = None, gpg: GpgConfig = None,
use_spack_global=False, cache_version_concretize='v1',
- cache_version_build='v1', view=SpackViewEnum.VIEW, override_cache=True, bashrc_path=os.path.expanduser("~/.bashrc")):
+ cache_version_build='v1', view=SpackViewEnum.VIEW, override_cache=True, bashrc_path=os.path.expanduser("~/.bashrc"),
+ spack_cache_storage_type: SpackCacheStorageEnum = SpackCacheStorageEnum.OCI):
self.env = env
if repos is None:
self.repos = []
@@ -46,6 +48,7 @@ class SpackConfig:
self.view = view
self.override_cache = override_cache
self.bashrc_path = bashrc_path
+ self.spack_cache_storage_type = spack_cache_storage_type
def add_repo(self, repo: SpackDescriptor):
if self.repos is None:
diff --git a/dedal/enum/SpackCacheStorageEnum.py b/dedal/enum/SpackCacheStorageEnum.py
new file mode 100644
index 00000000..21f5c47f
--- /dev/null
+++ b/dedal/enum/SpackCacheStorageEnum.py
@@ -0,0 +1,25 @@
+# Dedal library - Wrapper over Spack for building multiple target
+# environments: ESD, Virtual Boxes, HPC compatible kernels, etc.
+
+# (c) Copyright 2025 Dedal developers
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from enum import Enum
+
+
+class SpackCacheStorageEnum(Enum):
+ """
+ Different types of storages used for caching
+ """
+ OCI = 'oci_cache'
diff --git a/dedal/spack_factory/SpackEnvOperation.py b/dedal/spack_factory/SpackEnvOperation.py
index 01955aff..659b9c72 100644
--- a/dedal/spack_factory/SpackEnvOperation.py
+++ b/dedal/spack_factory/SpackEnvOperation.py
@@ -40,14 +40,16 @@ class SpackEnvOperation:
else:
raise MissingAttributeException(f'Missing attribute for class {__name__}')
- def create_fetch_spack_environment(self):
+ def setup_spack_environment(self):
"""Fetches a spack environment if the git path is defined, otherwise creates it."""
if self.spack_config.env and self.spack_config.env.git_path:
- git_clone_repo(self.spack_config.env.name, self.spack_config.env.path / self.spack_config.env.name,
+ git_clone_repo(self.spack_config.env.name, self.env_path,
self.spack_config.env.git_path,
logger=self.logger)
+ elif self.env_path and self.env_path.exists():
+ return
else:
- os.makedirs(self.spack_config.env.path / self.spack_config.env.name, exist_ok=True)
+ os.makedirs(self.env_path, exist_ok=True)
if self.env_path:
run_command("bash", "-c",
f'{self.spack_setup_script} && spack env create -d {self.env_path}',
diff --git a/dedal/spack_factory/SpackOperation.py b/dedal/spack_factory/SpackOperation.py
index 3b3645bb..b112ed28 100644
--- a/dedal/spack_factory/SpackOperation.py
+++ b/dedal/spack_factory/SpackOperation.py
@@ -99,7 +99,7 @@ class SpackOperation:
self.logger.error(f'Invalid installation path: {self.spack_dir}')
# Restart the bash after adding environment variables
if self.spack_config.env:
- self.spack_env_operation.create_fetch_spack_environment()
+ self.spack_env_operation.setup_spack_environment()
if self.spack_config.install_dir.exists():
for repo in self.spack_config.repos:
repo_dir = self.spack_config.install_dir / repo.path / repo.name
@@ -156,7 +156,7 @@ class SpackOperation:
return install_result
def create_fetch_spack_environment(self):
- self.spack_env_operation.create_fetch_spack_environment()
+ self.spack_env_operation.setup_spack_environment()
def add_spack_repo(self, repo_path: Path, repo_name: str):
self.spack_env_operation.add_spack_repo(repo_path, repo_name)
diff --git a/dedal/spack_factory/SpackOperationCreateCache.py b/dedal/spack_factory/SpackOperationCreateCache.py
index 6b5e30db..ee437d76 100644
--- a/dedal/spack_factory/SpackOperationCreateCache.py
+++ b/dedal/spack_factory/SpackOperationCreateCache.py
@@ -17,9 +17,9 @@
import os
+from dedal.build_cache.BuildCacheManagerCreator import BuildCacheManagerCreator
from dedal.utils.utils import copy_file
from dedal.wrapper.spack_wrapper import check_spack_env
-from dedal.build_cache.BuildCacheManager import BuildCacheManager
from dedal.configuration.SpackConfig import SpackConfig
from dedal.logger.logger_builder import get_logger
from dedal.spack_factory.SpackOperation import SpackOperation
@@ -32,16 +32,8 @@ class SpackOperationCreateCache(SpackOperation):
def __init__(self, spack_config: SpackConfig = SpackConfig()):
super().__init__(spack_config, logger=get_logger(__name__))
- self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
- os.environ.get('CONCRETIZE_OCI_PROJECT'),
- os.environ.get('CONCRETIZE_OCI_USERNAME'),
- os.environ.get('CONCRETIZE_OCI_PASSWORD'),
- cache_version=spack_config.cache_version_concretize)
- self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
- os.environ.get('BUILDCACHE_OCI_PROJECT'),
- os.environ.get('BUILDCACHE_OCI_USERNAME'),
- os.environ.get('BUILDCACHE_OCI_PASSWORD'),
- cache_version=spack_config.cache_version_build)
+ self.cache_dependency, self.build_cache = BuildCacheManagerCreator.get_build_cache_manager_operator(
+ spack_config)
self.signed = False
if self.spack_config.gpg:
self.signed = True
diff --git a/dedal/spack_factory/SpackOperationUseCache.py b/dedal/spack_factory/SpackOperationUseCache.py
index b7c9952e..900bc4db 100644
--- a/dedal/spack_factory/SpackOperationUseCache.py
+++ b/dedal/spack_factory/SpackOperationUseCache.py
@@ -19,9 +19,8 @@ import os
import subprocess
from pathlib import Path
-from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.build_cache.BuildCacheManagerCreator import BuildCacheManagerCreator
from dedal.configuration.SpackConfig import SpackConfig
-from dedal.enum.SpackConfigCommand import SpackConfigCommand
from dedal.error_handling.exceptions import SpackInstallPackagesException
from dedal.logger.logger_builder import get_logger
from dedal.spack_factory.SpackOperation import SpackOperation
@@ -36,16 +35,8 @@ class SpackOperationUseCache(SpackOperation):
def __init__(self, spack_config: SpackConfig = SpackConfig()):
super().__init__(spack_config, logger=get_logger(__name__))
- self.cache_dependency = BuildCacheManager(os.environ.get('CONCRETIZE_OCI_HOST'),
- os.environ.get('CONCRETIZE_OCI_PROJECT'),
- os.environ.get('CONCRETIZE_OCI_USERNAME'),
- os.environ.get('CONCRETIZE_OCI_PASSWORD'),
- cache_version=spack_config.cache_version_concretize)
- self.build_cache = BuildCacheManager(os.environ.get('BUILDCACHE_OCI_HOST'),
- os.environ.get('BUILDCACHE_OCI_PROJECT'),
- os.environ.get('BUILDCACHE_OCI_USERNAME'),
- os.environ.get('BUILDCACHE_OCI_PASSWORD'),
- cache_version=spack_config.cache_version_build)
+ self.cache_dependency, self.build_cache = BuildCacheManagerCreator.get_build_cache_manager_operator(
+ spack_config)
def setup_spack_env(self) -> None:
"""Set up the spack environment for using the cache.
diff --git a/dedal/tests/integration_tests/spack_from_scratch_test.py b/dedal/tests/integration_tests/spack_from_scratch_test.py
index 77618922..a7f6b2c2 100644
--- a/dedal/tests/integration_tests/spack_from_scratch_test.py
+++ b/dedal/tests/integration_tests/spack_from_scratch_test.py
@@ -68,7 +68,7 @@ def test_spack_reindex(tmp_path):
spack_operation.reindex()
-@pytest.mark.skip(reason="It does nopt work on bare metal operating systems")
+@pytest.mark.skip(reason="It does not work on bare metal operating systems")
def test_spack_spec(tmp_path):
install_dir = tmp_path
config = SpackConfig(install_dir=install_dir)
diff --git a/dedal/tests/testing_variables.py b/dedal/tests/testing_variables.py
index 116a9446..62785350 100644
--- a/dedal/tests/testing_variables.py
+++ b/dedal/tests/testing_variables.py
@@ -18,6 +18,6 @@
import os
ebrains_spack_builds_git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git'
-SPACK_VERSION = "0.23.0"
+SPACK_VERSION = "0.23.1"
SPACK_ENV_ACCESS_TOKEN = os.getenv("SPACK_ENV_ACCESS_TOKEN")
test_spack_env_git = f'https://oauth2:{SPACK_ENV_ACCESS_TOKEN}@gitlab.ebrains.eu/ri/projects-and-initiatives/virtualbraintwin/tools/test-spack-env.git'
diff --git a/dedal/tests/unit_tests/build_cache_manager_test.py b/dedal/tests/unit_tests/build_cache_manager_test.py
index 57e61498..1f459d0a 100644
--- a/dedal/tests/unit_tests/build_cache_manager_test.py
+++ b/dedal/tests/unit_tests/build_cache_manager_test.py
@@ -18,15 +18,15 @@
import pytest
from _pytest.fixtures import fixture
-from dedal.build_cache.BuildCacheManager import BuildCacheManager
+from dedal.build_cache.BuildCacheManagerOci import BuildCacheManagerOci
-class TestBuildCacheManager:
+class TestBuildCacheManagerOci:
@fixture(scope="function")
def mock_build_cache_manager(self, mocker):
- mocker.patch("dedal.build_cache.BuildCacheManager.get_logger")
- return BuildCacheManager("TEST_HOST", "TEST_PROJECT", "TEST_USERNAME", "TEST_PASSWORD", "TEST_VERSION")
+ mocker.patch("dedal.build_cache.BuildCacheManagerOci.get_logger")
+ return BuildCacheManagerOci("TEST_HOST", "TEST_PROJECT", "TEST_USERNAME", "TEST_PASSWORD", "TEST_VERSION")
def test_get_public_key_from_cache_success_path(self, mock_build_cache_manager, tmp_path):
@@ -149,7 +149,7 @@ class TestBuildCacheManager:
warn_message = "test message"
# Act
- mock_build_cache_manager._BuildCacheManager__log_warning_if_needed(warn_message, items)
+ mock_build_cache_manager._BuildCacheManagerOci__log_warning_if_needed(warn_message, items)
# Assert
mock_build_cache_manager._logger.warning.assert_called_once_with(warn_message, items, items[0])
@@ -165,7 +165,7 @@ class TestBuildCacheManager:
warn_message = "test message"
# Act
- mock_build_cache_manager._BuildCacheManager__log_warning_if_needed(warn_message, items)
+ mock_build_cache_manager._BuildCacheManagerOci__log_warning_if_needed(warn_message, items)
# Assert
mock_build_cache_manager._logger.warning.assert_not_called()
--
GitLab