diff --git a/dedal/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
index e1bd6824321c5a2b483800bc4b39824aa4715493..839ac52400cae71fa760672321de51761c9fd1a8 100644
--- a/dedal/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -13,47 +13,47 @@ class BuildCacheManager(BuildCacheManagerInterface):
"""
def __init__(self, auth_backend='basic', insecure=False):
- self.logger = get_logger(__name__, BuildCacheManager.__name__)
- self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.registry_project = os.environ.get("REGISTRY_PROJECT")
+ self._logger = get_logger(__name__, BuildCacheManager.__name__)
+ self._home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
+ self._registry_project = os.environ.get("REGISTRY_PROJECT")
self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
- self.registry_host = str(os.environ.get("REGISTRY_HOST"))
+ self._registry_host = str(os.environ.get("REGISTRY_HOST"))
# Initialize an OrasClient instance.
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
- self.client.login(username=self._registry_username, password=self._registry_password)
- self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
+ self._client = oras.client.OrasClient(hostname=self._registry_host, auth_backend=auth_backend, insecure=insecure)
+ self._client.login(username=self._registry_username, password=self._registry_password)
+ self._oci_registry_path = f'{self._registry_host}/{self._registry_project}/cache'
def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / out_dir
+ build_cache_path = self._home_path / out_dir
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- self.logger.error(f"Path {build_cache_path} not found.")
+ self._logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.env_name), "")
- target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.env_name)}"
+ target = f"{self._registry_host}/{self._registry_project}/cache:{str(sub_path.env_name)}"
try:
- self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
- self.client.push(
+ self._logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self._client.push(
files=[str(sub_path)],
target=target,
# save in manifest the relative path for reconstruction
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- self.logger.info(f"Successfully pushed {sub_path.env_name}")
+ self._logger.info(f"Successfully pushed {sub_path.env_name}")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"An error occurred while pushing: {e}")
# todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
# clean_up([str(build_cache_path)], self.logger)
@@ -63,37 +63,37 @@ class BuildCacheManager(BuildCacheManagerInterface):
This method retrieves all tags from an OCI Registry
"""
try:
- return self.client.get_tags(self.oci_registry_path)
+ return self._client.get_tags(self._oci_registry_path)
except Exception as e:
- self.logger.error(f"Failed to list tags: {e}")
+ self._logger.error(f"Failed to list tags: {e}")
return None
def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / in_dir
+ build_cache_path = self._home_path / in_dir
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
tags = self.list_tags()
if tags is not None:
for tag in tags:
- ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
+ ref = f"{self._registry_host}/{self._registry_project}/cache:{tag}"
# reconstruct the relative path of each artifact by getting it from the manifest
cache_path = \
- self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
+ self._client.get_manifest(f'{self._registry_host}/{self._registry_project}/cache:{tag}')[
'annotations'][
'path']
try:
- self.client.pull(
+ self._client.pull(
ref,
# missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- self.logger.info(f"Successfully pulled artifact {tag}.")
+ self._logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- self.logger.error(
+ self._logger.error(
f"Failed to pull artifact {tag} : {e}")
def delete(self):
@@ -106,8 +106,8 @@ class BuildCacheManager(BuildCacheManagerInterface):
tags = self.list_tags()
if tags is not None:
try:
- self.client.delete_tags(self.oci_registry_path, tags)
- self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
+ self._client.delete_tags(self._oci_registry_path, tags)
+ self._logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- self.logger.error(
+ self._logger.error(
f"Failed to delete artifacts: {e}")
diff --git a/esd/spack_manager/SpackManager.py b/esd/spack_manager/SpackManager.py
index 65a21d6e224cb7b3c0d4150de1fefa7adbe97b4f..bf381c394ca276e3e32664caa2319feec5077e7e 100644
--- a/esd/spack_manager/SpackManager.py
+++ b/esd/spack_manager/SpackManager.py
@@ -10,7 +10,7 @@ from esd.error_handling.exceptions import BashCommandException, NoSpackEnvironme
from esd.logger.logger_builder import get_logger
from esd.model.SpackModel import SpackModel
from esd.spack_manager.wrapper.spack_wrapper import check_spack_env
-from esd.utils.utils import run_command, git_clone_repo
+from esd.utils.utils import run_command, git_clone_repo, log_command
class SpackManager(ABC):
@@ -178,10 +178,7 @@ class SpackManager(ABC):
debug_msg=f"Installing spack packages for {self.env.env_name}",
exception_msg=f"Error installing spack packages for {self.env.env_name}",
exception=SpackInstallPackagesException)
- with open(str(Path(os.getcwd()).resolve() / ".generate_cache.log"), "w") as log_file:
- log_file.write(install_result.stdout)
- log_file.write("\n--- STDERR ---\n")
- log_file.write(install_result.stderr)
+ log_command(install_result, str(Path(os.getcwd()).resolve() / ".generate_cache.log"))
return install_result
def install_spack(self, spack_version="v0.22.0", spack_repo='https://github.com/spack/spack'):
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index 173347d06b08f6092a06b4590fd969d0fb66535e..3ce70f252a4b5ff701c76b59eeaf3d1c703799d2 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -1,4 +1,5 @@
import logging
+import os
import shutil
import subprocess
from pathlib import Path
@@ -55,3 +56,10 @@ def git_clone_repo(repo_name: str, dir: Path, git_path: str, logger: logging = l
def file_exists_and_not_empty(file: Path) -> bool:
return file.is_file() and file.stat().st_size > 0
+
+
+def log_command(results, log_file: str):
+ with open(log_file, "w") as log_file:
+ log_file.write(results.stdout)
+ log_file.write("\n--- STDERR ---\n")
+ log_file.write(results.stderr)