From 6471bd2dec7c3303828831b9be1670a921f2bb0a Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Mon, 27 Jan 2025 12:20:56 +0200
Subject: [PATCH] esd: manage build cache: added support for http and https;
logger; generate wheel and code restructure
---
.gitlab-ci.yml | 25 +++----
...ge_build_cache.py => BuildCacheManager.py} | 68 ++++++++-----------
esd/build_cache/BuildCacheManagerInterface.py | 17 +++++
esd/logger/__init__.py | 0
esd/logger/logger_builder.py | 51 ++++++++++++++
esd/logger/logging.conf | 55 +++++++++++++++
esd/utils/utils.py | 9 ++-
pyproject.toml | 5 +-
8 files changed, 176 insertions(+), 54 deletions(-)
rename esd/build_cache/{manage_build_cache.py => BuildCacheManager.py} (63%)
create mode 100644 esd/build_cache/BuildCacheManagerInterface.py
create mode 100644 esd/logger/__init__.py
create mode 100644 esd/logger/logger_builder.py
create mode 100644 esd/logger/logging.conf
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7dea24a3..9b2e92b5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,26 +5,27 @@ stages:
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
-build-spack-env-on-runner:
+build-wheel:
stage: build
tags:
- - docker-runner # esd_image
- image: $BUILD_ENV_DOCKER_IMAGE
+ - docker-runner
+ image: python:latest
+ before_script:
+ - python -m pip install --upgrade pip setuptools wheel build
script:
- - /usr/sbin/sysctl user.max_user_namespaces
- - /usr/sbin/sysctl kernel.unprivileged_userns_clone
- #- buildah build --isolation=chroot --runtime=crun --network=host --storage-opt="network.network_backend=cni" --storage-opt="overlay.mount_program=/usr/bin/fuse-overlayfs" -f Dockerfile .
- #- export APPTAINER_VERSION="1.3.6"
- #- |
- # mkdir -p apptainer-install/
- # curl -s https://raw.githubusercontent.com/apptainer/apptainer/main/tools/install-unprivileged.sh | bash -s - apptainer-install/
- - apptainer version
+ - python -m build --sdist --wheel
+ artifacts:
+ paths:
+ - dist/*.whl
+ - dist/*.tar.gz
+ expire_in: 1 week
+
testing:
stage: test
tags:
- docker-runner
- image: python:latest # $BUILD_ENV_DOCKER_IMAGE
+ image: python:latest
script:
- pip install -e .
- pytest ./esd/tests/ --junitxml=test-results.xml
diff --git a/esd/build_cache/manage_build_cache.py b/esd/build_cache/BuildCacheManager.py
similarity index 63%
rename from esd/build_cache/manage_build_cache.py
rename to esd/build_cache/BuildCacheManager.py
index 9697e88d..44e13b73 100644
--- a/esd/build_cache/manage_build_cache.py
+++ b/esd/build_cache/BuildCacheManager.py
@@ -1,19 +1,20 @@
import os
-import logging
import oras.client
from pathlib import Path
+from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from esd.logger.logger_builder import get_logger
from esd.utils.utils import clean_up
-class BuildCacheManager:
+class BuildCacheManager(BuildCacheManagerInterface):
"""
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', log_path='./'):
+ def __init__(self, auth_backend='basic', insecure=False):
+ self.logger = get_logger(__name__, BuildCacheManager.__name__)
self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.log_file = Path(log_path) / "log_oras.txt"
self.registry_project = os.environ.get("REGISTRY_PROJECT")
self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
@@ -24,36 +25,25 @@ class BuildCacheManager:
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend)
+ self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
self.client.login(username=self._registry_username, password=self._registry_password)
self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
- logging.basicConfig(
- level=logging.INFO,
- format='%(asctime)s - %(levelname)s - %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S',
- handlers=[
- logging.FileHandler(self.log_file),
- logging.StreamHandler()
- ]
- )
-
- def oci_registry_upload_build_cache(self, cache_dir: Path):
+ def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / "shared" / cache_dir
+ build_cache_path = self.home_path / out_dir
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- raise FileNotFoundError(
- f"BuildCacheManager::oci_registry_upload_build_cache::Path {build_cache_path} not found.")
+ self.logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
try:
- logging.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
self.client.push(
files=[str(sub_path)],
target=target,
@@ -61,31 +51,31 @@ class BuildCacheManager:
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- logging.info(f"Successfully pushed {sub_path.name}")
+ self.logger.info(f"Successfully pushed {sub_path.name}")
except Exception as e:
- logging.error(
- f"BuildCacheManager::registry_upload_build_cache::An error occurred while pushing: {e}")
- # delete the build cache after being pushed to the OCI Registry
- clean_up([str(build_cache_path)], logging)
+ self.logger.error(
+ f"An error occurred while pushing: {e}")
+ # todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
+ # clean_up([str(build_cache_path)], self.logger)
- def oci_registry_get_tags(self):
+ def list_tags(self):
"""
This method retrieves all tags from an OCI Registry
"""
try:
return self.client.get_tags(self.oci_registry_path)
except Exception as e:
- logging.error(f"BuildCacheManager::oci_registry_get_tags::Failed to list tags: {e}")
+ self.logger.error(f"Failed to list tags: {e}")
return None
- def oci_registry_download_build_cache(self, cache_dir: Path):
+ def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / "shared" / cache_dir
+ build_cache_path = self.home_path / in_dir
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
- tags = self.oci_registry_get_tags()
+ tags = self.list_tags()
if tags is not None:
for tag in tags:
ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
@@ -97,27 +87,27 @@ class BuildCacheManager:
try:
self.client.pull(
ref,
- # missing dirs to outdir are created automatically by OrasClient pull method
+ # missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- logging.info(f"Successfully pulled artifact {tag}.")
+ self.logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- logging.error(
- f"BuildCacheManager::registry_download_build_cache::Failed to pull artifact {tag} : {e}")
+ self.logger.error(
+ f"Failed to pull artifact {tag} : {e}")
- def oci_registry_delete_build_cache(self):
+ def delete(self):
"""
Deletes all artifacts from an OCI Registry based on their tags.
This method removes artifacts identified by their tags in the specified OCI Registry.
It requires appropriate permissions to delete artifacts from the registry.
If the registry or user does not have the necessary delete permissions, the operation might fail.
"""
- tags = self.oci_registry_get_tags()
+ tags = self.list_tags()
if tags is not None:
try:
self.client.delete_tags(self.oci_registry_path, tags)
- logging.info(f"Successfully deleted all artifacts form OCI registry.")
+ self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- logging.error(
- f"BuildCacheManager::registry_delete_build_cache::Failed to delete artifacts: {e}")
+ self.logger.error(
+ f"Failed to delete artifacts: {e}")
diff --git a/esd/build_cache/BuildCacheManagerInterface.py b/esd/build_cache/BuildCacheManagerInterface.py
new file mode 100644
index 00000000..3016590b
--- /dev/null
+++ b/esd/build_cache/BuildCacheManagerInterface.py
@@ -0,0 +1,17 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class BuildCacheManagerInterface(ABC):
+
+ @abstractmethod
+ def upload(self, out_dir: Path):
+ pass
+
+ @abstractmethod
+ def download(self, in_dir: Path):
+ pass
+
+ @abstractmethod
+ def delete(self):
+ pass
diff --git a/esd/logger/__init__.py b/esd/logger/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/logger/logger_builder.py b/esd/logger/logger_builder.py
new file mode 100644
index 00000000..c15a2f0e
--- /dev/null
+++ b/esd/logger/logger_builder.py
@@ -0,0 +1,51 @@
+import os
+import inspect
+import weakref
+import logging
+import logging.config
+
+
+class LoggerBuilder(object):
+ """
+ Class taking care of uniform Python logger initialization.
+ It uses the Python native logging package.
+ It's purpose is just to offer a common mechanism for initializing all modules in a package.
+ """
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(self, config_file_name='logging.conf'):
+ """
+ Prepare Python logger based on a configuration file.
+ :param: config_file_name - name of the logging configuration relative to the current package
+ """
+ current_folder = os.path.dirname(inspect.getfile(self.__class__))
+ config_file_path = os.path.join(current_folder, config_file_name)
+ logging.config.fileConfig(config_file_path, disable_existing_loggers=False)
+ self._loggers = weakref.WeakValueDictionary()
+
+ def build_logger(self, parent_module, parent_class):
+ """
+ Build a logger instance and return it
+ """
+ logger_key = f'{parent_module}.{parent_class}' if parent_class else parent_module
+ self._loggers[logger_key] = logger = logging.getLogger(logger_key)
+ return logger
+
+ def set_loggers_level(self, level):
+ for logger in self._loggers.values():
+ logger.setLevel(level)
+
+
+def get_logger(parent_module='', parent_class=None):
+ """
+ Function to retrieve a new Python logger instance for current module.
+
+ :param parent_module: module name for which to create logger.
+ :param parent_class: class name for which to create logger.
+ """
+ return LoggerBuilder().build_logger(parent_module, parent_class)
diff --git a/esd/logger/logging.conf b/esd/logger/logging.conf
new file mode 100644
index 00000000..d95ba87c
--- /dev/null
+++ b/esd/logger/logging.conf
@@ -0,0 +1,55 @@
+############################################
+## ESD - logging configuration. ##
+############################################
+[loggers]
+keys=root, esd, oras
+
+[handlers]
+keys=consoleHandler, fileHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=WARNING
+handlers=consoleHandler, fileHandler
+propagate=0
+
+############################################
+## esd specific logging ##
+############################################
+[logger_esd]
+level=DEBUG
+handlers=consoleHandler, fileHandler
+qualname=esd
+propagate=0
+
+[logger_oras]
+level=ERROR
+handlers=consoleHandler
+qualname=oras
+propagate=0
+
+############################################
+## Handlers ##
+############################################
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=handlers.TimedRotatingFileHandler
+level=INFO
+formatter=simpleFormatter
+args=('.esd.log', 'midnight', 1, 30, None, False, False)
+
+############################################
+## Formatters ##
+############################################
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(levelname)s - %(name)s::%(funcName)s - %(message)s
+datefmt = %d-%m-%Y %I:%M:%S
\ No newline at end of file
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index c1aa4c0d..811d258e 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -2,7 +2,7 @@ import shutil
from pathlib import Path
-def clean_up(dirs: list[str], logging):
+def clean_up(dirs: list[str], logging, ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
@@ -10,6 +10,11 @@ def clean_up(dirs: list[str], logging):
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
logging.info(f"Removing {cleanup_dir}")
- shutil.rmtree(Path(cleanup_dir))
+ try:
+ shutil.rmtree(Path(cleanup_dir))
+ except OSError as e:
+ logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ if not ignore_errors:
+ raise e
else:
logging.info(f"{cleanup_dir} does not exist")
diff --git a/pyproject.toml b/pyproject.toml
index cd4afe54..0f6d0cde 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,4 +18,7 @@ dependencies = [
"ruamel.yaml",
"pytest",
"pytest-mock",
-]
\ No newline at end of file
+]
+
+[tool.setuptools.data-files]
+"esd-tools" = ["esd/logger/logging.conf"]
\ No newline at end of file
--
GitLab