From de485ef194c54704d1d2d86c2c11dcb5946694e1 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 21 Jan 2025 23:06:12 +0200
Subject: [PATCH 1/4] esd: packaged the project (pyproject.toml) and added
build cache python scripts
---
esd/__init__.py | 0
esd/build_cache/__init__.py | 0
esd/build_cache/manage_build_cache.py | 123 ++++++++++++++++++
.../fetch_cached_buildresults.py | 0
.../fetch_cached_sources.py | 0
.../specfile_dag_hash.py | 2 +-
.../specfile_storage_path_build.py | 0
.../specfile_storage_path_source.py | 0
.../update_cached_buildresults.py | 1 -
.../update_cached_sources.py | 0
esd/utils/__init__.py | 0
esd/utils/utils.py | 15 +++
wscript => esd/wscript | 0
pyproject.toml | 19 +++
14 files changed, 158 insertions(+), 2 deletions(-)
create mode 100644 esd/__init__.py
create mode 100644 esd/build_cache/__init__.py
create mode 100644 esd/build_cache/manage_build_cache.py
rename fetch_cached_buildresults.py => esd/fetch_cached_buildresults.py (100%)
rename fetch_cached_sources.py => esd/fetch_cached_sources.py (100%)
rename specfile_dag_hash.py => esd/specfile_dag_hash.py (96%)
rename specfile_storage_path_build.py => esd/specfile_storage_path_build.py (100%)
rename specfile_storage_path_source.py => esd/specfile_storage_path_source.py (100%)
rename update_cached_buildresults.py => esd/update_cached_buildresults.py (99%)
rename update_cached_sources.py => esd/update_cached_sources.py (100%)
create mode 100644 esd/utils/__init__.py
create mode 100644 esd/utils/utils.py
rename wscript => esd/wscript (100%)
mode change 100755 => 100644
create mode 100644 pyproject.toml
diff --git a/esd/__init__.py b/esd/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/build_cache/__init__.py b/esd/build_cache/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/build_cache/manage_build_cache.py b/esd/build_cache/manage_build_cache.py
new file mode 100644
index 00000000..9697e88d
--- /dev/null
+++ b/esd/build_cache/manage_build_cache.py
@@ -0,0 +1,123 @@
+import os
+import logging
+import oras.client
+from pathlib import Path
+
+from esd.utils.utils import clean_up
+
+
+class BuildCacheManager:
+ """
+ This class aims to manage the push/pull/delete of build cache files
+ """
+
+ def __init__(self, auth_backend='basic', log_path='./'):
+ self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
+ self.log_file = Path(log_path) / "log_oras.txt"
+ self.registry_project = os.environ.get("REGISTRY_PROJECT")
+
+ self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
+ self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
+
+ self.registry_host = str(os.environ.get("REGISTRY_HOST"))
+ # Initialize an OrasClient instance.
+ # This method utilizes the OCI Registry for container image and artifact management.
+ # Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
+ # Supported authentication types may include basic authentication (username/password), token-based authentication,
+ self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend)
+ self.client.login(username=self._registry_username, password=self._registry_password)
+ self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
+
+ logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(levelname)s - %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ handlers=[
+ logging.FileHandler(self.log_file),
+ logging.StreamHandler()
+ ]
+ )
+
+ def oci_registry_upload_build_cache(self, cache_dir: Path):
+ """
+ This method pushed all the files from the build cache folder into the OCI Registry
+ """
+ build_cache_path = self.home_path / "shared" / cache_dir
+ # build cache folder must exist before pushing all the artifacts
+ if not build_cache_path.exists():
+ raise FileNotFoundError(
+ f"BuildCacheManager::oci_registry_upload_build_cache::Path {build_cache_path} not found.")
+
+ for sub_path in build_cache_path.rglob("*"):
+ if sub_path.is_file():
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
+ target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
+ try:
+ logging.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self.client.push(
+ files=[str(sub_path)],
+ target=target,
+ # save in manifest the relative path for reconstruction
+ manifest_annotations={"path": rel_path},
+ disable_path_validation=True,
+ )
+ logging.info(f"Successfully pushed {sub_path.name}")
+ except Exception as e:
+ logging.error(
+ f"BuildCacheManager::registry_upload_build_cache::An error occurred while pushing: {e}")
+ # delete the build cache after being pushed to the OCI Registry
+ clean_up([str(build_cache_path)], logging)
+
+ def oci_registry_get_tags(self):
+ """
+ This method retrieves all tags from an OCI Registry
+ """
+ try:
+ return self.client.get_tags(self.oci_registry_path)
+ except Exception as e:
+ logging.error(f"BuildCacheManager::oci_registry_get_tags::Failed to list tags: {e}")
+ return None
+
+ def oci_registry_download_build_cache(self, cache_dir: Path):
+ """
+ This method pulls all the files from the OCI Registry into the build cache folder
+ """
+ build_cache_path = self.home_path / "shared" / cache_dir
+ # create the buildcache dir if it does not exist
+ os.makedirs(build_cache_path, exist_ok=True)
+ tags = self.oci_registry_get_tags()
+ if tags is not None:
+ for tag in tags:
+ ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
+ # reconstruct the relative path of each artifact by getting it from the manifest
+ cache_path = \
+ self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
+ 'annotations'][
+ 'path']
+ try:
+ self.client.pull(
+ ref,
+ # missing dirs to outdir are created automatically by OrasClient pull method
+ outdir=str(build_cache_path / cache_path),
+ overwrite=True
+ )
+ logging.info(f"Successfully pulled artifact {tag}.")
+ except Exception as e:
+ logging.error(
+ f"BuildCacheManager::registry_download_build_cache::Failed to pull artifact {tag} : {e}")
+
+ def oci_registry_delete_build_cache(self):
+ """
+ Deletes all artifacts from an OCI Registry based on their tags.
+ This method removes artifacts identified by their tags in the specified OCI Registry.
+ It requires appropriate permissions to delete artifacts from the registry.
+ If the registry or user does not have the necessary delete permissions, the operation might fail.
+ """
+ tags = self.oci_registry_get_tags()
+ if tags is not None:
+ try:
+ self.client.delete_tags(self.oci_registry_path, tags)
+ logging.info(f"Successfully deleted all artifacts form OCI registry.")
+ except RuntimeError as e:
+ logging.error(
+ f"BuildCacheManager::registry_delete_build_cache::Failed to delete artifacts: {e}")
diff --git a/fetch_cached_buildresults.py b/esd/fetch_cached_buildresults.py
similarity index 100%
rename from fetch_cached_buildresults.py
rename to esd/fetch_cached_buildresults.py
diff --git a/fetch_cached_sources.py b/esd/fetch_cached_sources.py
similarity index 100%
rename from fetch_cached_sources.py
rename to esd/fetch_cached_sources.py
diff --git a/specfile_dag_hash.py b/esd/specfile_dag_hash.py
similarity index 96%
rename from specfile_dag_hash.py
rename to esd/specfile_dag_hash.py
index 6e001b84..e44e1c62 100644
--- a/specfile_dag_hash.py
+++ b/esd/specfile_dag_hash.py
@@ -3,7 +3,7 @@ from collections.abc import Iterable
import pathlib
import ruamel.yaml as yaml
import spack
-import spack.binary_distribution as bindist
+
parser = argparse.ArgumentParser(
prog='specfile_dag_hash.py',
diff --git a/specfile_storage_path_build.py b/esd/specfile_storage_path_build.py
similarity index 100%
rename from specfile_storage_path_build.py
rename to esd/specfile_storage_path_build.py
diff --git a/specfile_storage_path_source.py b/esd/specfile_storage_path_source.py
similarity index 100%
rename from specfile_storage_path_source.py
rename to esd/specfile_storage_path_source.py
diff --git a/update_cached_buildresults.py b/esd/update_cached_buildresults.py
similarity index 99%
rename from update_cached_buildresults.py
rename to esd/update_cached_buildresults.py
index caacf86e..58af242b 100644
--- a/update_cached_buildresults.py
+++ b/esd/update_cached_buildresults.py
@@ -1,5 +1,4 @@
import argparse
-import glob
import os
import pathlib
import subprocess
diff --git a/update_cached_sources.py b/esd/update_cached_sources.py
similarity index 100%
rename from update_cached_sources.py
rename to esd/update_cached_sources.py
diff --git a/esd/utils/__init__.py b/esd/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
new file mode 100644
index 00000000..c1aa4c0d
--- /dev/null
+++ b/esd/utils/utils.py
@@ -0,0 +1,15 @@
+import shutil
+from pathlib import Path
+
+
+def clean_up(dirs: list[str], logging):
+ """
+ All the folders from the list dirs are removed with all the content in them
+ """
+ for cleanup_dir in dirs:
+ cleanup_dir = Path(cleanup_dir).resolve()
+ if cleanup_dir.exists():
+ logging.info(f"Removing {cleanup_dir}")
+ shutil.rmtree(Path(cleanup_dir))
+ else:
+ logging.info(f"{cleanup_dir} does not exist")
diff --git a/wscript b/esd/wscript
old mode 100755
new mode 100644
similarity index 100%
rename from wscript
rename to esd/wscript
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..dd9ac102
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "esd"
+authors = [
+ {name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
+ {name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"},
+]
+description = "This package provides all the necessary tools to create an Ebrains Software Distribution environment"
+version = "0.1.0"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+ "oras",
+ "spack",
+ "ruamel.yaml"
+]
\ No newline at end of file
--
GitLab
From a7d409650c8e586deb8a4e5353aa03e63550c596 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 22 Jan 2025 17:07:43 +0200
Subject: [PATCH 2/4] esd: added test stage in CI/CD pipeline
---
.gitlab-ci.yml | 19 ++++++-
esd/cli/__init__.py | 0
esd/{ => cli}/fetch_cached_buildresults.py | 0
esd/tests/__init__.py | 0
esd/tests/utils_test.py | 63 ++++++++++++++++++++++
esd/wscript | 21 --------
pyproject.toml | 4 +-
7 files changed, 84 insertions(+), 23 deletions(-)
create mode 100644 esd/cli/__init__.py
rename esd/{ => cli}/fetch_cached_buildresults.py (100%)
create mode 100644 esd/tests/__init__.py
create mode 100644 esd/tests/utils_test.py
delete mode 100644 esd/wscript
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b8ce57f4..7dea24a3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,6 @@
stages:
- build
+ - test
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
@@ -7,7 +8,7 @@ variables:
build-spack-env-on-runner:
stage: build
tags:
- - esd_image
+ - docker-runner # esd_image
image: $BUILD_ENV_DOCKER_IMAGE
script:
- /usr/sbin/sysctl user.max_user_namespaces
@@ -18,3 +19,19 @@ build-spack-env-on-runner:
# mkdir -p apptainer-install/
# curl -s https://raw.githubusercontent.com/apptainer/apptainer/main/tools/install-unprivileged.sh | bash -s - apptainer-install/
- apptainer version
+
+testing:
+ stage: test
+ tags:
+ - docker-runner
+ image: python:latest # $BUILD_ENV_DOCKER_IMAGE
+ script:
+ - pip install -e .
+ - pytest ./esd/tests/ --junitxml=test-results.xml
+ artifacts:
+ when: always
+ reports:
+ junit: test-results.xml
+ paths:
+ - test-results.xml
+ expire_in: 1 week
\ No newline at end of file
diff --git a/esd/cli/__init__.py b/esd/cli/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/fetch_cached_buildresults.py b/esd/cli/fetch_cached_buildresults.py
similarity index 100%
rename from esd/fetch_cached_buildresults.py
rename to esd/cli/fetch_cached_buildresults.py
diff --git a/esd/tests/__init__.py b/esd/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/tests/utils_test.py b/esd/tests/utils_test.py
new file mode 100644
index 00000000..8bec6c58
--- /dev/null
+++ b/esd/tests/utils_test.py
@@ -0,0 +1,63 @@
+import pytest
+from pathlib import Path
+
+from esd.utils.utils import clean_up
+
+
+@pytest.fixture
+def temp_directories(tmp_path):
+ """
+ Create temporary directories with files and subdirectories for testing.
+ """
+ test_dirs = []
+
+ for i in range(3):
+ dir_path = tmp_path / f"test_dir_{i}"
+ dir_path.mkdir()
+ test_dirs.append(str(dir_path))
+
+ # Add a file to the directory
+ file_path = dir_path / f"file_{i}.txt"
+ file_path.write_text(f"This is a test file in {dir_path}")
+
+ # Add a subdirectory with a file
+ sub_dir = dir_path / f"subdir_{i}"
+ sub_dir.mkdir()
+ sub_file = sub_dir / f"sub_file_{i}.txt"
+ sub_file.write_text(f"This is a sub file in {sub_dir}")
+
+ return test_dirs
+
+
+def test_clean_up(temp_directories, mocker):
+ """
+ Test the clean_up function to ensure directories and contents are removed.
+ """
+ # Mock the logger using pytest-mock's mocker fixture
+ mock_logger = mocker.MagicMock()
+
+ # Ensure directories exist before calling clean_up
+ for dir_path in temp_directories:
+ assert Path(dir_path).exists()
+
+ clean_up(temp_directories, mock_logger)
+
+ for dir_path in temp_directories:
+ assert not Path(dir_path).exists()
+
+ for dir_path in temp_directories:
+ mock_logger.info.assert_any_call(f"Removing {Path(dir_path).resolve()}")
+
+
+def test_clean_up_nonexistent_dirs(mocker):
+ """
+ Test the clean_up function with nonexistent directories.
+ """
+ # Mock the logger using pytest-mock's mocker fixture
+ mock_logger = mocker.MagicMock()
+ nonexistent_dirs = ["nonexistent_dir_1", "nonexistent_dir_2"]
+
+ clean_up(nonexistent_dirs, mock_logger)
+
+ for dir_path in nonexistent_dirs:
+ mock_logger.info.assert_any_call(f"{Path(dir_path).resolve()} does not exist")
diff --git a/esd/wscript b/esd/wscript
deleted file mode 100644
index 7db8e31e..00000000
--- a/esd/wscript
+++ /dev/null
@@ -1,21 +0,0 @@
-def depends(ctx):
- ctx("spack", branch="visionary")
-
-def options(opt):
- pass
-
-def configure(cfg):
- pass
-
-def build(bld):
- # install /bin
- for bin in bld.path.ant_glob('bin/**/*'):
- bld.install_as('${PREFIX}/%s' % bin.path_from(bld.path), bin)
-
- # install /lib
- for lib in bld.path.ant_glob('lib/**/*'):
- bld.install_as('${PREFIX}/%s' % lib.path_from(bld.path), lib)
-
- # install /share
- for share in bld.path.ant_glob('share/**/*'):
- bld.install_as('${PREFIX}/%s' % share.path_from(bld.path), share)
diff --git a/pyproject.toml b/pyproject.toml
index dd9ac102..cd4afe54 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,5 +15,7 @@ requires-python = ">=3.10"
dependencies = [
"oras",
"spack",
- "ruamel.yaml"
+ "ruamel.yaml",
+ "pytest",
+ "pytest-mock",
]
\ No newline at end of file
--
GitLab
From 6471bd2dec7c3303828831b9be1670a921f2bb0a Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Mon, 27 Jan 2025 12:20:56 +0200
Subject: [PATCH 3/4] esd: manage build cache: added support for http and
https; logger; generate wheel and code restructure
---
.gitlab-ci.yml | 25 +++----
...ge_build_cache.py => BuildCacheManager.py} | 68 ++++++++-----------
esd/build_cache/BuildCacheManagerInterface.py | 17 +++++
esd/logger/__init__.py | 0
esd/logger/logger_builder.py | 51 ++++++++++++++
esd/logger/logging.conf | 55 +++++++++++++++
esd/utils/utils.py | 9 ++-
pyproject.toml | 5 +-
8 files changed, 176 insertions(+), 54 deletions(-)
rename esd/build_cache/{manage_build_cache.py => BuildCacheManager.py} (63%)
create mode 100644 esd/build_cache/BuildCacheManagerInterface.py
create mode 100644 esd/logger/__init__.py
create mode 100644 esd/logger/logger_builder.py
create mode 100644 esd/logger/logging.conf
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7dea24a3..9b2e92b5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,26 +5,27 @@ stages:
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
-build-spack-env-on-runner:
+build-wheel:
stage: build
tags:
- - docker-runner # esd_image
- image: $BUILD_ENV_DOCKER_IMAGE
+ - docker-runner
+ image: python:latest
+ before_script:
+ - python -m pip install --upgrade pip setuptools wheel build
script:
- - /usr/sbin/sysctl user.max_user_namespaces
- - /usr/sbin/sysctl kernel.unprivileged_userns_clone
- #- buildah build --isolation=chroot --runtime=crun --network=host --storage-opt="network.network_backend=cni" --storage-opt="overlay.mount_program=/usr/bin/fuse-overlayfs" -f Dockerfile .
- #- export APPTAINER_VERSION="1.3.6"
- #- |
- # mkdir -p apptainer-install/
- # curl -s https://raw.githubusercontent.com/apptainer/apptainer/main/tools/install-unprivileged.sh | bash -s - apptainer-install/
- - apptainer version
+ - python -m build --sdist --wheel
+ artifacts:
+ paths:
+ - dist/*.whl
+ - dist/*.tar.gz
+ expire_in: 1 week
+
testing:
stage: test
tags:
- docker-runner
- image: python:latest # $BUILD_ENV_DOCKER_IMAGE
+ image: python:latest
script:
- pip install -e .
- pytest ./esd/tests/ --junitxml=test-results.xml
diff --git a/esd/build_cache/manage_build_cache.py b/esd/build_cache/BuildCacheManager.py
similarity index 63%
rename from esd/build_cache/manage_build_cache.py
rename to esd/build_cache/BuildCacheManager.py
index 9697e88d..44e13b73 100644
--- a/esd/build_cache/manage_build_cache.py
+++ b/esd/build_cache/BuildCacheManager.py
@@ -1,19 +1,20 @@
import os
-import logging
import oras.client
from pathlib import Path
+from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from esd.logger.logger_builder import get_logger
from esd.utils.utils import clean_up
-class BuildCacheManager:
+class BuildCacheManager(BuildCacheManagerInterface):
"""
This class aims to manage the push/pull/delete of build cache files
"""
- def __init__(self, auth_backend='basic', log_path='./'):
+ def __init__(self, auth_backend='basic', insecure=False):
+ self.logger = get_logger(__name__, BuildCacheManager.__name__)
self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
- self.log_file = Path(log_path) / "log_oras.txt"
self.registry_project = os.environ.get("REGISTRY_PROJECT")
self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
@@ -24,36 +25,25 @@ class BuildCacheManager:
# This method utilizes the OCI Registry for container image and artifact management.
# Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
# Supported authentication types may include basic authentication (username/password), token-based authentication,
- self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend)
+ self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend, insecure=insecure)
self.client.login(username=self._registry_username, password=self._registry_password)
self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
- logging.basicConfig(
- level=logging.INFO,
- format='%(asctime)s - %(levelname)s - %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S',
- handlers=[
- logging.FileHandler(self.log_file),
- logging.StreamHandler()
- ]
- )
-
- def oci_registry_upload_build_cache(self, cache_dir: Path):
+ def upload(self, out_dir: Path):
"""
This method pushed all the files from the build cache folder into the OCI Registry
"""
- build_cache_path = self.home_path / "shared" / cache_dir
+ build_cache_path = self.home_path / out_dir
# build cache folder must exist before pushing all the artifacts
if not build_cache_path.exists():
- raise FileNotFoundError(
- f"BuildCacheManager::oci_registry_upload_build_cache::Path {build_cache_path} not found.")
+ self.logger.error(f"Path {build_cache_path} not found.")
for sub_path in build_cache_path.rglob("*"):
if sub_path.is_file():
rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
try:
- logging.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self.logger.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
self.client.push(
files=[str(sub_path)],
target=target,
@@ -61,31 +51,31 @@ class BuildCacheManager:
manifest_annotations={"path": rel_path},
disable_path_validation=True,
)
- logging.info(f"Successfully pushed {sub_path.name}")
+ self.logger.info(f"Successfully pushed {sub_path.name}")
except Exception as e:
- logging.error(
- f"BuildCacheManager::registry_upload_build_cache::An error occurred while pushing: {e}")
- # delete the build cache after being pushed to the OCI Registry
- clean_up([str(build_cache_path)], logging)
+ self.logger.error(
+ f"An error occurred while pushing: {e}")
+ # todo to be discussed hot to delete the build cache after being pushed to the OCI Registry
+ # clean_up([str(build_cache_path)], self.logger)
- def oci_registry_get_tags(self):
+ def list_tags(self):
"""
This method retrieves all tags from an OCI Registry
"""
try:
return self.client.get_tags(self.oci_registry_path)
except Exception as e:
- logging.error(f"BuildCacheManager::oci_registry_get_tags::Failed to list tags: {e}")
+ self.logger.error(f"Failed to list tags: {e}")
return None
- def oci_registry_download_build_cache(self, cache_dir: Path):
+ def download(self, in_dir: Path):
"""
This method pulls all the files from the OCI Registry into the build cache folder
"""
- build_cache_path = self.home_path / "shared" / cache_dir
+ build_cache_path = self.home_path / in_dir
# create the buildcache dir if it does not exist
os.makedirs(build_cache_path, exist_ok=True)
- tags = self.oci_registry_get_tags()
+ tags = self.list_tags()
if tags is not None:
for tag in tags:
ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
@@ -97,27 +87,27 @@ class BuildCacheManager:
try:
self.client.pull(
ref,
- # missing dirs to outdir are created automatically by OrasClient pull method
+ # missing dirs to output dir are created automatically by OrasClient pull method
outdir=str(build_cache_path / cache_path),
overwrite=True
)
- logging.info(f"Successfully pulled artifact {tag}.")
+ self.logger.info(f"Successfully pulled artifact {tag}.")
except Exception as e:
- logging.error(
- f"BuildCacheManager::registry_download_build_cache::Failed to pull artifact {tag} : {e}")
+ self.logger.error(
+ f"Failed to pull artifact {tag} : {e}")
- def oci_registry_delete_build_cache(self):
+ def delete(self):
"""
Deletes all artifacts from an OCI Registry based on their tags.
This method removes artifacts identified by their tags in the specified OCI Registry.
It requires appropriate permissions to delete artifacts from the registry.
If the registry or user does not have the necessary delete permissions, the operation might fail.
"""
- tags = self.oci_registry_get_tags()
+ tags = self.list_tags()
if tags is not None:
try:
self.client.delete_tags(self.oci_registry_path, tags)
- logging.info(f"Successfully deleted all artifacts form OCI registry.")
+ self.logger.info(f"Successfully deleted all artifacts form OCI registry.")
except RuntimeError as e:
- logging.error(
- f"BuildCacheManager::registry_delete_build_cache::Failed to delete artifacts: {e}")
+ self.logger.error(
+ f"Failed to delete artifacts: {e}")
diff --git a/esd/build_cache/BuildCacheManagerInterface.py b/esd/build_cache/BuildCacheManagerInterface.py
new file mode 100644
index 00000000..3016590b
--- /dev/null
+++ b/esd/build_cache/BuildCacheManagerInterface.py
@@ -0,0 +1,17 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class BuildCacheManagerInterface(ABC):
+
+ @abstractmethod
+ def upload(self, out_dir: Path):
+ pass
+
+ @abstractmethod
+ def download(self, in_dir: Path):
+ pass
+
+ @abstractmethod
+ def delete(self):
+ pass
diff --git a/esd/logger/__init__.py b/esd/logger/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/esd/logger/logger_builder.py b/esd/logger/logger_builder.py
new file mode 100644
index 00000000..c15a2f0e
--- /dev/null
+++ b/esd/logger/logger_builder.py
@@ -0,0 +1,51 @@
+import os
+import inspect
+import weakref
+import logging
+import logging.config
+
+
+class LoggerBuilder(object):
+ """
+ Class taking care of uniform Python logger initialization.
+ It uses the Python native logging package.
+ It's purpose is just to offer a common mechanism for initializing all modules in a package.
+ """
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(self, config_file_name='logging.conf'):
+ """
+ Prepare Python logger based on a configuration file.
+ :param: config_file_name - name of the logging configuration relative to the current package
+ """
+ current_folder = os.path.dirname(inspect.getfile(self.__class__))
+ config_file_path = os.path.join(current_folder, config_file_name)
+ logging.config.fileConfig(config_file_path, disable_existing_loggers=False)
+ self._loggers = weakref.WeakValueDictionary()
+
+ def build_logger(self, parent_module, parent_class):
+ """
+ Build a logger instance and return it
+ """
+ logger_key = f'{parent_module}.{parent_class}' if parent_class else parent_module
+ self._loggers[logger_key] = logger = logging.getLogger(logger_key)
+ return logger
+
+ def set_loggers_level(self, level):
+ for logger in self._loggers.values():
+ logger.setLevel(level)
+
+
+def get_logger(parent_module='', parent_class=None):
+ """
+ Function to retrieve a new Python logger instance for current module.
+
+ :param parent_module: module name for which to create logger.
+ :param parent_class: class name for which to create logger.
+ """
+ return LoggerBuilder().build_logger(parent_module, parent_class)
diff --git a/esd/logger/logging.conf b/esd/logger/logging.conf
new file mode 100644
index 00000000..d95ba87c
--- /dev/null
+++ b/esd/logger/logging.conf
@@ -0,0 +1,55 @@
+############################################
+## ESD - logging configuration. ##
+############################################
+[loggers]
+keys=root, esd, oras
+
+[handlers]
+keys=consoleHandler, fileHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=WARNING
+handlers=consoleHandler, fileHandler
+propagate=0
+
+############################################
+## esd specific logging ##
+############################################
+[logger_esd]
+level=DEBUG
+handlers=consoleHandler, fileHandler
+qualname=esd
+propagate=0
+
+[logger_oras]
+level=ERROR
+handlers=consoleHandler
+qualname=oras
+propagate=0
+
+############################################
+## Handlers ##
+############################################
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=handlers.TimedRotatingFileHandler
+level=INFO
+formatter=simpleFormatter
+args=('.esd.log', 'midnight', 1, 30, None, False, False)
+
+############################################
+## Formatters ##
+############################################
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(levelname)s - %(name)s::%(funcName)s - %(message)s
+datefmt = %d-%m-%Y %I:%M:%S
\ No newline at end of file
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
index c1aa4c0d..811d258e 100644
--- a/esd/utils/utils.py
+++ b/esd/utils/utils.py
@@ -2,7 +2,7 @@ import shutil
from pathlib import Path
-def clean_up(dirs: list[str], logging):
+def clean_up(dirs: list[str], logging, ignore_errors=True):
"""
All the folders from the list dirs are removed with all the content in them
"""
@@ -10,6 +10,11 @@ def clean_up(dirs: list[str], logging):
cleanup_dir = Path(cleanup_dir).resolve()
if cleanup_dir.exists():
logging.info(f"Removing {cleanup_dir}")
- shutil.rmtree(Path(cleanup_dir))
+ try:
+ shutil.rmtree(Path(cleanup_dir))
+ except OSError as e:
+ logging.error(f"Failed to remove {cleanup_dir}: {e}")
+ if not ignore_errors:
+ raise e
else:
logging.info(f"{cleanup_dir} does not exist")
diff --git a/pyproject.toml b/pyproject.toml
index cd4afe54..0f6d0cde 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,4 +18,7 @@ dependencies = [
"ruamel.yaml",
"pytest",
"pytest-mock",
-]
\ No newline at end of file
+]
+
+[tool.setuptools.data-files]
+"esd-tools" = ["esd/logger/logging.conf"]
\ No newline at end of file
--
GitLab
From 0c03c2d486d6bb6b23f513b126bfb8b279b29452 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 18 Feb 2025 13:40:38 +0200
Subject: [PATCH 4/4] esd: package renaming to dedal
---
.gitlab-ci.yml | 4 ++--
{esd => dedal}/__init__.py | 0
{esd => dedal}/build_cache/BuildCacheManager.py | 6 +++---
.../build_cache/BuildCacheManagerInterface.py | 0
{esd => dedal}/build_cache/__init__.py | 0
{esd => dedal}/cli/__init__.py | 0
{esd => dedal}/cli/fetch_cached_buildresults.py | 0
{esd => dedal}/fetch_cached_sources.py | 0
{esd => dedal}/logger/__init__.py | 0
{esd => dedal}/logger/logger_builder.py | 0
{esd => dedal}/logger/logging.conf | 12 ++++++------
{esd => dedal}/specfile_dag_hash.py | 0
{esd => dedal}/specfile_storage_path_build.py | 0
{esd => dedal}/specfile_storage_path_source.py | 0
{esd => dedal}/tests/__init__.py | 0
{esd => dedal}/tests/utils_test.py | 2 +-
{esd => dedal}/update_cached_buildresults.py | 0
{esd => dedal}/update_cached_sources.py | 0
{esd => dedal}/utils/__init__.py | 0
{esd => dedal}/utils/utils.py | 0
pyproject.toml | 4 ++--
21 files changed, 14 insertions(+), 14 deletions(-)
rename {esd => dedal}/__init__.py (100%)
rename {esd => dedal}/build_cache/BuildCacheManager.py (96%)
rename {esd => dedal}/build_cache/BuildCacheManagerInterface.py (100%)
rename {esd => dedal}/build_cache/__init__.py (100%)
rename {esd => dedal}/cli/__init__.py (100%)
rename {esd => dedal}/cli/fetch_cached_buildresults.py (100%)
rename {esd => dedal}/fetch_cached_sources.py (100%)
rename {esd => dedal}/logger/__init__.py (100%)
rename {esd => dedal}/logger/logger_builder.py (100%)
rename {esd => dedal}/logger/logging.conf (85%)
rename {esd => dedal}/specfile_dag_hash.py (100%)
rename {esd => dedal}/specfile_storage_path_build.py (100%)
rename {esd => dedal}/specfile_storage_path_source.py (100%)
rename {esd => dedal}/tests/__init__.py (100%)
rename {esd => dedal}/tests/utils_test.py (97%)
rename {esd => dedal}/update_cached_buildresults.py (100%)
rename {esd => dedal}/update_cached_sources.py (100%)
rename {esd => dedal}/utils/__init__.py (100%)
rename {esd => dedal}/utils/utils.py (100%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9b2e92b5..4512455a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,7 +3,7 @@ stages:
- test
variables:
- BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/esd/tmp:latest
+ BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/dedal/tmp:latest
build-wheel:
stage: build
@@ -28,7 +28,7 @@ testing:
image: python:latest
script:
- pip install -e .
- - pytest ./esd/tests/ --junitxml=test-results.xml
+ - pytest ./dedal/tests/ --junitxml=test-results.xml
artifacts:
when: always
reports:
diff --git a/esd/__init__.py b/dedal/__init__.py
similarity index 100%
rename from esd/__init__.py
rename to dedal/__init__.py
diff --git a/esd/build_cache/BuildCacheManager.py b/dedal/build_cache/BuildCacheManager.py
similarity index 96%
rename from esd/build_cache/BuildCacheManager.py
rename to dedal/build_cache/BuildCacheManager.py
index 44e13b73..2da39e25 100644
--- a/esd/build_cache/BuildCacheManager.py
+++ b/dedal/build_cache/BuildCacheManager.py
@@ -2,9 +2,9 @@ import os
import oras.client
from pathlib import Path
-from esd.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
-from esd.logger.logger_builder import get_logger
-from esd.utils.utils import clean_up
+from dedal.build_cache.BuildCacheManagerInterface import BuildCacheManagerInterface
+from dedal.logger.logger_builder import get_logger
+from dedal.utils.utils import clean_up
class BuildCacheManager(BuildCacheManagerInterface):
diff --git a/esd/build_cache/BuildCacheManagerInterface.py b/dedal/build_cache/BuildCacheManagerInterface.py
similarity index 100%
rename from esd/build_cache/BuildCacheManagerInterface.py
rename to dedal/build_cache/BuildCacheManagerInterface.py
diff --git a/esd/build_cache/__init__.py b/dedal/build_cache/__init__.py
similarity index 100%
rename from esd/build_cache/__init__.py
rename to dedal/build_cache/__init__.py
diff --git a/esd/cli/__init__.py b/dedal/cli/__init__.py
similarity index 100%
rename from esd/cli/__init__.py
rename to dedal/cli/__init__.py
diff --git a/esd/cli/fetch_cached_buildresults.py b/dedal/cli/fetch_cached_buildresults.py
similarity index 100%
rename from esd/cli/fetch_cached_buildresults.py
rename to dedal/cli/fetch_cached_buildresults.py
diff --git a/esd/fetch_cached_sources.py b/dedal/fetch_cached_sources.py
similarity index 100%
rename from esd/fetch_cached_sources.py
rename to dedal/fetch_cached_sources.py
diff --git a/esd/logger/__init__.py b/dedal/logger/__init__.py
similarity index 100%
rename from esd/logger/__init__.py
rename to dedal/logger/__init__.py
diff --git a/esd/logger/logger_builder.py b/dedal/logger/logger_builder.py
similarity index 100%
rename from esd/logger/logger_builder.py
rename to dedal/logger/logger_builder.py
diff --git a/esd/logger/logging.conf b/dedal/logger/logging.conf
similarity index 85%
rename from esd/logger/logging.conf
rename to dedal/logger/logging.conf
index d95ba87c..fd24c026 100644
--- a/esd/logger/logging.conf
+++ b/dedal/logger/logging.conf
@@ -1,8 +1,8 @@
############################################
-## ESD - logging configuration. ##
+## Dedal - logging configuration. ##
############################################
[loggers]
-keys=root, esd, oras
+keys=root, dedal, oras
[handlers]
keys=consoleHandler, fileHandler
@@ -16,12 +16,12 @@ handlers=consoleHandler, fileHandler
propagate=0
############################################
-## esd specific logging ##
+## dedal specific logging ##
############################################
-[logger_esd]
+[logger_dedal]
level=DEBUG
handlers=consoleHandler, fileHandler
-qualname=esd
+qualname=dedal
propagate=0
[logger_oras]
@@ -44,7 +44,7 @@ args=(sys.stdout,)
class=handlers.TimedRotatingFileHandler
level=INFO
formatter=simpleFormatter
-args=('.esd.log', 'midnight', 1, 30, None, False, False)
+args=('.dedal.log', 'midnight', 1, 30, None, False, False)
############################################
## Formatters ##
diff --git a/esd/specfile_dag_hash.py b/dedal/specfile_dag_hash.py
similarity index 100%
rename from esd/specfile_dag_hash.py
rename to dedal/specfile_dag_hash.py
diff --git a/esd/specfile_storage_path_build.py b/dedal/specfile_storage_path_build.py
similarity index 100%
rename from esd/specfile_storage_path_build.py
rename to dedal/specfile_storage_path_build.py
diff --git a/esd/specfile_storage_path_source.py b/dedal/specfile_storage_path_source.py
similarity index 100%
rename from esd/specfile_storage_path_source.py
rename to dedal/specfile_storage_path_source.py
diff --git a/esd/tests/__init__.py b/dedal/tests/__init__.py
similarity index 100%
rename from esd/tests/__init__.py
rename to dedal/tests/__init__.py
diff --git a/esd/tests/utils_test.py b/dedal/tests/utils_test.py
similarity index 97%
rename from esd/tests/utils_test.py
rename to dedal/tests/utils_test.py
index 8bec6c58..14795726 100644
--- a/esd/tests/utils_test.py
+++ b/dedal/tests/utils_test.py
@@ -1,7 +1,7 @@
import pytest
from pathlib import Path
-from esd.utils.utils import clean_up
+from dedal.utils.utils import clean_up
@pytest.fixture
diff --git a/esd/update_cached_buildresults.py b/dedal/update_cached_buildresults.py
similarity index 100%
rename from esd/update_cached_buildresults.py
rename to dedal/update_cached_buildresults.py
diff --git a/esd/update_cached_sources.py b/dedal/update_cached_sources.py
similarity index 100%
rename from esd/update_cached_sources.py
rename to dedal/update_cached_sources.py
diff --git a/esd/utils/__init__.py b/dedal/utils/__init__.py
similarity index 100%
rename from esd/utils/__init__.py
rename to dedal/utils/__init__.py
diff --git a/esd/utils/utils.py b/dedal/utils/utils.py
similarity index 100%
rename from esd/utils/utils.py
rename to dedal/utils/utils.py
diff --git a/pyproject.toml b/pyproject.toml
index 0f6d0cde..757f370c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools", "setuptools-scm"]
build-backend = "setuptools.build_meta"
[project]
-name = "esd"
+name = "dedal"
authors = [
{name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
{name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"},
@@ -21,4 +21,4 @@ dependencies = [
]
[tool.setuptools.data-files]
-"esd-tools" = ["esd/logger/logging.conf"]
\ No newline at end of file
+"dedal" = ["dedal/logger/logging.conf"]
\ No newline at end of file
--
GitLab