diff --git a/esd/__init__.py b/esd/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/esd/build_cache/__init__.py b/esd/build_cache/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/esd/build_cache/manage_build_cache.py b/esd/build_cache/manage_build_cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..9697e88d0f8ef0eab7b9f75f23b7b7873fc491cd
--- /dev/null
+++ b/esd/build_cache/manage_build_cache.py
@@ -0,0 +1,123 @@
+import os
+import logging
+import oras.client
+from pathlib import Path
+
+from esd.utils.utils import clean_up
+
+
+class BuildCacheManager:
+ """
+ This class aims to manage the push/pull/delete of build cache files
+ """
+
+ def __init__(self, auth_backend='basic', log_path='./'):
+ self.home_path = Path(os.environ.get("HOME_PATH", os.getcwd()))
+ self.log_file = Path(log_path) / "log_oras.txt"
+ self.registry_project = os.environ.get("REGISTRY_PROJECT")
+
+ self._registry_username = str(os.environ.get("REGISTRY_USERNAME"))
+ self._registry_password = str(os.environ.get("REGISTRY_PASSWORD"))
+
+ self.registry_host = str(os.environ.get("REGISTRY_HOST"))
+ # Initialize an OrasClient instance.
+ # This method utilizes the OCI Registry for container image and artifact management.
+ # Refer to the official OCI Registry documentation for detailed information on the available authentication methods.
+ # Supported authentication types may include basic authentication (username/password), token-based authentication,
+ self.client = oras.client.OrasClient(hostname=self.registry_host, auth_backend=auth_backend)
+ self.client.login(username=self._registry_username, password=self._registry_password)
+ self.oci_registry_path = f'{self.registry_host}/{self.registry_project}/cache'
+
+ logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(levelname)s - %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ handlers=[
+ logging.FileHandler(self.log_file),
+ logging.StreamHandler()
+ ]
+ )
+
+ def oci_registry_upload_build_cache(self, cache_dir: Path):
+ """
+ This method pushed all the files from the build cache folder into the OCI Registry
+ """
+ build_cache_path = self.home_path / "shared" / cache_dir
+ # build cache folder must exist before pushing all the artifacts
+ if not build_cache_path.exists():
+ raise FileNotFoundError(
+ f"BuildCacheManager::oci_registry_upload_build_cache::Path {build_cache_path} not found.")
+
+ for sub_path in build_cache_path.rglob("*"):
+ if sub_path.is_file():
+ rel_path = str(sub_path.relative_to(build_cache_path)).replace(str(sub_path.name), "")
+ target = f"{self.registry_host}/{self.registry_project}/cache:{str(sub_path.name)}"
+ try:
+ logging.info(f"Pushing folder '{sub_path}' to ORAS target '{target}' ...")
+ self.client.push(
+ files=[str(sub_path)],
+ target=target,
+ # save in manifest the relative path for reconstruction
+ manifest_annotations={"path": rel_path},
+ disable_path_validation=True,
+ )
+ logging.info(f"Successfully pushed {sub_path.name}")
+ except Exception as e:
+ logging.error(
+ f"BuildCacheManager::registry_upload_build_cache::An error occurred while pushing: {e}")
+ # delete the build cache after being pushed to the OCI Registry
+ clean_up([str(build_cache_path)], logging)
+
+ def oci_registry_get_tags(self):
+ """
+ This method retrieves all tags from an OCI Registry
+ """
+ try:
+ return self.client.get_tags(self.oci_registry_path)
+ except Exception as e:
+ logging.error(f"BuildCacheManager::oci_registry_get_tags::Failed to list tags: {e}")
+ return None
+
+ def oci_registry_download_build_cache(self, cache_dir: Path):
+ """
+ This method pulls all the files from the OCI Registry into the build cache folder
+ """
+ build_cache_path = self.home_path / "shared" / cache_dir
+ # create the buildcache dir if it does not exist
+ os.makedirs(build_cache_path, exist_ok=True)
+ tags = self.oci_registry_get_tags()
+ if tags is not None:
+ for tag in tags:
+ ref = f"{self.registry_host}/{self.registry_project}/cache:{tag}"
+ # reconstruct the relative path of each artifact by getting it from the manifest
+ cache_path = \
+ self.client.get_manifest(f'{self.registry_host}/{self.registry_project}/cache:{tag}')[
+ 'annotations'][
+ 'path']
+ try:
+ self.client.pull(
+ ref,
+ # missing dirs to outdir are created automatically by OrasClient pull method
+ outdir=str(build_cache_path / cache_path),
+ overwrite=True
+ )
+ logging.info(f"Successfully pulled artifact {tag}.")
+ except Exception as e:
+ logging.error(
+ f"BuildCacheManager::registry_download_build_cache::Failed to pull artifact {tag} : {e}")
+
+ def oci_registry_delete_build_cache(self):
+ """
+ Deletes all artifacts from an OCI Registry based on their tags.
+ This method removes artifacts identified by their tags in the specified OCI Registry.
+ It requires appropriate permissions to delete artifacts from the registry.
+ If the registry or user does not have the necessary delete permissions, the operation might fail.
+ """
+ tags = self.oci_registry_get_tags()
+ if tags is not None:
+ try:
+ self.client.delete_tags(self.oci_registry_path, tags)
+ logging.info(f"Successfully deleted all artifacts form OCI registry.")
+ except RuntimeError as e:
+ logging.error(
+ f"BuildCacheManager::registry_delete_build_cache::Failed to delete artifacts: {e}")
diff --git a/fetch_cached_buildresults.py b/esd/fetch_cached_buildresults.py
similarity index 100%
rename from fetch_cached_buildresults.py
rename to esd/fetch_cached_buildresults.py
diff --git a/fetch_cached_sources.py b/esd/fetch_cached_sources.py
similarity index 100%
rename from fetch_cached_sources.py
rename to esd/fetch_cached_sources.py
diff --git a/specfile_dag_hash.py b/esd/specfile_dag_hash.py
similarity index 96%
rename from specfile_dag_hash.py
rename to esd/specfile_dag_hash.py
index 6e001b84accd5b4cd7c11a79842b7d88df306ec8..e44e1c6296c2499580f286d1b2ec7e753243f53a 100644
--- a/specfile_dag_hash.py
+++ b/esd/specfile_dag_hash.py
@@ -3,7 +3,7 @@ from collections.abc import Iterable
import pathlib
import ruamel.yaml as yaml
import spack
-import spack.binary_distribution as bindist
+
parser = argparse.ArgumentParser(
prog='specfile_dag_hash.py',
diff --git a/specfile_storage_path_build.py b/esd/specfile_storage_path_build.py
similarity index 100%
rename from specfile_storage_path_build.py
rename to esd/specfile_storage_path_build.py
diff --git a/specfile_storage_path_source.py b/esd/specfile_storage_path_source.py
similarity index 100%
rename from specfile_storage_path_source.py
rename to esd/specfile_storage_path_source.py
diff --git a/update_cached_buildresults.py b/esd/update_cached_buildresults.py
similarity index 99%
rename from update_cached_buildresults.py
rename to esd/update_cached_buildresults.py
index caacf86e46efec1ddb0c8b8c044f4ff8506eace1..58af242b3bcfb03f153a0f4fa1fa22ff2c66d371 100644
--- a/update_cached_buildresults.py
+++ b/esd/update_cached_buildresults.py
@@ -1,5 +1,4 @@
import argparse
-import glob
import os
import pathlib
import subprocess
diff --git a/update_cached_sources.py b/esd/update_cached_sources.py
similarity index 100%
rename from update_cached_sources.py
rename to esd/update_cached_sources.py
diff --git a/esd/utils/__init__.py b/esd/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/esd/utils/utils.py b/esd/utils/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1aa4c0da97a36ed386ebd28a8ad211bef91b3df
--- /dev/null
+++ b/esd/utils/utils.py
@@ -0,0 +1,15 @@
+import shutil
+from pathlib import Path
+
+
+def clean_up(dirs: list[str], logging):
+ """
+ All the folders from the list dirs are removed with all the content in them
+ """
+ for cleanup_dir in dirs:
+ cleanup_dir = Path(cleanup_dir).resolve()
+ if cleanup_dir.exists():
+ logging.info(f"Removing {cleanup_dir}")
+ shutil.rmtree(Path(cleanup_dir))
+ else:
+ logging.info(f"{cleanup_dir} does not exist")
diff --git a/wscript b/esd/wscript
old mode 100755
new mode 100644
similarity index 100%
rename from wscript
rename to esd/wscript
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000000000000000000000000000000000..dd9ac1021b7091a7d1d4bb8066b7c5d8300cb76e
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "esd"
+authors = [
+ {name = "Eric Müller", email = "mueller@kip.uni-heidelberg.de"},
+ {name = "Adrian Ciu", email = "adrian.ciu@codemart.ro"},
+]
+description = "This package provides all the necessary tools to create an Ebrains Software Distribution environment"
+version = "0.1.0"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+ "oras",
+ "spack",
+ "ruamel.yaml"
+]
\ No newline at end of file