From 7a9e9e4075df8149fcde545656356e6867d393e5 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 14 Mar 2025 17:26:04 +0200
Subject: [PATCH 01/13] VT-94: migrate esd bash scripts to Dedal library
---
.gitlab-ci.yml | 6 +-
__init__.py | 0
install_spack_env.py | 137 +++++++++++++++++++++++++++++++++++++++++++
utils/__init__.py | 0
utils/utils.py | 13 ++++
utils/ymerge.py | 15 +++++
6 files changed, 168 insertions(+), 3 deletions(-)
create mode 100644 __init__.py
create mode 100644 install_spack_env.py
create mode 100644 utils/__init__.py
create mode 100644 utils/utils.py
create mode 100644 utils/ymerge.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 91b188c3..aefca059 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -231,7 +231,7 @@ build-spack-env-on-runner:
- >
echo " view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml
# run installation script
- - bash install_spack_env.sh $SPACK_JOBS $CI_PROJECT_DIR $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
+ - python3 install_spack_env.py $SPACK_JOBS $CI_PROJECT_DIR $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
after_script:
- mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
# for succesfully installed packages: keep the spack logs for any package modified during this CI job
@@ -272,7 +272,7 @@ sync-esd-image:
# run installation script inside future container environment
# => DAG concretization, subsequent cache access + fetching and actual build should be separate steps
- mkdir --mode=0777 -p ${SANDBOX_ROOT}/${INSTALLATION_ROOT}
- - apptainer exec --writable --bind ${CI_PROJECT_DIR}:${INSTALLATION_ROOT} --cwd ${INSTALLATION_ROOT} ${SANDBOX_ROOT} bash ./install_spack_env.sh $SPACK_JOBS $INSTALLATION_ROOT ${INSTALLATION_ROOT} $CI_SPACK_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
+ - apptainer exec --writable --bind ${CI_PROJECT_DIR}:${INSTALLATION_ROOT} --cwd ${INSTALLATION_ROOT} ${SANDBOX_ROOT} python3 install_spack_env.py $SPACK_JOBS $INSTALLATION_ROOT ${INSTALLATION_ROOT} $CI_SPACK_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
- echo "export SYSTEMNAME=${SYSTEMNAME}" >> ${SANDBOX_ROOT}/.singularity.d/env/90-environment.sh
- echo ". ${INSTALLATION_ROOT}/vendor/spack/var/spack/environments/${CI_SPACK_ENV}/load_env.sh" >> ${SANDBOX_ROOT}/.singularity.d/env/90-environment.sh
# preparing to assemble the image: move in the CI project contents...
@@ -326,7 +326,7 @@ sync-gitlab-spack-instance:
- git reset --hard $CI_COMMIT_SHA
- git submodule update --force
# run installation script
- - bash install_spack_env.sh $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_REPO_PATH $SPACK_NFS_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
+ - python3 install_spack_env.py $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_REPO_PATH $SPACK_NFS_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
# create kernel spec, so that the environment can be used in gitlab CI jobs
- RELEASE_NAME=$(case $CI_COMMIT_BRANCH in experimental_rel) echo ebrains-experimental;; ebrains*) echo ${CI_COMMIT_BRANCH:0:10}.${CI_COMMIT_BRANCH:11};; *) echo $CI_COMMIT_BRANCH;; esac);
- bash create_JupyterLab_kernel.sh $SPACK_PATH_GITLAB $SPACK_NFS_ENV $RELEASE_NAME /mnt/ebrains_env
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/install_spack_env.py b/install_spack_env.py
new file mode 100644
index 00000000..31a6ab16
--- /dev/null
+++ b/install_spack_env.py
@@ -0,0 +1,137 @@
+import os
+import argparse
+import shutil
+import sys
+from pathlib import Path
+from dedal.configuration.SpackConfig import SpackConfig
+from dedal.enum.SpackConfigCommand import SpackConfigCommand
+from dedal.enum.SpackViewEnum import SpackViewEnum
+from dedal.model.SpackDescriptor import SpackDescriptor
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.utils.utils import set_bashrc_variable
+from utils.utils import find_first_upstream_prefix
+from utils.ymerge import merge_spack_envs
+
+parser = argparse.ArgumentParser(
+ prog='install_spack_env.py',
+ description='ESD install spack env.',
+ epilog='...')
+
+parser.add_argument(
+ "spack_jobs",
+ type=int,
+ help="number of jobs"
+)
+
+parser.add_argument(
+ "installation_root",
+ type=str,
+ help="where to set up the installation"
+)
+
+parser.add_argument(
+ "ebrains_repo",
+ type=str,
+ help="location of ebrains-spack-builds repository"
+)
+
+parser.add_argument(
+ "ebrains_spack_env",
+ type=str,
+ help="name of EBRAINS Spack environment to be created/updated"
+)
+
+parser.add_argument(
+ "upstream_instance",
+ type=str,
+ help="path to Spack instance to use as upstream (optional)"
+)
+
+parser.add_argument(
+ "update_spack_oci_caches",
+ type=bool,
+ help="true enables updating the OCI cache for spack sources and build results"
+)
+
+parser.add_argument(
+ "oci_cache_prefix",
+ type=str,
+)
+
+args = parser.parse_args()
+
+spack_jobs = args.spack_jobs
+installation_root = Path(args.installation_root).resolve()
+ebrains_repo = args.ebrains_repo
+ebrains_spack_env = args.ebrains_spack_env
+upstream_instance = args.upstream_instance
+update_spack_oci_caches = args.update_spack_oci_caches
+oci_cache_prefix = args.oci_cache_prefix
+
+set_bashrc_variable('OCI_CACHE_PREFIX', oci_cache_prefix)
+
+ci_spack_root = installation_root / 'spack'
+
+set_bashrc_variable('CACHE_SPECFILE', f'env_specfile.yaml')
+set_bashrc_variable('DEDAL_HOME', f'{ebrains_repo}/vendor/yashchiki')
+set_bashrc_variable('SPACK_CACHE_SOURCE', f'{ci_spack_root}/var/spack/cache')
+set_bashrc_variable('SPACK_CACHE_BUILD', f'{ci_spack_root}/var/spack/cache')
+
+ci_spack_root = installation_root / 'spack'
+spack_root_existed = 1
+if ci_spack_root and ebrains_repo:
+ if not os.path.isdir(ci_spack_root):
+ spack_source = os.path.join(ebrains_repo, 'vendor', 'spack')
+ try:
+ os.symlink(spack_source, ci_spack_root)
+ spack_root_existed = 0
+ except FileExistsError:
+ pass
+
+data_dir = installation_root / 'cashing'
+env_repo = SpackDescriptor('ebrains-spack-builds', data_dir, ebrains_repo)
+spack_config = SpackConfig(env=env_repo,
+ repos=[env_repo],
+ install_dir=installation_root,
+ upstream_instance=upstream_instance,
+ system_name='VBT',
+ concretization_dir=data_dir / 'concretize_cache',
+ buildcache_dir=data_dir / 'binary_cache',
+ use_spack_global=False,
+ cache_version_build='spack_cache',
+ view=SpackViewEnum.WITHOUT_VIEW)
+
+spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=False)
+spack_operation.install_spack()
+if upstream_instance:
+ upstream_prefix = find_first_upstream_prefix(upstream_instance)
+ spack_operation.config(SpackConfigCommand.ADD, f'upstreams:upstream-spack-instance:install_tree:{upstream_prefix}')
+if spack_root_existed == 0:
+ spack_operation.config(SpackConfigCommand.ADD, 'config:install_tree:padded_length:128')
+
+spack_operation.spec_pacakge('aida')
+spack_operation.reindex()
+spack_operation.add_mirror(mirror_name='local_cache', mirror_path=Path(os.getenv('SPACK_CACHE_BUILD')).resolve())
+spack_operation.setup_spack_env()
+
+site_config_dest = ci_spack_root / 'var/spack/environments' / ebrains_spack_env
+if site_config_dest.exists():
+ site_config_dest.rmdir()
+ site_config_dest.mkdir(parents=True, exist_ok=True)
+ site_config_esd = Path(ebrains_spack_env) / 'site-config'
+ shutil.copy(site_config_esd, site_config_dest)
+ merged_envs = merge_spack_envs(Path(ebrains_spack_env) / 'spack.yaml', site_config_esd / os.getenv('SYSTEMNAME') / 'spack.yaml')
+ tmp_spack_yaml = Path("/tmp/spack.yaml").resolve()
+ try:
+ with open(tmp_spack_yaml, "w") as f:
+ f.write(merged_envs)
+ print(f"Written merged spack.yaml to {tmp_spack_yaml}")
+ except Exception as e:
+ sys.stderr.write(f"Error writing {tmp_spack_yaml}: {e}\n")
+ sys.exit(1)
+ shutil.copy(tmp_spack_yaml, site_config_dest)
+
+spack_operation.concretize_spack_env()
+
+spack_operation.install_packages(jobs=int(os.getenv('SPACK_JOBS')), signed=False, test='root')
+spack_operation.reindex()
\ No newline at end of file
diff --git a/utils/__init__.py b/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/utils/utils.py b/utils/utils.py
new file mode 100644
index 00000000..2edf146c
--- /dev/null
+++ b/utils/utils.py
@@ -0,0 +1,13 @@
+import os
+
+
+def find_first_upstream_prefix(upstream_instance):
+ """
+ Search for directories named '.spack-db' within the spack opt directory
+ under upstream_instance, and return a list of their parent directories.
+ """
+ base_path = os.path.join(upstream_instance, "spack", "opt", "spack")
+ for upstream_prefix, dirs, _ in os.walk(base_path):
+ if ".spack-db" in dirs:
+ return upstream_prefix
+ return None
diff --git a/utils/ymerge.py b/utils/ymerge.py
new file mode 100644
index 00000000..79567d63
--- /dev/null
+++ b/utils/ymerge.py
@@ -0,0 +1,15 @@
+# spack-python script that merges two environment configuration files (spack.yaml) into one
+# Usage: spack-python /path/to/first/spack.yaml /path/to/second/spack.yaml
+# (note: if the second file does not exist, the output is the first file
+
+import os
+from spack.config import merge_yaml, read_config_file, syaml
+
+
+def merge_spack_envs(top_yaml_path, site_yaml_path):
+ if not os.path.exists(site_yaml_path):
+ merged = syaml.dump(read_config_file(top_yaml_path))
+ else:
+ merged = syaml.dump(merge_yaml(read_config_file(top_yaml_path), read_config_file(site_yaml_path)))
+ return merged
+
--
GitLab
From 817188aab39fb800c6fc16458556d9713892d3f0 Mon Sep 17 00:00:00 2001
From: Eleni Mathioulaki <emathioulaki@athenarc.gr>
Date: Tue, 15 Apr 2025 11:03:14 +0300
Subject: [PATCH 02/13] feat: update dedal to 0.1.0
---
vendor/yashchiki | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/vendor/yashchiki b/vendor/yashchiki
index 5690db7d..11785be8 160000
--- a/vendor/yashchiki
+++ b/vendor/yashchiki
@@ -1 +1 @@
-Subproject commit 5690db7dbccd78f3ceef2123a605e662bb8b2c0f
+Subproject commit 11785be8a6e914a8404757670f58b4e78fca2bf9
--
GitLab
From 2041e603e5ede0134decafd64e7a0514d7aa4544 Mon Sep 17 00:00:00 2001
From: Eleni Mathioulaki <emathioulaki@athenarc.gr>
Date: Tue, 15 Apr 2025 11:05:01 +0300
Subject: [PATCH 03/13] fix(CI): install dedal python library
---
.gitlab-ci.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aefca059..6f1ae550 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -227,6 +227,8 @@ build-spack-env-on-runner:
OCI_CACHE_PREFIX: ""
UPDATE_SPACK_OCI_CACHES: false
script:
+ # install dedal python library
+ - pip3 install --break-system-packages vendor/yashchiki
# deactivate environment views (we don't need them for the test build-job)
- >
echo " view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml
--
GitLab
From 1ff8f0898b515db091bcfe450b2d931080934211 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Mon, 28 Apr 2025 15:23:47 +0300
Subject: [PATCH 04/13] VT-94: fixing incompatibilities
---
.gitmodules | 2 +-
install_spack_env.py | 40 +++++++++++++++++++++++++++-------------
utils/ymerge.py | 13 ++++++-------
vendor/yashchiki | 2 +-
4 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/.gitmodules b/.gitmodules
index 30354655..0b9fab48 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -5,4 +5,4 @@
[submodule "vendor/yashchiki"]
path = vendor/yashchiki
url = https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/dedal
- shallow = true
+ shallow = true
\ No newline at end of file
diff --git a/install_spack_env.py b/install_spack_env.py
index 31a6ab16..55628af4 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -8,9 +8,8 @@ from dedal.enum.SpackConfigCommand import SpackConfigCommand
from dedal.enum.SpackViewEnum import SpackViewEnum
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
-from dedal.utils.utils import set_bashrc_variable
+from dedal.utils.utils import set_bashrc_variable, run_command
from utils.utils import find_first_upstream_prefix
-from utils.ymerge import merge_spack_envs
parser = argparse.ArgumentParser(
prog='install_spack_env.py',
@@ -56,6 +55,8 @@ parser.add_argument(
parser.add_argument(
"oci_cache_prefix",
type=str,
+ nargs="?", # optional
+ default=None,
)
args = parser.parse_args()
@@ -68,6 +69,8 @@ upstream_instance = args.upstream_instance
update_spack_oci_caches = args.update_spack_oci_caches
oci_cache_prefix = args.oci_cache_prefix
+system_name = os.getenv('SYSTEMNAME')
+
set_bashrc_variable('OCI_CACHE_PREFIX', oci_cache_prefix)
ci_spack_root = installation_root / 'spack'
@@ -77,32 +80,32 @@ set_bashrc_variable('DEDAL_HOME', f'{ebrains_repo}/vendor/yashchiki')
set_bashrc_variable('SPACK_CACHE_SOURCE', f'{ci_spack_root}/var/spack/cache')
set_bashrc_variable('SPACK_CACHE_BUILD', f'{ci_spack_root}/var/spack/cache')
-ci_spack_root = installation_root / 'spack'
-spack_root_existed = 1
+spack_root_existed = True
if ci_spack_root and ebrains_repo:
if not os.path.isdir(ci_spack_root):
spack_source = os.path.join(ebrains_repo, 'vendor', 'spack')
try:
os.symlink(spack_source, ci_spack_root)
- spack_root_existed = 0
+ spack_root_existed = False
except FileExistsError:
pass
-data_dir = installation_root / 'cashing'
-env_repo = SpackDescriptor('ebrains-spack-builds', data_dir, ebrains_repo)
+data_dir = installation_root / 'caching'
+print(installation_root.parent)
+env_repo = SpackDescriptor('ebrains-spack-builds', path=installation_root.parent)
spack_config = SpackConfig(env=env_repo,
repos=[env_repo],
- install_dir=installation_root,
+ install_dir=installation_root.parent,
upstream_instance=upstream_instance,
system_name='VBT',
concretization_dir=data_dir / 'concretize_cache',
buildcache_dir=data_dir / 'binary_cache',
use_spack_global=False,
cache_version_build='spack_cache',
- view=SpackViewEnum.WITHOUT_VIEW)
+ view=SpackViewEnum.WITHOUT_VIEW,
+ spack_dir=Path(ci_spack_root).resolve())
spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=False)
-spack_operation.install_spack()
if upstream_instance:
upstream_prefix = find_first_upstream_prefix(upstream_instance)
spack_operation.config(SpackConfigCommand.ADD, f'upstreams:upstream-spack-instance:install_tree:{upstream_prefix}')
@@ -120,11 +123,22 @@ if site_config_dest.exists():
site_config_dest.mkdir(parents=True, exist_ok=True)
site_config_esd = Path(ebrains_spack_env) / 'site-config'
shutil.copy(site_config_esd, site_config_dest)
- merged_envs = merge_spack_envs(Path(ebrains_spack_env) / 'spack.yaml', site_config_esd / os.getenv('SYSTEMNAME') / 'spack.yaml')
+ # spack-python /path/to/first/spack.yaml /path/to/second/spack.yaml
+ # merged_envs = merge_spack_envs(Path(ebrains_spack_env) / 'spack.yaml', site_config_esd / os.getenv('SYSTEMNAME') / 'spack.yaml')
+ merged_envs = result = run_command(
+ "spack-python",
+ f'{ebrains_repo}/site-config/ymerge.py {ebrains_repo}/spack.yaml',
+ f'{ebrains_repo}/site-config/{system_name}/spack.yaml',
+ info_msg='Merging top-level and site-specific spack.yaml files.',
+ exception_msg='Failed to merge top-level and site-specific spack.yaml files.',
+ capture_output=True,
+ text=True,
+ check=True
+ )
tmp_spack_yaml = Path("/tmp/spack.yaml").resolve()
try:
with open(tmp_spack_yaml, "w") as f:
- f.write(merged_envs)
+ f.write(merged_envs.stdout.strip())
print(f"Written merged spack.yaml to {tmp_spack_yaml}")
except Exception as e:
sys.stderr.write(f"Error writing {tmp_spack_yaml}: {e}\n")
@@ -134,4 +148,4 @@ if site_config_dest.exists():
spack_operation.concretize_spack_env()
spack_operation.install_packages(jobs=int(os.getenv('SPACK_JOBS')), signed=False, test='root')
-spack_operation.reindex()
\ No newline at end of file
+spack_operation.reindex()
diff --git a/utils/ymerge.py b/utils/ymerge.py
index 79567d63..accb8779 100644
--- a/utils/ymerge.py
+++ b/utils/ymerge.py
@@ -2,14 +2,13 @@
# Usage: spack-python /path/to/first/spack.yaml /path/to/second/spack.yaml
# (note: if the second file does not exist, the output is the first file
-import os
+import sys, os
from spack.config import merge_yaml, read_config_file, syaml
+if not os.path.exists(sys.argv[2]):
+ merged = syaml.dump(read_config_file(sys.argv[1]))
+else:
+ merged = syaml.dump(merge_yaml(read_config_file(sys.argv[1]), read_config_file(sys.argv[2])))
-def merge_spack_envs(top_yaml_path, site_yaml_path):
- if not os.path.exists(site_yaml_path):
- merged = syaml.dump(read_config_file(top_yaml_path))
- else:
- merged = syaml.dump(merge_yaml(read_config_file(top_yaml_path), read_config_file(site_yaml_path)))
- return merged
+print(merged)
diff --git a/vendor/yashchiki b/vendor/yashchiki
index 11785be8..7a3d1c14 160000
--- a/vendor/yashchiki
+++ b/vendor/yashchiki
@@ -1 +1 @@
-Subproject commit 11785be8a6e914a8404757670f58b4e78fca2bf9
+Subproject commit 7a3d1c1422fcec27b62c491dd03078b73b933a0b
--
GitLab
From 457bf472ec94502d62eb6de330351850faea9ee0 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Wed, 30 Apr 2025 17:25:37 +0300
Subject: [PATCH 05/13] VT-94: adapt to new changes from dedal
---
.gitlab-ci.yml | 6 ++
install_spack_env.py | 194 +++++++++++++++++++++++++++++++++++--------
vendor/yashchiki | 2 +-
3 files changed, 166 insertions(+), 36 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6f1ae550..fa2c7e8f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -246,6 +246,8 @@ build-spack-env-on-runner:
artifacts:
paths:
- spack_logs
+ - .dedal.log
+ - .generate_cache.log
# - spack_tests
when: always
timeout: 2 days
@@ -297,6 +299,8 @@ sync-esd-image:
artifacts:
paths:
- spack_logs
+ - .dedal.log
+ - .generate_cache.log
when: always
timeout: 2 days
resource_group: registry-esd-master-image
@@ -344,6 +348,8 @@ sync-gitlab-spack-instance:
artifacts:
paths:
- spack_logs
+ - .dedal.log
+ - .generate_cache.log
when: always
rules:
# branches that update the gitlab-runner upstream (read-only) installation
diff --git a/install_spack_env.py b/install_spack_env.py
index 55628af4..4e225163 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -69,17 +69,41 @@ upstream_instance = args.upstream_instance
update_spack_oci_caches = args.update_spack_oci_caches
oci_cache_prefix = args.oci_cache_prefix
-system_name = os.getenv('SYSTEMNAME')
+# define SYSTEMNAME variable in sites where it's not already defined
+system_name = (
+ os.getenv('SYSTEMNAME')
+ or os.getenv('HPC_SYSTEM')
+ or os.getenv('BSC_MACHINE')
+)
+
+# disable local configuration and cache directories
+set_bashrc_variable('SPACK_DISABLE_LOCAL_CONFIG', 'true')
+os.environ['SPACK_DISABLE_LOCAL_CONFIG'] = 'true'
+set_bashrc_variable('SPACK_USER_CACHE_PATH', '/tmp/spack')
+os.environ['SPACK_USER_CACHE_PATH'] = '/tmp/spack'
-set_bashrc_variable('OCI_CACHE_PREFIX', oci_cache_prefix)
+if oci_cache_prefix:
+ set_bashrc_variable('OCI_CACHE_PREFIX', oci_cache_prefix)
+ os.environ['OCI_CACHE_PREFIX'] = oci_cache_prefix
+# make sure spack uses the symlinked folder as path
ci_spack_root = installation_root / 'spack'
-set_bashrc_variable('CACHE_SPECFILE', f'env_specfile.yaml')
-set_bashrc_variable('DEDAL_HOME', f'{ebrains_repo}/vendor/yashchiki')
-set_bashrc_variable('SPACK_CACHE_SOURCE', f'{ci_spack_root}/var/spack/cache')
-set_bashrc_variable('SPACK_CACHE_BUILD', f'{ci_spack_root}/var/spack/cache')
+# cache related variables
+cache_specfile = os.environ.get("CACHE_SPECFILE", "env_specfile.yaml")
+set_bashrc_variable('CACHE_SPECFILE', cache_specfile)
+os.environ['CACHE_SPECFILE'] = cache_specfile
+dedal_home = f'{ebrains_repo}/vendor/yashchiki'
+os.environ['DEDAL_HOME'] = dedal_home
+set_bashrc_variable('DEDAL_HOME', dedal_home)
+spack_cache_source = f'{ci_spack_root}/var/spack/cache'
+set_bashrc_variable('SPACK_CACHE_SOURCE', spack_cache_source)
+os.environ['SPACK_CACHE_SOURCE'] = spack_cache_source
+set_bashrc_variable('SPACK_CACHE_BUILD', spack_cache_source)
+os.environ['SPACK_CACHE_BUILD'] = spack_cache_source
+spack_cache_build = spack_cache_source
+# initial setup: use spack submodule if spack dir doesn't already exist
spack_root_existed = True
if ci_spack_root and ebrains_repo:
if not os.path.isdir(ci_spack_root):
@@ -91,61 +115,161 @@ if ci_spack_root and ebrains_repo:
pass
data_dir = installation_root / 'caching'
-print(installation_root.parent)
-env_repo = SpackDescriptor('ebrains-spack-builds', path=installation_root.parent)
+
+env_repo = SpackDescriptor(ebrains_spack_env, path=ci_spack_root / 'var/spack/environments')
+
+binary_cache_path = Path(os.getenv('SPACK_CACHE_BUILD'))
spack_config = SpackConfig(env=env_repo,
- repos=[env_repo],
+ repos=[],
install_dir=installation_root.parent,
upstream_instance=upstream_instance,
- system_name='VBT',
- concretization_dir=data_dir / 'concretize_cache',
- buildcache_dir=data_dir / 'binary_cache',
+ system_name=system_name,
+ buildcache_dir=binary_cache_path,
use_spack_global=False,
cache_version_build='spack_cache',
view=SpackViewEnum.WITHOUT_VIEW,
spack_dir=Path(ci_spack_root).resolve())
spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=False)
+spack_operation.setup_spack_env()
+
if upstream_instance:
upstream_prefix = find_first_upstream_prefix(upstream_instance)
spack_operation.config(SpackConfigCommand.ADD, f'upstreams:upstream-spack-instance:install_tree:{upstream_prefix}')
if spack_root_existed == 0:
spack_operation.config(SpackConfigCommand.ADD, 'config:install_tree:padded_length:128')
+# make sure all fetching/clingo stuff happens before anything else
spack_operation.spec_pacakge('aida')
+# rebuild spack's database (could be an debugging session)
spack_operation.reindex()
-spack_operation.add_mirror(mirror_name='local_cache', mirror_path=Path(os.getenv('SPACK_CACHE_BUILD')).resolve())
-spack_operation.setup_spack_env()
+spack_operation.add_mirror(mirror_name='local_cache', mirror_path=binary_cache_path)
-site_config_dest = ci_spack_root / 'var/spack/environments' / ebrains_spack_env
+site_config_dest = ci_spack_root / 'var/spack/environments' / ebrains_spack_env / 'site-config'
if site_config_dest.exists():
- site_config_dest.rmdir()
- site_config_dest.mkdir(parents=True, exist_ok=True)
- site_config_esd = Path(ebrains_spack_env) / 'site-config'
- shutil.copy(site_config_esd, site_config_dest)
- # spack-python /path/to/first/spack.yaml /path/to/second/spack.yaml
- # merged_envs = merge_spack_envs(Path(ebrains_spack_env) / 'spack.yaml', site_config_esd / os.getenv('SYSTEMNAME') / 'spack.yaml')
- merged_envs = result = run_command(
- "spack-python",
- f'{ebrains_repo}/site-config/ymerge.py {ebrains_repo}/spack.yaml',
- f'{ebrains_repo}/site-config/{system_name}/spack.yaml',
- info_msg='Merging top-level and site-specific spack.yaml files.',
- exception_msg='Failed to merge top-level and site-specific spack.yaml files.',
- capture_output=True,
- text=True,
- check=True
- )
+ shutil.rmtree(site_config_dest / 'site-config')
+ os.makedirs(site_config_dest / 'site-config')
+
+site_config = Path(ebrains_repo) / 'site-config'
+shutil.copytree(site_config, site_config_dest)
+
+y_merge_path = Path(ebrains_repo) / 'site-config/ymerge.py'
+merge_path_1 = Path(ebrains_repo) / 'spack.yaml'
+merge_path_2 = Path(ebrains_repo) / f'site-config/{system_name}/spack.yaml'
+
+# update environment site-configs
+merged_envs = run_command(
+ "bash", "-c",
+ f'{spack_operation.spack_setup_script} && spack-python {y_merge_path} {merge_path_1} {merge_path_2}',
+ info_msg='Merging top-level and site-specific spack.yaml files.',
+ exception_msg='Failed to merge top-level and site-specific spack.yaml files.',
+ capture_output=True,
+ text=True,
+ check=True
+).stdout
+
+if merged_envs is None:
+ sys.exit(-1)
+else:
tmp_spack_yaml = Path("/tmp/spack.yaml").resolve()
try:
with open(tmp_spack_yaml, "w") as f:
- f.write(merged_envs.stdout.strip())
- print(f"Written merged spack.yaml to {tmp_spack_yaml}")
+ f.write(merged_envs)
except Exception as e:
sys.stderr.write(f"Error writing {tmp_spack_yaml}: {e}\n")
sys.exit(1)
- shutil.copy(tmp_spack_yaml, site_config_dest)
+ shutil.copy(tmp_spack_yaml, site_config_dest.parent)
+
+# add repo if it does not exist
+spack_operation.add_spack_repo(repo_name='ebrains-spack-builds', repo_path=installation_root.parent)
+spack_operation.concretize_spack_env(force=True, fresh=True, test='root')
+
+dump_dag = spack_operation.spack_env_operation.spec_pacakge('-y', True)
-spack_operation.concretize_spack_env()
+try:
+ with open(cache_specfile, "w") as f:
+ f.write(dump_dag)
+except Exception as e:
+ sys.stderr.write(f"Failed to dump dag to file: {dump_dag}: {e}\n")
+ sys.exit(1)
+
+if oci_cache_prefix:
+ # fetch missing sources (if packages not yet installed)
+ fetch_cached_sources = Path(dedal_home).resolve() / '/cli' / 'fetch_cached_sources.py'
+ run_command(
+ "bash", "-c",
+ f"{spack_operation.spack_setup_script} && python3 "
+ f"{fetch_cached_sources} "
+ f"--local-cache spack_cache_source "
+ f"--remote-cache-type oci "
+ f"--remote-cache {oci_cache_prefix}/source_cache "
+ f"--yashchiki-home {dedal_home} "
+ f"/tmp/missing_paths_sources.dat cache_specfile",
+ info_msg="Fetching missing sources",
+ exception_msg="Failed to fetch missing sources",
+ check=True
+ )
+ # fetch missing build results (if packages not yet installed)
+ fetch_cached_buildresults = Path(dedal_home).resolve() / 'cache' / 'fetch_cached_buildresults.dat'
+ run_command(
+ "bash", "-c",
+ f"{spack_operation.spack_setup_script} && python3 "
+ f"{fetch_cached_buildresults} "
+ f"--local-cache",
+ f"{spack_cache_build}/build_cache "
+ f"--remote-cache-type oci "
+ f"--remote-cache {oci_cache_prefix}/build_cache "
+ f"--yashchiki-home {dedal_home} "
+ f"/tmp/missing_paths_buildresults.dat {cache_specfile}",
+ info_msg="Fetching missing build results",
+ exception_msg="Failed to fetch missing build results",
+ check=True
+ )
+
+run_command(
+ "bash", "-c",
+ f'{spack_operation.spack_setup_script} && spack-python',
+ f'exit(not len(spack.environment.active_environment().uninstalled_specs()))',
+ info_msg='Checking for uninstalled Spack specs',
+ check=True
+)
+
+spack_operation.fetch(dependencies=True, missing=True)
+
+if oci_cache_prefix and os.environ.get('UPDATE_SPACK_OCI_CACHES', 'false') == 'true':
+ print("Performing update of the source cache")
+ update_cached_sources = Path(dedal_home).resolve() / 'cli/update_cached_sources.py'
+ cache_cmd = run_command(
+ "bash", "-c",
+ f'{spack_operation.spack_setup_script} && python3 {update_cached_sources} '
+ f'--local-cache {spack_cache_source}'
+ f'--remote-cache-type oci '
+ f'--remote-cache f"{oci_cache_prefix}/source_cache '
+ f'/tmp/missing_paths_sources.dat',
+ info_msg='Updating remote OCI cache',
+ check=True
+ )
+ if cache_cmd is None or cache_cmd.returncode != 0:
+ print("Cache update failed.")
+ sys.exit(cache_cmd.returncode if cache_cmd else 1)
+else:
+ print("Updating of the source cache disabled.")
+
+if oci_cache_prefix:
+ specfile_dag_hash = Path(dedal_home).resolve() / 'cli/specfile_dag_hash.py'
+ dag_hashes_pre_install = run_command(
+ "bash", "-c",
+ f'{spack_operation.spack_setup_script} && spack-python specfile_dag_hash {cache_specfile}',
+ capture_output=True,
+ text=True,
+ check=True
+ ).stdout
spack_operation.install_packages(jobs=int(os.getenv('SPACK_JOBS')), signed=False, test='root')
+
+if not spack_operation.check_installed_spack_packages(Path(ebrains_repo).resolve()):
+ print('Some spack packages failed to install.')
+ sys.exit(-1)
+
+print('Installed all spack packages.')
spack_operation.reindex()
diff --git a/vendor/yashchiki b/vendor/yashchiki
index 7a3d1c14..ffc59776 160000
--- a/vendor/yashchiki
+++ b/vendor/yashchiki
@@ -1 +1 @@
-Subproject commit 7a3d1c1422fcec27b62c491dd03078b73b933a0b
+Subproject commit ffc597764769b6b01e8f75b7275caa4693cb3ac4
--
GitLab
From 4ef5addbb98b1d7ffb53335bf5f1587d8daa9bab Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 9 May 2025 10:54:12 +0300
Subject: [PATCH 06/13] VT-94: Caching migration to python
---
install_spack_env.py | 56 ++++++++++++++++++++++++++++++++++++++------
vendor/yashchiki | 2 +-
2 files changed, 50 insertions(+), 8 deletions(-)
diff --git a/install_spack_env.py b/install_spack_env.py
index 4e225163..aef88437 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -1,6 +1,7 @@
import os
import argparse
import shutil
+import subprocess
import sys
from pathlib import Path
from dedal.configuration.SpackConfig import SpackConfig
@@ -195,13 +196,13 @@ except Exception as e:
if oci_cache_prefix:
# fetch missing sources (if packages not yet installed)
- fetch_cached_sources = Path(dedal_home).resolve() / '/cli' / 'fetch_cached_sources.py'
+ fetch_cached_sources = Path(dedal_home).resolve() / 'cli' / 'fetch_cached_sources.py'
run_command(
"bash", "-c",
f"{spack_operation.spack_setup_script} && python3 "
f"{fetch_cached_sources} "
f"--local-cache spack_cache_source "
- f"--remote-cache-type oci "
+ f"--remote-cache-type=oci "
f"--remote-cache {oci_cache_prefix}/source_cache "
f"--yashchiki-home {dedal_home} "
f"/tmp/missing_paths_sources.dat cache_specfile",
@@ -210,14 +211,14 @@ if oci_cache_prefix:
check=True
)
# fetch missing build results (if packages not yet installed)
- fetch_cached_buildresults = Path(dedal_home).resolve() / 'cache' / 'fetch_cached_buildresults.dat'
+ fetch_cached_buildresults = Path(dedal_home).resolve() / 'cache' / 'fetch_cached_buildresults.py'
run_command(
"bash", "-c",
f"{spack_operation.spack_setup_script} && python3 "
f"{fetch_cached_buildresults} "
f"--local-cache",
f"{spack_cache_build}/build_cache "
- f"--remote-cache-type oci "
+ f"--remote-cache-type=oci "
f"--remote-cache {oci_cache_prefix}/build_cache "
f"--yashchiki-home {dedal_home} "
f"/tmp/missing_paths_buildresults.dat {cache_specfile}",
@@ -243,7 +244,7 @@ if oci_cache_prefix and os.environ.get('UPDATE_SPACK_OCI_CACHES', 'false') == 't
"bash", "-c",
f'{spack_operation.spack_setup_script} && python3 {update_cached_sources} '
f'--local-cache {spack_cache_source}'
- f'--remote-cache-type oci '
+ f'--remote-cache-type=oci'
f'--remote-cache f"{oci_cache_prefix}/source_cache '
f'/tmp/missing_paths_sources.dat',
info_msg='Updating remote OCI cache',
@@ -255,18 +256,59 @@ if oci_cache_prefix and os.environ.get('UPDATE_SPACK_OCI_CACHES', 'false') == 't
else:
print("Updating of the source cache disabled.")
+dag_hashes_pre_install = []
if oci_cache_prefix:
specfile_dag_hash = Path(dedal_home).resolve() / 'cli/specfile_dag_hash.py'
dag_hashes_pre_install = run_command(
"bash", "-c",
- f'{spack_operation.spack_setup_script} && spack-python specfile_dag_hash {cache_specfile}',
+ f'{spack_operation.spack_setup_script} && spack-python {specfile_dag_hash} {cache_specfile}',
capture_output=True,
text=True,
check=True
- ).stdout
+ ).stdout.strip().split()
spack_operation.install_packages(jobs=int(os.getenv('SPACK_JOBS')), signed=False, test='root')
+if oci_cache_prefix and update_spack_oci_caches.lower() == "true":
+ for dag_hash in dag_hashes_pre_install:
+ package = Path(spack_cache_build).resolve() / dag_hash
+ print(package)
+ result = spack_operation.create_build_cache(package=package, unsigned=True, only=True)
+ if result != 0:
+ print(f'Failed to push {dag_hash}, trying to call spack find on it:')
+ spack_operation.find_package(package=package, long=True, variants=True, paths=True)
+
+ # upload packages from local to remote cache
+ print("Performing update of the build cache")
+ update_cached_buildresults = Path(dedal_home).resolve() / 'cli/update_cached_buildresults.py'
+ local_cache = Path(spack_cache_build).resolve() / 'build_cache'
+ run_command(
+ "bash", "-c",
+ f'{spack_operation.spack_setup_script} && python3 {update_cached_buildresults} '
+ f'--local-cache {local_cache}'
+ f'--remote-cache-type=oci '
+ f'--remote-cache f"{local_cache} '
+ f'/tmp/missing_paths_sources.dat',
+ check=True
+ )
+else:
+ print('Updating of the build cache disabled.')
+
+spack_operation.reindex()
+
+exports = run_command("bash", "-c",
+ f'spack env activate --sh {ebrains_spack_env}',
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ capture_output=True,
+ text=True,
+ ).stdout
+
+out_file = Path(ci_spack_root).resolve() / f'var/spack/environments/{ebrains_spack_env}/load_env.sh'
+with open(out_file, "w") as f:
+ f.write(exports)
+
if not spack_operation.check_installed_spack_packages(Path(ebrains_repo).resolve()):
print('Some spack packages failed to install.')
sys.exit(-1)
diff --git a/vendor/yashchiki b/vendor/yashchiki
index ffc59776..e21c2c08 160000
--- a/vendor/yashchiki
+++ b/vendor/yashchiki
@@ -1 +1 @@
-Subproject commit ffc597764769b6b01e8f75b7275caa4693cb3ac4
+Subproject commit e21c2c085cdf1712aedda75caea961f152d460f0
--
GitLab
From 5e53a433cbfab85749f3e442226c3f84a5ced4a0 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Mon, 12 May 2025 15:23:08 +0300
Subject: [PATCH 07/13] VT-94: Installing spack before running
install_spack_env.py script
---
.gitlab-ci.yml | 6 ++++++
install_spack_env.py | 19 ++++++++++---------
2 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fa2c7e8f..e4b80592 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -229,6 +229,8 @@ build-spack-env-on-runner:
script:
# install dedal python library
- pip3 install --break-system-packages vendor/yashchiki
+ # install spack
+ - source vendor/spack/share/spack/setup-env.sh
# deactivate environment views (we don't need them for the test build-job)
- >
echo " view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml
@@ -276,6 +278,8 @@ sync-esd-image:
# run installation script inside future container environment
# => DAG concretization, subsequent cache access + fetching and actual build should be separate steps
- mkdir --mode=0777 -p ${SANDBOX_ROOT}/${INSTALLATION_ROOT}
+ # install spack
+ - source vendor/spack/share/spack/setup-env.sh
- apptainer exec --writable --bind ${CI_PROJECT_DIR}:${INSTALLATION_ROOT} --cwd ${INSTALLATION_ROOT} ${SANDBOX_ROOT} python3 install_spack_env.py $SPACK_JOBS $INSTALLATION_ROOT ${INSTALLATION_ROOT} $CI_SPACK_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
- echo "export SYSTEMNAME=${SYSTEMNAME}" >> ${SANDBOX_ROOT}/.singularity.d/env/90-environment.sh
- echo ". ${INSTALLATION_ROOT}/vendor/spack/var/spack/environments/${CI_SPACK_ENV}/load_env.sh" >> ${SANDBOX_ROOT}/.singularity.d/env/90-environment.sh
@@ -331,6 +335,8 @@ sync-gitlab-spack-instance:
- git fetch origin
- git reset --hard $CI_COMMIT_SHA
- git submodule update --force
+ # install spack
+ - source vendor/spack/share/spack/setup-env.sh
# run installation script
- python3 install_spack_env.py $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_REPO_PATH $SPACK_NFS_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
# create kernel spec, so that the environment can be used in gitlab CI jobs
diff --git a/install_spack_env.py b/install_spack_env.py
index aef88437..ea0ca80c 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -146,7 +146,8 @@ spack_operation.spec_pacakge('aida')
spack_operation.reindex()
spack_operation.add_mirror(mirror_name='local_cache', mirror_path=binary_cache_path)
-site_config_dest = ci_spack_root / 'var/spack/environments' / ebrains_spack_env / 'site-config'
+env_path = ci_spack_root / 'var/spack/environments' / ebrains_spack_env
+site_config_dest = env_path / 'site-config'
if site_config_dest.exists():
shutil.rmtree(site_config_dest / 'site-config')
os.makedirs(site_config_dest / 'site-config')
@@ -161,7 +162,7 @@ merge_path_2 = Path(ebrains_repo) / f'site-config/{system_name}/spack.yaml'
# update environment site-configs
merged_envs = run_command(
"bash", "-c",
- f'{spack_operation.spack_setup_script} && spack-python {y_merge_path} {merge_path_1} {merge_path_2}',
+ f'spack-python {y_merge_path} {merge_path_1} {merge_path_2}',
info_msg='Merging top-level and site-specific spack.yaml files.',
exception_msg='Failed to merge top-level and site-specific spack.yaml files.',
capture_output=True,
@@ -199,7 +200,7 @@ if oci_cache_prefix:
fetch_cached_sources = Path(dedal_home).resolve() / 'cli' / 'fetch_cached_sources.py'
run_command(
"bash", "-c",
- f"{spack_operation.spack_setup_script} && python3 "
+ f"python3 "
f"{fetch_cached_sources} "
f"--local-cache spack_cache_source "
f"--remote-cache-type=oci "
@@ -214,7 +215,7 @@ if oci_cache_prefix:
fetch_cached_buildresults = Path(dedal_home).resolve() / 'cache' / 'fetch_cached_buildresults.py'
run_command(
"bash", "-c",
- f"{spack_operation.spack_setup_script} && python3 "
+ f"python3 "
f"{fetch_cached_buildresults} "
f"--local-cache",
f"{spack_cache_build}/build_cache "
@@ -229,7 +230,7 @@ if oci_cache_prefix:
run_command(
"bash", "-c",
- f'{spack_operation.spack_setup_script} && spack-python',
+ f'spack-python',
f'exit(not len(spack.environment.active_environment().uninstalled_specs()))',
info_msg='Checking for uninstalled Spack specs',
check=True
@@ -242,7 +243,7 @@ if oci_cache_prefix and os.environ.get('UPDATE_SPACK_OCI_CACHES', 'false') == 't
update_cached_sources = Path(dedal_home).resolve() / 'cli/update_cached_sources.py'
cache_cmd = run_command(
"bash", "-c",
- f'{spack_operation.spack_setup_script} && python3 {update_cached_sources} '
+ f'python3 {update_cached_sources} '
f'--local-cache {spack_cache_source}'
f'--remote-cache-type=oci'
f'--remote-cache f"{oci_cache_prefix}/source_cache '
@@ -261,7 +262,7 @@ if oci_cache_prefix:
specfile_dag_hash = Path(dedal_home).resolve() / 'cli/specfile_dag_hash.py'
dag_hashes_pre_install = run_command(
"bash", "-c",
- f'{spack_operation.spack_setup_script} && spack-python {specfile_dag_hash} {cache_specfile}',
+ f'spack-python {specfile_dag_hash} {cache_specfile}',
capture_output=True,
text=True,
check=True
@@ -284,7 +285,7 @@ if oci_cache_prefix and update_spack_oci_caches.lower() == "true":
local_cache = Path(spack_cache_build).resolve() / 'build_cache'
run_command(
"bash", "-c",
- f'{spack_operation.spack_setup_script} && python3 {update_cached_buildresults} '
+ f'python3 {update_cached_buildresults} '
f'--local-cache {local_cache}'
f'--remote-cache-type=oci '
f'--remote-cache f"{local_cache} '
@@ -297,7 +298,7 @@ else:
spack_operation.reindex()
exports = run_command("bash", "-c",
- f'spack env activate --sh {ebrains_spack_env}',
+ f'spack env activate --sh {env_path}',
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
--
GitLab
From 2ec82ccdb981ab395a53d390e5e1cd90c06e76c4 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Tue, 13 May 2025 10:57:24 +0300
Subject: [PATCH 08/13] VT-94: compatibility with spack-python and added
dynamic dedal spack package
VT-94: compatibility with spack-python and added dynamic dedal spack package
VT-94: compatibility with spack-python
---
install_spack_env.py | 186 ++++++++++++++++---------------
packages/py-dedal/package.py | 30 +++++
{utils => tools}/__init__.py | 0
utils/utils.py => tools/tools.py | 0
{utils => tools}/ymerge.py | 0
5 files changed, 126 insertions(+), 90 deletions(-)
create mode 100644 packages/py-dedal/package.py
rename {utils => tools}/__init__.py (100%)
rename utils/utils.py => tools/tools.py (100%)
rename {utils => tools}/ymerge.py (100%)
diff --git a/install_spack_env.py b/install_spack_env.py
index ea0ca80c..f3d504f4 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -1,7 +1,7 @@
import os
import argparse
import shutil
-import subprocess
+import spack
import sys
from pathlib import Path
from dedal.configuration.SpackConfig import SpackConfig
@@ -10,7 +10,8 @@ from dedal.enum.SpackViewEnum import SpackViewEnum
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.utils.utils import set_bashrc_variable, run_command
-from utils.utils import find_first_upstream_prefix
+
+from tools.tools import find_first_upstream_prefix
parser = argparse.ArgumentParser(
prog='install_spack_env.py',
@@ -198,9 +199,10 @@ except Exception as e:
if oci_cache_prefix:
# fetch missing sources (if packages not yet installed)
fetch_cached_sources = Path(dedal_home).resolve() / 'cli' / 'fetch_cached_sources.py'
+ print(fetch_cached_sources)
run_command(
"bash", "-c",
- f"python3 "
+ f"{spack_operation.spack_command_on_env} && python3 "
f"{fetch_cached_sources} "
f"--local-cache spack_cache_source "
f"--remote-cache-type=oci "
@@ -213,9 +215,10 @@ if oci_cache_prefix:
)
# fetch missing build results (if packages not yet installed)
fetch_cached_buildresults = Path(dedal_home).resolve() / 'cache' / 'fetch_cached_buildresults.py'
+ print(fetch_cached_buildresults)
run_command(
"bash", "-c",
- f"python3 "
+ f"{spack_operation.spack_command_on_env} && python3 "
f"{fetch_cached_buildresults} "
f"--local-cache",
f"{spack_cache_build}/build_cache "
@@ -228,91 +231,94 @@ if oci_cache_prefix:
check=True
)
-run_command(
+packages_not_installed = run_command(
"bash", "-c",
- f'spack-python',
- f'exit(not len(spack.environment.active_environment().uninstalled_specs()))',
+ f'{spack_operation.spack_command_on_env} && spack-python exit(not len(spack.environment.active_environment().uninstalled_specs()))',
info_msg='Checking for uninstalled Spack specs',
- check=True
-)
-
-spack_operation.fetch(dependencies=True, missing=True)
-
-if oci_cache_prefix and os.environ.get('UPDATE_SPACK_OCI_CACHES', 'false') == 'true':
- print("Performing update of the source cache")
- update_cached_sources = Path(dedal_home).resolve() / 'cli/update_cached_sources.py'
- cache_cmd = run_command(
- "bash", "-c",
- f'python3 {update_cached_sources} '
- f'--local-cache {spack_cache_source}'
- f'--remote-cache-type=oci'
- f'--remote-cache f"{oci_cache_prefix}/source_cache '
- f'/tmp/missing_paths_sources.dat',
- info_msg='Updating remote OCI cache',
- check=True
- )
- if cache_cmd is None or cache_cmd.returncode != 0:
- print("Cache update failed.")
- sys.exit(cache_cmd.returncode if cache_cmd else 1)
-else:
- print("Updating of the source cache disabled.")
-
-dag_hashes_pre_install = []
-if oci_cache_prefix:
- specfile_dag_hash = Path(dedal_home).resolve() / 'cli/specfile_dag_hash.py'
- dag_hashes_pre_install = run_command(
- "bash", "-c",
- f'spack-python {specfile_dag_hash} {cache_specfile}',
- capture_output=True,
- text=True,
- check=True
- ).stdout.strip().split()
-
-spack_operation.install_packages(jobs=int(os.getenv('SPACK_JOBS')), signed=False, test='root')
-
-if oci_cache_prefix and update_spack_oci_caches.lower() == "true":
- for dag_hash in dag_hashes_pre_install:
- package = Path(spack_cache_build).resolve() / dag_hash
- print(package)
- result = spack_operation.create_build_cache(package=package, unsigned=True, only=True)
- if result != 0:
- print(f'Failed to push {dag_hash}, trying to call spack find on it:')
- spack_operation.find_package(package=package, long=True, variants=True, paths=True)
-
- # upload packages from local to remote cache
- print("Performing update of the build cache")
- update_cached_buildresults = Path(dedal_home).resolve() / 'cli/update_cached_buildresults.py'
- local_cache = Path(spack_cache_build).resolve() / 'build_cache'
- run_command(
- "bash", "-c",
- f'python3 {update_cached_buildresults} '
- f'--local-cache {local_cache}'
- f'--remote-cache-type=oci '
- f'--remote-cache f"{local_cache} '
- f'/tmp/missing_paths_sources.dat',
- check=True
- )
-else:
- print('Updating of the build cache disabled.')
-
-spack_operation.reindex()
-
-exports = run_command("bash", "-c",
- f'spack env activate --sh {env_path}',
- check=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- capture_output=True,
- text=True,
- ).stdout
-
-out_file = Path(ci_spack_root).resolve() / f'var/spack/environments/{ebrains_spack_env}/load_env.sh'
-with open(out_file, "w") as f:
- f.write(exports)
-
-if not spack_operation.check_installed_spack_packages(Path(ebrains_repo).resolve()):
- print('Some spack packages failed to install.')
- sys.exit(-1)
-
-print('Installed all spack packages.')
-spack_operation.reindex()
+).returncode == 0
+
+print(f'results: {packages_not_installed}')
+
+if packages_not_installed:
+ spack_operation.fetch(dependencies=True, missing=True)
+#
+# if oci_cache_prefix and os.environ.get('UPDATE_SPACK_OCI_CACHES', 'false') == 'true':
+# print("Performing update of the source cache")
+# update_cached_sources = Path(dedal_home).resolve() / 'cli/update_cached_sources.py'
+# cache_cmd = run_command(
+# "bash", "-c",
+# f'{spack_operation.spack_command_on_env} && python3 {update_cached_sources} '
+# f'--local-cache {spack_cache_source}'
+# f'--remote-cache-type=oci'
+# f'--remote-cache f"{oci_cache_prefix}/source_cache '
+# f'/tmp/missing_paths_sources.dat',
+# info_msg='Updating remote OCI cache',
+# check=True
+# )
+# if cache_cmd is None or cache_cmd.returncode != 0:
+# print("Cache update failed.")
+# sys.exit(cache_cmd.returncode if cache_cmd else 1)
+# else:
+# print("Updating of the source cache disabled.")
+#
+# dag_hashes_pre_install = []
+# if oci_cache_prefix:
+# specfile_dag_hash = Path(dedal_home).resolve() / 'cli/specfile_dag_hash.py'
+# dag_hashes_pre_install = run_command(
+# "bash", "-c",
+# f'{spack_operation.spack_command_on_env} && spack-python {specfile_dag_hash} {cache_specfile}',
+# capture_output=True,
+# text=True,
+# check=True
+# ).stdout.strip().split()
+#
+# print(dag_hashes_pre_install)
+
+# spack_operation.install_packages(jobs=int(os.getenv('SPACK_JOBS')), signed=False, test='root')
+#
+# if oci_cache_prefix and update_spack_oci_caches.lower() == "true":
+# for dag_hash in dag_hashes_pre_install:
+# package = Path(spack_cache_build).resolve() / dag_hash
+# print(package)
+# result = spack_operation.create_build_cache(package=package, unsigned=True, only=True)
+# if result != 0:
+# print(f'Failed to push {dag_hash}, trying to call spack find on it:')
+# spack_operation.find_package(package=package, long=True, variants=True, paths=True)
+#
+# # upload packages from local to remote cache
+# print("Performing update of the build cache")
+# update_cached_buildresults = Path(dedal_home).resolve() / 'cli/update_cached_buildresults.py'
+# local_cache = Path(spack_cache_build).resolve() / 'build_cache'
+# run_command(
+# "bash", "-c",
+# f'{spack_operation.spack_command_on_env} && python3 {update_cached_buildresults} '
+# f'--local-cache {local_cache}'
+# f'--remote-cache-type=oci '
+# f'--remote-cache f"{local_cache} '
+# f'/tmp/missing_paths_sources.dat',
+# check=True
+# )
+# else:
+# print('Updating of the build cache disabled.')
+#
+# spack_operation.reindex()
+#
+# exports = run_command("bash", "-c",
+# f'spack env activate --sh {env_path}',
+# check=True,
+# stdout=subprocess.PIPE,
+# stderr=subprocess.PIPE,
+# capture_output=True,
+# text=True,
+# ).stdout
+#
+# out_file = Path(ci_spack_root).resolve() / f'var/spack/environments/{ebrains_spack_env}/load_env.sh'
+# with open(out_file, "w") as f:
+# f.write(exports)
+#
+# if not spack_operation.check_installed_spack_packages(Path(ebrains_repo).resolve()):
+# print('Some spack packages failed to install.')
+# sys.exit(-1)
+#
+# print('Installed all spack packages.')
+# spack_operation.reindex()
diff --git a/packages/py-dedal/package.py b/packages/py-dedal/package.py
new file mode 100644
index 00000000..bac5d942
--- /dev/null
+++ b/packages/py-dedal/package.py
@@ -0,0 +1,30 @@
+import subprocess
+from pathlib import Path
+
+from spack.package import PythonPackage
+
+_project_path = Path('.').resolve() / 'vendor/yashchiki'
+
+def _current_head():
+ """Return the full SHA of HEAD in the local git repo."""
+ return (
+ subprocess
+ .check_output(["git", "rev-parse", "HEAD"], cwd=_project_path)
+ .decode("utf-8")
+ .strip()
+ )
+
+class Myproject(PythonPackage):
+ homepage = f"file://{_project_path}"
+ git = f"file://{_project_path}"
+ print(homepage)
+ version(
+ "experimental",
+ commit=_current_head(),
+ get_full_repo=True,
+ submodules=True,
+ preferred=True
+ )
+
+ depends_on("python@3.10:", type=("build", "run"))
+ depends_on("py-setuptools", type="build")
\ No newline at end of file
diff --git a/utils/__init__.py b/tools/__init__.py
similarity index 100%
rename from utils/__init__.py
rename to tools/__init__.py
diff --git a/utils/utils.py b/tools/tools.py
similarity index 100%
rename from utils/utils.py
rename to tools/tools.py
diff --git a/utils/ymerge.py b/tools/ymerge.py
similarity index 100%
rename from utils/ymerge.py
rename to tools/ymerge.py
--
GitLab
From 39c7f08cd6222149f5fefe4fa9eb6e06db80a04e Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Thu, 15 May 2025 11:11:19 +0300
Subject: [PATCH 09/13] VT-94: added dynamic dedal spack package
---
install_spack_env.py | 15 ++++---
oras/package.py | 47 ++++++++++++++++++++++
{packages/py-dedal => py-dedal}/package.py | 14 +++++--
3 files changed, 68 insertions(+), 8 deletions(-)
create mode 100644 oras/package.py
rename {packages/py-dedal => py-dedal}/package.py (56%)
diff --git a/install_spack_env.py b/install_spack_env.py
index f3d504f4..de6d44dc 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -1,6 +1,8 @@
import os
import argparse
import shutil
+import subprocess
+
import spack
import sys
from pathlib import Path
@@ -231,11 +233,14 @@ if oci_cache_prefix:
check=True
)
-packages_not_installed = run_command(
- "bash", "-c",
- f'{spack_operation.spack_command_on_env} && spack-python exit(not len(spack.environment.active_environment().uninstalled_specs()))',
- info_msg='Checking for uninstalled Spack specs',
-).returncode == 0
+cmd = [
+ "spack-python",
+ "-c",
+ "exit(not len(spack.environment.active_environment().uninstalled_specs()))"
+]
+
+packages_not_installed = subprocess.run(cmd).returncode == 0
+
print(f'results: {packages_not_installed}')
diff --git a/oras/package.py b/oras/package.py
new file mode 100644
index 00000000..1d723b4f
--- /dev/null
+++ b/oras/package.py
@@ -0,0 +1,47 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Oras(Package):
+ """OCI Registry as Storage enables libraries to push OCI Artifacts to OCI Conformant registries. This is a Python SDK for Python developers to empower them to do this in their applications."""
+
+ homepage = "https://oras.land"
+ git = "https://github.com/oras-project/oras-py"
+ pypi="oras/oras-0.2.31.tar.gz"
+
+ maintainers("vsoch")
+
+ license("MIT")
+
+ version("main", branch="main")
+ version("0.2.31", sha256="95c0a341359458747c2946dab47d584cc444d1f9d379b6d63fb7a84cabc54de4")
+
+ depends_on("go", type="build")
+ depends_on("gmake", type="build")
+
+ def setup_build_environment(self, env: EnvironmentModifications) -> None:
+ # Point GOPATH at the top of the staging dir for the build step.
+ env.prepend_path("GOPATH", self.stage.path)
+
+ def install(self, spec, prefix):
+ if self.spec.satisfies("platform=linux target=aarch64:"):
+ make("build-linux-arm64")
+ elif self.spec.satisfies("platform=linux"):
+ make("build-linux")
+ elif self.spec.satisfies("platform=darwin target=aarch64:"):
+ make("build-mac-arm64")
+ elif self.spec.satisfies("platform=darwin"):
+ make("build-mac")
+ elif self.spec.satisfies("platform=windows"):
+ make("build-windows")
+ mkdirp(prefix.bin)
+
+ oras = find("bin", "oras")
+ if not oras:
+ raise InstallError("Oras executable missing in bin.")
+ tty.debug("Found oras executable %s to move into install bin" % oras[0])
+ install(oras[0], prefix.bin)
\ No newline at end of file
diff --git a/packages/py-dedal/package.py b/py-dedal/package.py
similarity index 56%
rename from packages/py-dedal/package.py
rename to py-dedal/package.py
index bac5d942..61f3b043 100644
--- a/packages/py-dedal/package.py
+++ b/py-dedal/package.py
@@ -3,7 +3,7 @@ from pathlib import Path
from spack.package import PythonPackage
-_project_path = Path('.').resolve() / 'vendor/yashchiki'
+_project_path = Path('').resolve() / 'vendor/yashchiki'
def _current_head():
"""Return the full SHA of HEAD in the local git repo."""
@@ -14,7 +14,7 @@ def _current_head():
.strip()
)
-class Myproject(PythonPackage):
+class PyDedal(PythonPackage):
homepage = f"file://{_project_path}"
git = f"file://{_project_path}"
print(homepage)
@@ -27,4 +27,12 @@ class Myproject(PythonPackage):
)
depends_on("python@3.10:", type=("build", "run"))
- depends_on("py-setuptools", type="build")
\ No newline at end of file
+ depends_on("py-setuptools", type="build")
+ depends_on("oras", type="run")
+ # depends_on("spack", type="run")
+
+ # Python libraries (Spack packages are named py-<name>):
+ depends_on("py-ruamel-yaml", type=("build", "run"))
+ depends_on("py-click", type=("build", "run"))
+ depends_on("py-jsonpickle", type=("build", "run"))
+ depends_on("py-yaml", type=("build", "run"))
\ No newline at end of file
--
GitLab
From 97b6d01dbe7dd59e2d2e58edc8885fe0ed0ffa6a Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Thu, 15 May 2025 15:09:35 +0300
Subject: [PATCH 10/13] VT-94: compatibility with spack-python
---
.gitlab-ci.yml | 2 +-
create_spack_env.py | 0
dedal_install_spack_env.sh | 19 +++++++++++++++++++
install_spack_env.py | 3 ---
spack_tools/packages/__init__.py | 0
spack_tools/packages/oras/__init__.py | 0
.../packages/oras}/package.py | 0
spack_tools/packages/py-dedal/__init__.py | 0
.../packages/py-dedal}/package.py | 7 ++++---
spack_tools/repo.yaml | 2 ++
spack_tools/spack.yaml | 6 ++++++
11 files changed, 32 insertions(+), 7 deletions(-)
create mode 100644 create_spack_env.py
create mode 100644 dedal_install_spack_env.sh
create mode 100644 spack_tools/packages/__init__.py
create mode 100644 spack_tools/packages/oras/__init__.py
rename {oras => spack_tools/packages/oras}/package.py (100%)
create mode 100644 spack_tools/packages/py-dedal/__init__.py
rename {py-dedal => spack_tools/packages/py-dedal}/package.py (85%)
create mode 100644 spack_tools/repo.yaml
create mode 100644 spack_tools/spack.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e4b80592..140628ef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -235,7 +235,7 @@ build-spack-env-on-runner:
- >
echo " view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml
# run installation script
- - python3 install_spack_env.py $SPACK_JOBS $CI_PROJECT_DIR $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
+ - bash dedal_install_spack_env.sh $SPACK_JOBS $CI_PROJECT_DIR $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
after_script:
- mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
# for succesfully installed packages: keep the spack logs for any package modified during this CI job
diff --git a/create_spack_env.py b/create_spack_env.py
new file mode 100644
index 00000000..e69de29b
diff --git a/dedal_install_spack_env.sh b/dedal_install_spack_env.sh
new file mode 100644
index 00000000..f63d7885
--- /dev/null
+++ b/dedal_install_spack_env.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eo pipefail
+
+SPACK_JOBS=$1 # number of jobs
+INSTALLATION_ROOT=$2 # where to set up the installation
+EBRAINS_REPO=$3 # location of ebrains-spack-builds repository
+EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated
+UPSTREAM_INSTANCE=$5 # path to Spack instance to use as upstream (optional)
+UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sources and build results
+export OCI_CACHE_PREFIX=$7
+#python3 create_spack_env.py
+
+spack env activate -p ./spack_tools
+spack repo add ./spack_tools
+spack install --fresh
+spack load dedal
+
+spack-python install_spack_env.py "$SPACK_JOBS" "$INSTALLATION_ROOT" "$EBRAINS_REPO" "$EBRAINS_SPACK_ENV" "$UPSTREAM_INSTANCE" "$UPDATE_SPACK_OCI_CACHES" "$OCI_CACHE_PREFIX"
\ No newline at end of file
diff --git a/install_spack_env.py b/install_spack_env.py
index de6d44dc..77ab6efd 100644
--- a/install_spack_env.py
+++ b/install_spack_env.py
@@ -2,8 +2,6 @@ import os
import argparse
import shutil
import subprocess
-
-import spack
import sys
from pathlib import Path
from dedal.configuration.SpackConfig import SpackConfig
@@ -12,7 +10,6 @@ from dedal.enum.SpackViewEnum import SpackViewEnum
from dedal.model.SpackDescriptor import SpackDescriptor
from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
from dedal.utils.utils import set_bashrc_variable, run_command
-
from tools.tools import find_first_upstream_prefix
parser = argparse.ArgumentParser(
diff --git a/spack_tools/packages/__init__.py b/spack_tools/packages/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/spack_tools/packages/oras/__init__.py b/spack_tools/packages/oras/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/oras/package.py b/spack_tools/packages/oras/package.py
similarity index 100%
rename from oras/package.py
rename to spack_tools/packages/oras/package.py
diff --git a/spack_tools/packages/py-dedal/__init__.py b/spack_tools/packages/py-dedal/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/py-dedal/package.py b/spack_tools/packages/py-dedal/package.py
similarity index 85%
rename from py-dedal/package.py
rename to spack_tools/packages/py-dedal/package.py
index 61f3b043..b1cbca50 100644
--- a/py-dedal/package.py
+++ b/spack_tools/packages/py-dedal/package.py
@@ -3,7 +3,7 @@ from pathlib import Path
from spack.package import PythonPackage
-_project_path = Path('').resolve() / 'vendor/yashchiki'
+_project_path = Path('.').resolve() / 'vendor/yashchiki'
def _current_head():
"""Return the full SHA of HEAD in the local git repo."""
@@ -15,8 +15,9 @@ def _current_head():
)
class PyDedal(PythonPackage):
- homepage = f"file://{_project_path}"
- git = f"file://{_project_path}"
+ homepage = f"file:/{_project_path}"
+ git = f"file:/{_project_path}"
+ print('home page')
print(homepage)
version(
"experimental",
diff --git a/spack_tools/repo.yaml b/spack_tools/repo.yaml
new file mode 100644
index 00000000..40649dbe
--- /dev/null
+++ b/spack_tools/repo.yaml
@@ -0,0 +1,2 @@
+repo:
+ namespace: spack_tools
diff --git a/spack_tools/spack.yaml b/spack_tools/spack.yaml
new file mode 100644
index 00000000..289213eb
--- /dev/null
+++ b/spack_tools/spack.yaml
@@ -0,0 +1,6 @@
+spack:
+ specs:
+ - dedal
+
+ concretizer:
+ unify: true
--
GitLab
From 5ae4031e5cab5782a40e7fc72ba0cff1cd74dce7 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Thu, 15 May 2025 15:55:52 +0300
Subject: [PATCH 11/13] VT-94: compatibility with spack-python
---
dedal_install_spack_env.sh | 7 ++--
packages/py-dedal/package.py | 20 ++++++++++
packages/py-oras/package.py | 31 +++++++++++++++
spack_tools/packages/__init__.py | 0
spack_tools/packages/oras/__init__.py | 0
spack_tools/packages/oras/package.py | 47 -----------------------
spack_tools/packages/py-dedal/__init__.py | 0
spack_tools/packages/py-dedal/package.py | 39 -------------------
spack_tools/repo.yaml | 2 -
spack_tools/spack.yaml | 2 +-
10 files changed, 56 insertions(+), 92 deletions(-)
create mode 100644 packages/py-dedal/package.py
create mode 100644 packages/py-oras/package.py
delete mode 100644 spack_tools/packages/__init__.py
delete mode 100644 spack_tools/packages/oras/__init__.py
delete mode 100644 spack_tools/packages/oras/package.py
delete mode 100644 spack_tools/packages/py-dedal/__init__.py
delete mode 100644 spack_tools/packages/py-dedal/package.py
delete mode 100644 spack_tools/repo.yaml
diff --git a/dedal_install_spack_env.sh b/dedal_install_spack_env.sh
index f63d7885..6cf38ac7 100644
--- a/dedal_install_spack_env.sh
+++ b/dedal_install_spack_env.sh
@@ -12,8 +12,9 @@ export OCI_CACHE_PREFIX=$7
#python3 create_spack_env.py
spack env activate -p ./spack_tools
-spack repo add ./spack_tools
-spack install --fresh
-spack load dedal
+spack repo add ./
+spack concretize --force --fresh --test root
+spack install -j$SPACK_JOBS --fresh --test root
+spack load py-dedal
spack-python install_spack_env.py "$SPACK_JOBS" "$INSTALLATION_ROOT" "$EBRAINS_REPO" "$EBRAINS_SPACK_ENV" "$UPSTREAM_INSTANCE" "$UPDATE_SPACK_OCI_CACHES" "$OCI_CACHE_PREFIX"
\ No newline at end of file
diff --git a/packages/py-dedal/package.py b/packages/py-dedal/package.py
new file mode 100644
index 00000000..06bf1884
--- /dev/null
+++ b/packages/py-dedal/package.py
@@ -0,0 +1,20 @@
+from spack.package import PythonPackage
+
+class PyDedal(PythonPackage):
+ homepage = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/dedal.git'
+ git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/dedal.git'
+
+ version(
+ '1.0.0',
+ branch='VT-109-restructure-spack'
+ )
+
+ depends_on("python@3.10:", type=("build", "run"))
+ depends_on("py-setuptools", type="build")
+ depends_on("py-oras@0.2.31:", type="run")
+ # depends_on("spack", type="run")
+
+ depends_on("py-ruamel-yaml", type=("build", "run"))
+ depends_on("py-click", type=("build", "run"))
+ depends_on("py-jsonpickle", type=("build", "run"))
+ depends_on("py-pyyaml", type=("build", "run"))
\ No newline at end of file
diff --git a/packages/py-oras/package.py b/packages/py-oras/package.py
new file mode 100644
index 00000000..fa92ed11
--- /dev/null
+++ b/packages/py-oras/package.py
@@ -0,0 +1,31 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyOras(PythonPackage):
+ """ORAS Python SDK: OCI Registry as Storage Python SDK."""
+
+ homepage = "https://oras.land"
+ git = "https://github.com/oras-project/oras-py"
+ url = "https://files.pythonhosted.org/packages/28/86/cbae8797a1041fe3bef37f31aa1ecdd3f8914fbc7cfb663532255b6ec16e/oras-0.2.31.tar.gz"
+
+ maintainers("vsoch")
+
+ license("Apache 2.0 License")
+
+ version("0.2.31", sha256="95c0a341359458747c2946dab47d584cc444d1f9d379b6d63fb7a84cabc54de4")
+
+ depends_on("python@3.7:", type=("build", "run"))
+
+ depends_on("py-jsonschema", type=("build", "run"))
+ depends_on("py-requests", type=("build", "run"))
+
+ variant("tests", default=False, description="Enable test suite")
+ depends_on("py-pytest@4.6.2:", when="+tests", type=("build", "run"))
+
+ variant("docker", default=False, description="Enable Docker extra support")
+ depends_on("py-docker@5.0.1", when="+docker", type=("build", "run"))
\ No newline at end of file
diff --git a/spack_tools/packages/__init__.py b/spack_tools/packages/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/spack_tools/packages/oras/__init__.py b/spack_tools/packages/oras/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/spack_tools/packages/oras/package.py b/spack_tools/packages/oras/package.py
deleted file mode 100644
index 1d723b4f..00000000
--- a/spack_tools/packages/oras/package.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-
-from spack.package import *
-
-
-class Oras(Package):
- """OCI Registry as Storage enables libraries to push OCI Artifacts to OCI Conformant registries. This is a Python SDK for Python developers to empower them to do this in their applications."""
-
- homepage = "https://oras.land"
- git = "https://github.com/oras-project/oras-py"
- pypi="oras/oras-0.2.31.tar.gz"
-
- maintainers("vsoch")
-
- license("MIT")
-
- version("main", branch="main")
- version("0.2.31", sha256="95c0a341359458747c2946dab47d584cc444d1f9d379b6d63fb7a84cabc54de4")
-
- depends_on("go", type="build")
- depends_on("gmake", type="build")
-
- def setup_build_environment(self, env: EnvironmentModifications) -> None:
- # Point GOPATH at the top of the staging dir for the build step.
- env.prepend_path("GOPATH", self.stage.path)
-
- def install(self, spec, prefix):
- if self.spec.satisfies("platform=linux target=aarch64:"):
- make("build-linux-arm64")
- elif self.spec.satisfies("platform=linux"):
- make("build-linux")
- elif self.spec.satisfies("platform=darwin target=aarch64:"):
- make("build-mac-arm64")
- elif self.spec.satisfies("platform=darwin"):
- make("build-mac")
- elif self.spec.satisfies("platform=windows"):
- make("build-windows")
- mkdirp(prefix.bin)
-
- oras = find("bin", "oras")
- if not oras:
- raise InstallError("Oras executable missing in bin.")
- tty.debug("Found oras executable %s to move into install bin" % oras[0])
- install(oras[0], prefix.bin)
\ No newline at end of file
diff --git a/spack_tools/packages/py-dedal/__init__.py b/spack_tools/packages/py-dedal/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/spack_tools/packages/py-dedal/package.py b/spack_tools/packages/py-dedal/package.py
deleted file mode 100644
index b1cbca50..00000000
--- a/spack_tools/packages/py-dedal/package.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import subprocess
-from pathlib import Path
-
-from spack.package import PythonPackage
-
-_project_path = Path('.').resolve() / 'vendor/yashchiki'
-
-def _current_head():
- """Return the full SHA of HEAD in the local git repo."""
- return (
- subprocess
- .check_output(["git", "rev-parse", "HEAD"], cwd=_project_path)
- .decode("utf-8")
- .strip()
- )
-
-class PyDedal(PythonPackage):
- homepage = f"file:/{_project_path}"
- git = f"file:/{_project_path}"
- print('home page')
- print(homepage)
- version(
- "experimental",
- commit=_current_head(),
- get_full_repo=True,
- submodules=True,
- preferred=True
- )
-
- depends_on("python@3.10:", type=("build", "run"))
- depends_on("py-setuptools", type="build")
- depends_on("oras", type="run")
- # depends_on("spack", type="run")
-
- # Python libraries (Spack packages are named py-<name>):
- depends_on("py-ruamel-yaml", type=("build", "run"))
- depends_on("py-click", type=("build", "run"))
- depends_on("py-jsonpickle", type=("build", "run"))
- depends_on("py-yaml", type=("build", "run"))
\ No newline at end of file
diff --git a/spack_tools/repo.yaml b/spack_tools/repo.yaml
deleted file mode 100644
index 40649dbe..00000000
--- a/spack_tools/repo.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-repo:
- namespace: spack_tools
diff --git a/spack_tools/spack.yaml b/spack_tools/spack.yaml
index 289213eb..b618d6e7 100644
--- a/spack_tools/spack.yaml
+++ b/spack_tools/spack.yaml
@@ -1,6 +1,6 @@
spack:
specs:
- - dedal
+ - py-dedal
concretizer:
unify: true
--
GitLab
From 64c219811076c46a88be0051af56bce7b4a48bf1 Mon Sep 17 00:00:00 2001
From: adrianciu <adrian.ciu@codemart.ro>
Date: Fri, 16 May 2025 20:08:29 +0300
Subject: [PATCH 12/13] VT-94: spack dedal cache
---
{spack_tools => dedal_env}/spack.yaml | 0
dedal_install.py | 41 +++++++++++++++++++++++++++
dedal_install_spack_env.sh | 13 +++++++--
packages/py-dedal/package.py | 2 +-
vendor/yashchiki | 2 +-
5 files changed, 53 insertions(+), 5 deletions(-)
rename {spack_tools => dedal_env}/spack.yaml (100%)
create mode 100644 dedal_install.py
diff --git a/spack_tools/spack.yaml b/dedal_env/spack.yaml
similarity index 100%
rename from spack_tools/spack.yaml
rename to dedal_env/spack.yaml
diff --git a/dedal_install.py b/dedal_install.py
new file mode 100644
index 00000000..98681041
--- /dev/null
+++ b/dedal_install.py
@@ -0,0 +1,41 @@
+from pathlib import Path
+from dedal.configuration.GpgConfig import GpgConfig
+from dedal.configuration.SpackConfig import SpackConfig
+import os
+from dedal.spack_factory.SpackOperationCreator import SpackOperationCreator
+from dedal.utils.utils import count_files_in_folder
+
+from dedal.model.SpackDescriptor import SpackDescriptor
+
+if __name__ == "__main__":
+ dedal_env = SpackDescriptor(name='dedal_env', path=Path('./'))
+ ebrains_repo = SpackDescriptor(name='ebrains-spack-builds', path=Path('../'))
+ dedal_dir = 'dedal_install_dir'
+ install_dir = Path('./') / dedal_dir
+ os.makedirs(install_dir, exist_ok=True)
+ concretization_dir = install_dir / 'concretization'
+ buildcache_dir = install_dir / 'buildcache'
+ spack_config = SpackConfig(env=dedal_env,
+ repos=[ebrains_repo],
+ install_dir=install_dir,
+ upstream_instance=None,
+ system_name='Dedal',
+ concretization_dir=concretization_dir,
+ buildcache_dir=buildcache_dir,
+ gpg=None,
+ use_spack_global=False,
+ cache_version_build='latest',
+ cache_version_concretize='latest',
+ override_cache=False,
+ spack_dir=Path('./vendor').resolve() / 'spack'
+ )
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+ spack_operation.setup_spack_env()
+ if count_files_in_folder(concretization_dir) == 0 or count_files_in_folder(buildcache_dir) == 0:
+ print('No cache available')
+ gpg = GpgConfig('dedal', 'science@codemart.ro')
+ spack_config.gpg = gpg
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=False)
+
+ spack_operation.concretize_spack_env()
+ spack_operation.install_packages(os.cpu_count())
diff --git a/dedal_install_spack_env.sh b/dedal_install_spack_env.sh
index 6cf38ac7..ca48b31d 100644
--- a/dedal_install_spack_env.sh
+++ b/dedal_install_spack_env.sh
@@ -11,10 +11,17 @@ UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sou
export OCI_CACHE_PREFIX=$7
#python3 create_spack_env.py
+#spack env activate -p ./spack_tools
+#spack repo add ./
+#spack concretize --force --fresh --test root
+#spack install -j$SPACK_JOBS --fresh --test root
+python3 dedal_install.py
+
spack env activate -p ./spack_tools
-spack repo add ./
-spack concretize --force --fresh --test root
-spack install -j$SPACK_JOBS --fresh --test root
+spack uninstall dedal -y
+spack mirror remove local_cache
+# install the latest version of dedal from the specified branch (the cache version might not be the latest yet)
+spack install dedal
spack load py-dedal
spack-python install_spack_env.py "$SPACK_JOBS" "$INSTALLATION_ROOT" "$EBRAINS_REPO" "$EBRAINS_SPACK_ENV" "$UPSTREAM_INSTANCE" "$UPDATE_SPACK_OCI_CACHES" "$OCI_CACHE_PREFIX"
\ No newline at end of file
diff --git a/packages/py-dedal/package.py b/packages/py-dedal/package.py
index 06bf1884..f59e5fcd 100644
--- a/packages/py-dedal/package.py
+++ b/packages/py-dedal/package.py
@@ -6,7 +6,7 @@ class PyDedal(PythonPackage):
version(
'1.0.0',
- branch='VT-109-restructure-spack'
+ branch='VT-109-HPC'
)
depends_on("python@3.10:", type=("build", "run"))
diff --git a/vendor/yashchiki b/vendor/yashchiki
index e21c2c08..fce1d819 160000
--- a/vendor/yashchiki
+++ b/vendor/yashchiki
@@ -1 +1 @@
-Subproject commit e21c2c085cdf1712aedda75caea961f152d460f0
+Subproject commit fce1d81985701c79d8c8812719aa2719ec7fddfd
--
GitLab
From 87293d654d167dfddde1359581fdfe6321d6e0a5 Mon Sep 17 00:00:00 2001
From: adrianciu <adrianciu25@gmail.com>
Date: Mon, 19 May 2025 16:49:23 +0300
Subject: [PATCH 13/13] VT-94: fixing py-dedal spack package
---
dedal_install.py | 61 ++++++++++++++++++------------------
packages/py-dedal/package.py | 10 +++---
packages/py-oras/package.py | 2 ++
3 files changed, 38 insertions(+), 35 deletions(-)
diff --git a/dedal_install.py b/dedal_install.py
index 98681041..750efd15 100644
--- a/dedal_install.py
+++ b/dedal_install.py
@@ -7,35 +7,34 @@ from dedal.utils.utils import count_files_in_folder
from dedal.model.SpackDescriptor import SpackDescriptor
-if __name__ == "__main__":
- dedal_env = SpackDescriptor(name='dedal_env', path=Path('./'))
- ebrains_repo = SpackDescriptor(name='ebrains-spack-builds', path=Path('../'))
- dedal_dir = 'dedal_install_dir'
- install_dir = Path('./') / dedal_dir
- os.makedirs(install_dir, exist_ok=True)
- concretization_dir = install_dir / 'concretization'
- buildcache_dir = install_dir / 'buildcache'
- spack_config = SpackConfig(env=dedal_env,
- repos=[ebrains_repo],
- install_dir=install_dir,
- upstream_instance=None,
- system_name='Dedal',
- concretization_dir=concretization_dir,
- buildcache_dir=buildcache_dir,
- gpg=None,
- use_spack_global=False,
- cache_version_build='latest',
- cache_version_concretize='latest',
- override_cache=False,
- spack_dir=Path('./vendor').resolve() / 'spack'
- )
- spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
- spack_operation.setup_spack_env()
- if count_files_in_folder(concretization_dir) == 0 or count_files_in_folder(buildcache_dir) == 0:
- print('No cache available')
- gpg = GpgConfig('dedal', 'science@codemart.ro')
- spack_config.gpg = gpg
- spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=False)
+dedal_env = SpackDescriptor(name='dedal_env', path=Path('./'))
+ebrains_repo = SpackDescriptor(name='ebrains-spack-builds', path=Path('../'))
+dedal_dir = 'dedal_install_dir'
+install_dir = Path('./') / dedal_dir
+os.makedirs(install_dir, exist_ok=True)
+concretization_dir = install_dir / 'concretization'
+buildcache_dir = install_dir / 'buildcache'
+spack_config = SpackConfig(env=dedal_env,
+ repos=[ebrains_repo],
+ install_dir=install_dir,
+ upstream_instance=None,
+ system_name='Dedal',
+ concretization_dir=concretization_dir,
+ buildcache_dir=buildcache_dir,
+ gpg=None,
+ use_spack_global=False,
+ cache_version_build='latest',
+ cache_version_concretize='latest',
+ override_cache=False,
+ spack_dir=Path('./vendor').resolve() / 'spack'
+ )
+spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=True)
+spack_operation.setup_spack_env()
+if count_files_in_folder(concretization_dir) == 0 or count_files_in_folder(buildcache_dir) == 0:
+ print('No cache available')
+ gpg = GpgConfig('dedal', 'science@codemart.ro')
+ spack_config.gpg = gpg
+ spack_operation = SpackOperationCreator.get_spack_operator(spack_config, use_cache=False)
- spack_operation.concretize_spack_env()
- spack_operation.install_packages(os.cpu_count())
+spack_operation.concretize_spack_env()
+spack_operation.install_packages(os.cpu_count())
diff --git a/packages/py-dedal/package.py b/packages/py-dedal/package.py
index f59e5fcd..5514acb2 100644
--- a/packages/py-dedal/package.py
+++ b/packages/py-dedal/package.py
@@ -5,14 +5,16 @@ class PyDedal(PythonPackage):
git = 'https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/dedal.git'
version(
- '1.0.0',
+ '0.9.0',
branch='VT-109-HPC'
)
+ depends_on('py-setuptools', type='build')
+
depends_on("python@3.10:", type=("build", "run"))
- depends_on("py-setuptools", type="build")
- depends_on("py-oras@0.2.31:", type="run")
- # depends_on("spack", type="run")
+ depends_on("py-setuptools", type=("build", "run"))
+ depends_on("py-oras@0.2.31:", type=("build", "run"))
+ depends_on("spack", type=("build", "run"))
depends_on("py-ruamel-yaml", type=("build", "run"))
depends_on("py-click", type=("build", "run"))
diff --git a/packages/py-oras/package.py b/packages/py-oras/package.py
index fa92ed11..9f6b2226 100644
--- a/packages/py-oras/package.py
+++ b/packages/py-oras/package.py
@@ -21,6 +21,8 @@ class PyOras(PythonPackage):
depends_on("python@3.7:", type=("build", "run"))
+ depends_on('py-setuptools', type='build')
+
depends_on("py-jsonschema", type=("build", "run"))
depends_on("py-requests", type=("build", "run"))
--
GitLab