Skip to content
Snippets Groups Projects
Commit 7cf47d92 authored by Klaus Noelp's avatar Klaus Noelp
Browse files

use pyproject.toml

parent 743d2930
No related branches found
No related tags found
No related merge requests found
Showing
with 54 additions and 52 deletions
......@@ -76,6 +76,8 @@ def create_executor_to_replace_placeholder(
for key, value in placeholder_replacements.items():
launcher_args["prepend"] = launcher_args["prepend"].replace(key, value)
for key, value in placeholder_replacements.items():
provider_args["worker_init"] = provider_args["worker_init"].replace(key, value)
provider_args["launcher"] = old_executor.provider.launcher.__class__(
**launcher_args
)
......@@ -98,30 +100,38 @@ def create_apptainer_sandbox(
stdout="runinfo/cwlout_create_apptainer_sandbox.txt",
stderr="runinfo/cwlerr_create_apptainer_sandbox.txt",
) -> str:
return f"""
bash --login -c '
if [ ! -d "{sandbox_path}" ]; then
apptainer --verbose build --sandbox {sandbox_path} docker://{container_image}
apptainer --verbose exec --no-mount cwd,home --writable {sandbox_path} sh -c "
if [ -f /etc/alpine-release ]; then
apk add gcc musl-dev linux-headers bash
fi
pip install parsl@https://gitlab.ebrains.eu/noelp/parsl/-/archive/master/parsl-master.tar.gz \
globus-compute-endpoint@https://gitlab.ebrains.eu/noelp/globus-compute/-/archive/main/globus-compute-main.tar.gz#subdirectory=compute_endpoint \
"
touch {sandbox_path}/sandboxready
# TODO: Alpine linux tries to build "psutil" but has no gcc
# # uv is not a solution, it still needs to build psutil
# # uv supports musl Python distributions for versions >= v0.6.6
# # uv python install 3.11
# apptainer --verbose exec --no-mount home --writable {sandbox_path} sh -c "pip install uv && \
# uv venv --python 3.11.9 && source .venv/bin/activate && \
# uv pip install parsl@https://gitlab.ebrains.eu/noelp/parsl/-/archive/master/parsl-master.tar.gz \
# globus-compute-endpoint@https://gitlab.ebrains.eu/noelp/globus-compute/-/archive/main/globus-compute-main.tar.gz#subdirectory=compute_endpoint"
return f"""\
if [ ! -d "{sandbox_path}" ]; then
mkdir -p {sandbox_path}
apptainer --verbose build --force --sandbox {sandbox_path} docker://{container_image}
mkdir -p {sandbox_path}/{sandbox_path}
apptainer --verbose run --cleanenv --no-mount home --writable {sandbox_path} sh -c '
if [ -f /etc/alpine-release ]; then
apk add gcc musl-dev linux-headers bash
fi
if command -v uv > /dev/null; then
BIN=uv\ pip
elif command -v pip > /dev/null; then
BIN=pip
elif command -v python > /dev/null; then
python -m ensurepip
BIN=pip
else
echo uv, pip or python not found.
exit 1
fi
'
"""
\$BIN install --force-reinstall parsl@https://gitlab.ebrains.eu/noelp/parsl/-/archive/master/parsl-master.tar.gz \
globus-compute-endpoint@https://gitlab.ebrains.eu/noelp/globus-compute/-/archive/main/globus-compute-main.tar.gz#subdirectory=compute_endpoint \
simplewormhole@https://gitlab.ebrains.eu/noelp/simplewormhole/-/archive/main/simplewormhole-main.tar.gz \
cwl_app@https://gitlab.ebrains.eu/noelp/cwl_app/-/archive/master/cwl_app-master.tar.gz
'
touch {sandbox_path}/sandboxready
fi
while [ ! -f {sandbox_path}/sandboxready ];
do sleep 1
done
"""
def get_bash_command(cwl_obj, prepend_cmd, **user_args):
......@@ -321,7 +331,7 @@ class CWLApp:
executor_for_replacements : string or ParslExecutor
Executor to use as a template where placeholder_replacements are applied.
placeholder_replacements : dict
Apply replacements to executor template. Example: {"{sandbox_directory}": "/tmp/container_1"}
Apply replacements to executor template. Example: {"{sandbox_path}": "/tmp/container_1"}
apptainer_sandbox : bool
Write container image to a sandbox directory and install Parsl and Globus Compute in the sandbox.
cache : bool
......@@ -419,7 +429,7 @@ class CWLApp:
executors=[self.executor_for_setup],
)
sandbox_future = create_sandbox_app(
self.placeholder_replacements["{sandbox_directory}"],
self.placeholder_replacements["{sandbox_path}"],
self.cwl_container_image,
)
# make bash_app wait for sandbox
......@@ -431,12 +441,11 @@ class CWLApp:
f"{bash_app_args['command']}"
)
# TODO: check if container image contains Parsl and Globus Compute?
logger.warning(
f"Can't run a container image without Parsl or Globus Compute. Try replacements with apptainer=True or build new container image. Image: {self.cwl_container_image}"
logger.error(
f"Container image requires Parsl or Globus Compute. Try replacements with apptainer_sandbox=True or build new container image. Image: {self.cwl_container_image}"
)
if self.executor_for_replacements:
if not self.placeholder_replacements:
self.placeholder_replacements = {"{container_image}": self.cwl_container_image}
self.placeholder_replacements["{container_image}"] = self.cwl_container_image
new_executor = create_executor_to_replace_placeholder(
executor=self.executor_for_replacements,
placeholder_replacements=self.placeholder_replacements,
......@@ -445,6 +454,8 @@ class CWLApp:
else:
executors = self.executors
logger.warning(f"{executors=}")
@bash_app(
data_flow_kernel=self.data_flow_kernel,
cache=self.cache,
......
[project]
name = "cwl_app"
version = "0.2"
dependencies = [
# "parsl@https://gitlab.ebrains.eu/noelp/parsl/-/archive/master/parsl-master.tar.gz",
# "globus-compute-sdk@https://gitlab.ebrains.eu/noelp/globus-compute/-/archive/main/globus-compute-main.tar.gz#subdirectory=compute_sdk",
"cwl-utils==0.36",
"prettyprint",
"pyyaml",
"schema",
]
[project.optional-dependencies]
dev = ["pytest", "pytest-dependency"]
# Original code: https://github.com/Parsl/cwl-parsl/
from setuptools import setup
setup(
name="cwl_app",
version="0.1",
py_modules=["cwl_app"], # List of modules to include
install_requires=[
# "parsl",
# "globus-compute-sdk",
"cwl-utils==0.36",
"prettyprint",
"pyyaml",
"schema",
"pytest",
"pytest-dependency",
],
)
#!/bin/bash
uv venv && source .venv/bin/activate
uv pip install -e ../cwl_app
uv pip install -e ../parsl
uv pip install pytest
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
File moved
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment