From 9d190ba3e08aa224da1f4ec69c2dbd315207d669 Mon Sep 17 00:00:00 2001
From: Michail Alexakis <alexakis@athenarc.gr>
Date: Sat, 3 Jun 2023 16:40:14 +0300
Subject: [PATCH] Add some basic examples of workflows
---
README.md | 26 +++++++
base/job.yml | 118 +++++++++++++++++++++++++++++++
base/kustomization.yml | 10 +++
base/repositories.yaml | 4 ++
compile-java/1/env | 1 +
compile-java/1/input.yml | 4 ++
compile-java/1/kustomization.yml | 13 ++++
compile-java/1/streamflow.yml | 33 +++++++++
compile-java/1/work.cwl | 58 +++++++++++++++
compile-java/2/env | 1 +
compile-java/2/input.yml | 6 ++
compile-java/2/kustomization.yml | 13 ++++
compile-java/2/streamflow.yml | 33 +++++++++
compile-java/2/work.cwl | 61 ++++++++++++++++
echo/1/env | 1 +
echo/1/input.yml | 1 +
echo/1/kustomization.yml | 13 ++++
echo/1/streamflow.yml | 25 +++++++
echo/1/work.cwl | 50 +++++++++++++
echo/2/env | 1 +
echo/2/input.yml | 1 +
echo/2/kustomization.yml | 13 ++++
echo/2/streamflow.yml | 37 ++++++++++
echo/2/work.cwl | 51 +++++++++++++
echo/2a/env | 1 +
echo/2a/kustomization.yml | 12 ++++
echo/2a/streamflow.yml | 28 ++++++++
echo/3/env | 1 +
echo/3/input.yml | 1 +
echo/3/kustomization.yml | 13 ++++
echo/3/streamflow.yml | 25 +++++++
echo/3/work.cwl | 56 +++++++++++++++
echo/4/env | 1 +
echo/4/input.yml | 3 +
echo/4/kustomization.yml | 13 ++++
echo/4/streamflow.yml | 25 +++++++
echo/4/work.cwl | 52 ++++++++++++++
psd/1/env | 1 +
psd/1/input.yml | 7 ++
psd/1/kustomization.yml | 17 +++++
psd/1/psd_calc.cwl | 26 +++++++
psd/1/psd_vis.cwl | 26 +++++++
psd/1/streamflow.yml | 33 +++++++++
psd/1/work.cwl | 30 ++++++++
psd/README | 1 +
45 files changed, 946 insertions(+)
create mode 100644 base/job.yml
create mode 100644 base/kustomization.yml
create mode 100644 base/repositories.yaml
create mode 100644 compile-java/1/env
create mode 100644 compile-java/1/input.yml
create mode 100644 compile-java/1/kustomization.yml
create mode 100644 compile-java/1/streamflow.yml
create mode 100644 compile-java/1/work.cwl
create mode 100644 compile-java/2/env
create mode 100644 compile-java/2/input.yml
create mode 100644 compile-java/2/kustomization.yml
create mode 100644 compile-java/2/streamflow.yml
create mode 100644 compile-java/2/work.cwl
create mode 100644 echo/1/env
create mode 100644 echo/1/input.yml
create mode 100644 echo/1/kustomization.yml
create mode 100644 echo/1/streamflow.yml
create mode 100644 echo/1/work.cwl
create mode 100644 echo/2/env
create mode 100644 echo/2/input.yml
create mode 100644 echo/2/kustomization.yml
create mode 100644 echo/2/streamflow.yml
create mode 100644 echo/2/work.cwl
create mode 100644 echo/2a/env
create mode 100644 echo/2a/kustomization.yml
create mode 100644 echo/2a/streamflow.yml
create mode 100644 echo/3/env
create mode 100644 echo/3/input.yml
create mode 100644 echo/3/kustomization.yml
create mode 100644 echo/3/streamflow.yml
create mode 100644 echo/3/work.cwl
create mode 100644 echo/4/env
create mode 100644 echo/4/input.yml
create mode 100644 echo/4/kustomization.yml
create mode 100644 echo/4/streamflow.yml
create mode 100644 echo/4/work.cwl
create mode 100644 psd/1/env
create mode 100644 psd/1/input.yml
create mode 100644 psd/1/kustomization.yml
create mode 100644 psd/1/psd_calc.cwl
create mode 100644 psd/1/psd_vis.cwl
create mode 100644 psd/1/streamflow.yml
create mode 100644 psd/1/work.cwl
create mode 100644 psd/README
diff --git a/README.md b/README.md
index e69de29..ac0ebc9 100644
--- a/README.md
+++ b/README.md
@@ -0,0 +1,26 @@
+# README
+
+## 1. Prerequisites
+
+### 1.1. Prepare a service account for running Streamflow
+
+Create a serviceaccount for running Streamflow:
+
+ kubectl create serviceaccount streamflow
+
+Make our serviceaccount capable of installing Helm charts (see also: https://helm.sh/docs/topics/rbac/). For example, assuming that `streamflow` servica account lives inside the `default` namespace:
+
+ kubectl create rolebinding streamflow-edit --clusterrole edit --serviceaccount default:streamflow
+
+### 1.2. Prepare volume for job data
+
+Create a PVC named `job-data` and make sure it can bind to a PV (lazily or eagerly). This PVC will be used to store input/output data for workflows.
+
+For workflows that expect input as files, prepare those files under `JOB_ID/input` subpath of the volume. All output files will be under `JOB_ID/output` subpath.
+
+## 2. Run a job
+
+Prepare the configuration files under a kustomization directory `echo/1`. Then, apply:
+
+ kubectl apply -k echo/1
+
diff --git a/base/job.yml b/base/job.yml
new file mode 100644
index 0000000..f27e15b
--- /dev/null
+++ b/base/job.yml
@@ -0,0 +1,118 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: streamflow
+spec:
+ backoffLimit: 2
+ template:
+ metadata:
+ {}
+ spec:
+ securityContext:
+ runAsUser: 1001200000
+ runAsGroup: 1001200000
+ fsGroup: 1001200000
+ serviceAccountName: streamflow
+ volumes:
+ - name: config
+ configMap:
+ name: streamflow-config
+ - name: helm-config
+ configMap:
+ name: streamflow-helm-config
+ - name: cwl-defs
+ configMap:
+ name: cwl-defs
+ optional: true
+ - name: temp
+ emptyDir: {}
+ - name: cache
+ emptyDir: {}
+ - name: data
+ persistentVolumeClaim:
+ claimName: job-data
+ initContainers:
+ - name: update-helm-repos
+ image: docker-registry.ebrains.eu/tc/streamflow:0.2-dev
+ command:
+ - helm
+ - repo
+ - update
+ volumeMounts:
+ - name: helm-config
+ mountPath: /.config/helm/repositories.yaml
+ subPath: repositories.yaml
+ readOnly: true
+ - name: cache
+ mountPath: /.cache
+ - name: generate-kubeconfig
+ image: docker-registry.ebrains.eu/tc/busybox:1.32
+ # NOTE: generate an empty kubeconfig file (otherwise, helm will complain)
+ command:
+ - sh
+ - -c
+ - >-
+ touch /.streamflow/kubeconfig && chmod 0600 /.streamflow/kubeconfig
+ volumeMounts:
+ - name: temp
+ mountPath: /.streamflow
+ containers:
+ - name: streamflow
+ image: docker-registry.ebrains.eu/tc/streamflow:0.2-dev
+ workingDir: /streamflow/project/
+ command:
+ - streamflow
+ - run
+ - --debug
+ - --outdir
+ - /streamflow/results
+ - streamflow.yml
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: JOB_ID
+ valueFrom:
+ configMapKeyRef:
+ name: streamflow-config
+ key: JOB_ID
+ - name: KUBECONFIG
+ value: /.streamflow/kubeconfig
+ volumeMounts:
+ - name: config
+ mountPath: /streamflow/project/work.cwl
+ subPath: work.cwl
+ readOnly: true
+ - name: config
+ mountPath: /streamflow/project/input.yml
+ subPath: input.yml
+ readOnly: true
+ - name: config
+ mountPath: /streamflow/project/streamflow.yml
+ subPath: streamflow.yml
+ readOnly: true
+ - name: helm-config
+ mountPath: /.config/helm/repositories.yaml
+ subPath: repositories.yaml
+ readOnly: true
+ - name: cwl-defs
+ mountPath: /streamflow/project/cwl
+ readOnly: true
+ - name: temp
+ mountPath: /.streamflow
+ - name: cache
+ mountPath: /.cache
+ - name: data
+ mountPath: /streamflow/results
+ subPathExpr: $(JOB_ID)/output
+ - name: data
+ mountPath: /streamflow/project/input
+ subPathExpr: $(JOB_ID)/input
+ readOnly: true
+ resources:
+ limits:
+ memory: 1Gi
+ requests:
+ memory: 256Mi
+ restartPolicy: Never
diff --git a/base/kustomization.yml b/base/kustomization.yml
new file mode 100644
index 0000000..cfc3acf
--- /dev/null
+++ b/base/kustomization.yml
@@ -0,0 +1,10 @@
+resources:
+- job.yml
+
+configMapGenerator:
+- name: streamflow-helm-config
+ files:
+ - repositories.yaml
+
+#generatorOptions:
+# disableNameSuffixHash: true
diff --git a/base/repositories.yaml b/base/repositories.yaml
new file mode 100644
index 0000000..87104e2
--- /dev/null
+++ b/base/repositories.yaml
@@ -0,0 +1,4 @@
+apiVersion: ""
+repositories:
+- name: opertusmundi
+ url: https://opertusmundi.github.io/helm-charts/
diff --git a/compile-java/1/env b/compile-java/1/env
new file mode 100644
index 0000000..bab63af
--- /dev/null
+++ b/compile-java/1/env
@@ -0,0 +1 @@
+JOB_ID=compile-java-1
diff --git a/compile-java/1/input.yml b/compile-java/1/input.yml
new file mode 100644
index 0000000..d43d0e7
--- /dev/null
+++ b/compile-java/1/input.yml
@@ -0,0 +1,4 @@
+tarball: # type 'File'
+ class: File
+ path: input/hello.tgz
+name_of_file_to_extract: Hello.java # type 'string'
diff --git a/compile-java/1/kustomization.yml b/compile-java/1/kustomization.yml
new file mode 100644
index 0000000..acc3053
--- /dev/null
+++ b/compile-java/1/kustomization.yml
@@ -0,0 +1,13 @@
+resources:
+- ../../base/
+
+nameSuffix: "-compile-java-1"
+
+configMapGenerator:
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/compile-java/1/streamflow.yml b/compile-java/1/streamflow.yml
new file mode 100644
index 0000000..5ef010b
--- /dev/null
+++ b/compile-java/1/streamflow.yml
@@ -0,0 +1,33 @@
+version: v1.0
+workflows:
+ extract-and-compile-java:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /untar
+ target:
+ deployment: busybox
+ service: busybox
+ - step: /compile
+ target:
+ deployment: openjdk
+ service: debian
+deployments:
+ busybox:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+ openjdk:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/debian
+ chartVersion: '0.0.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/openjdk,image.tag=11-jdk,serviceAccount.create=false
diff --git a/compile-java/1/work.cwl b/compile-java/1/work.cwl
new file mode 100644
index 0000000..191424d
--- /dev/null
+++ b/compile-java/1/work.cwl
@@ -0,0 +1,58 @@
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+ tarball: File
+ name_of_file_to_extract: string
+
+outputs:
+ compiled_class:
+ type: File
+ outputSource: compile/classfile
+
+steps:
+ untar:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - tar
+ - xvf
+ inputs:
+ tarfile:
+ type: File
+ inputBinding:
+ position: 1
+ name_of_file_to_extract:
+ type: string
+ inputBinding:
+ position: 2
+ outputs:
+ extracted_file:
+ type: File
+ outputBinding:
+ glob: "*.java"
+ in:
+ tarfile: tarball
+ name_of_file_to_extract: name_of_file_to_extract
+ out: [extracted_file]
+
+ compile:
+ run:
+ class: CommandLineTool
+ baseCommand: javac
+ arguments:
+ - -d
+ - "$(runtime.outdir)"
+ inputs:
+ src:
+ type: File
+ inputBinding:
+ position: 1
+ outputs:
+ classfile:
+ type: File
+ outputBinding:
+ glob: "*.class"
+ in:
+ src: untar/extracted_file
+ out: [classfile]
diff --git a/compile-java/2/env b/compile-java/2/env
new file mode 100644
index 0000000..ffd8077
--- /dev/null
+++ b/compile-java/2/env
@@ -0,0 +1 @@
+JOB_ID=compile-java-2
diff --git a/compile-java/2/input.yml b/compile-java/2/input.yml
new file mode 100644
index 0000000..64060ae
--- /dev/null
+++ b/compile-java/2/input.yml
@@ -0,0 +1,6 @@
+tarball: # type 'File'
+ class: File
+ path: input/hello.tgz
+names_of_files_to_extract: # array of type 'string'
+ - Hello.java
+ - HelloUrlConnection.java
diff --git a/compile-java/2/kustomization.yml b/compile-java/2/kustomization.yml
new file mode 100644
index 0000000..e816221
--- /dev/null
+++ b/compile-java/2/kustomization.yml
@@ -0,0 +1,13 @@
+resources:
+- ../../base/
+
+nameSuffix: "-compile-java-2"
+
+configMapGenerator:
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/compile-java/2/streamflow.yml b/compile-java/2/streamflow.yml
new file mode 100644
index 0000000..00ec19a
--- /dev/null
+++ b/compile-java/2/streamflow.yml
@@ -0,0 +1,33 @@
+version: v1.0
+workflows:
+ extract-and-compile-java:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /untar
+ target:
+ deployment: openjdk
+ service: debian
+ - step: /compile
+ target:
+ deployment: openjdk
+ service: debian
+deployments:
+ busybox:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+ openjdk:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/debian
+ chartVersion: '0.0.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/openjdk,image.tag=11-jdk,serviceAccount.create=false
diff --git a/compile-java/2/work.cwl b/compile-java/2/work.cwl
new file mode 100644
index 0000000..5d4367b
--- /dev/null
+++ b/compile-java/2/work.cwl
@@ -0,0 +1,61 @@
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+ tarball: File
+ names_of_files_to_extract: string[]
+
+outputs:
+ classfiles:
+ type: File[]
+ outputSource: compile/classfiles
+
+steps:
+ untar:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - tar
+ - x
+ - -zvo
+ stdout: output.txt
+ inputs:
+ tarfile:
+ type: File
+ inputBinding:
+ position: 1
+ prefix: -f
+ names_of_files_to_extract:
+ type: string[]
+ inputBinding:
+ position: 2
+ outputs:
+ extracted_files:
+ type: File[]
+ outputBinding:
+ glob: "*.java"
+ in:
+ tarfile: tarball
+ names_of_files_to_extract: names_of_files_to_extract
+ out: [extracted_files]
+
+ compile:
+ run:
+ class: CommandLineTool
+ baseCommand: javac
+ arguments:
+ - -d
+ - "$(runtime.outdir)"
+ inputs:
+ src:
+ type: File[]
+ inputBinding:
+ position: 1
+ outputs:
+ classfiles:
+ type: File[]
+ outputBinding:
+ glob: "*.class"
+ in:
+ src: untar/extracted_files
+ out: [classfiles]
diff --git a/echo/1/env b/echo/1/env
new file mode 100644
index 0000000..cbcbfca
--- /dev/null
+++ b/echo/1/env
@@ -0,0 +1 @@
+JOB_ID=echo-1
diff --git a/echo/1/input.yml b/echo/1/input.yml
new file mode 100644
index 0000000..146103b
--- /dev/null
+++ b/echo/1/input.yml
@@ -0,0 +1 @@
+message1: Hello CWL workflow!
diff --git a/echo/1/kustomization.yml b/echo/1/kustomization.yml
new file mode 100644
index 0000000..814642d
--- /dev/null
+++ b/echo/1/kustomization.yml
@@ -0,0 +1,13 @@
+resources:
+- ../../base/
+
+nameSuffix: "-echo-1"
+
+configMapGenerator:
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/echo/1/streamflow.yml b/echo/1/streamflow.yml
new file mode 100644
index 0000000..38cf4e4
--- /dev/null
+++ b/echo/1/streamflow.yml
@@ -0,0 +1,25 @@
+version: v1.0
+workflows:
+ echo-and-uppercase:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /echo
+ target:
+ deployment: echo
+ service: busybox
+deployments:
+ echo:
+ type: helm
+ config:
+ inCluster: true
+ #chart: https://opertusmundi.github.io/helm-charts/busybox-0.1.0.tgz
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox
+ #releaseName: echo1
+ #namespace: workflows-1
+ timeout: '30s'
diff --git a/echo/1/work.cwl b/echo/1/work.cwl
new file mode 100644
index 0000000..3124814
--- /dev/null
+++ b/echo/1/work.cwl
@@ -0,0 +1,50 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+ InlineJavascriptRequirement: {}
+
+inputs:
+ message1: string
+
+outputs:
+ out:
+ type: string
+ outputSource: uppercase/uppercase_message
+
+steps:
+ echo:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - echo
+ - -n
+ stdout: output.txt
+ inputs:
+ message:
+ type: string
+ inputBinding: {}
+ outputs:
+ out1:
+ type: string
+ outputBinding:
+ glob: output.txt
+ loadContents: true
+ outputEval: $(self[0].contents)
+ in:
+ message: message1
+ out: [out1]
+ uppercase:
+ run:
+ class: ExpressionTool
+ requirements:
+ InlineJavascriptRequirement: {}
+ inputs:
+ message: string
+ outputs:
+ uppercase_message: string
+ expression: |
+ ${ return {"uppercase_message": inputs.message.toUpperCase()}; }
+ in:
+ message: echo/out1
+ out: [uppercase_message]
diff --git a/echo/2/env b/echo/2/env
new file mode 100644
index 0000000..d0ddc61
--- /dev/null
+++ b/echo/2/env
@@ -0,0 +1 @@
+JOB_ID=echo-2
diff --git a/echo/2/input.yml b/echo/2/input.yml
new file mode 100644
index 0000000..a896374
--- /dev/null
+++ b/echo/2/input.yml
@@ -0,0 +1 @@
+message1: Hello CWL workflow!!
diff --git a/echo/2/kustomization.yml b/echo/2/kustomization.yml
new file mode 100644
index 0000000..5d15cfb
--- /dev/null
+++ b/echo/2/kustomization.yml
@@ -0,0 +1,13 @@
+resources:
+- ../../base/
+
+nameSuffix: "-echo-2"
+
+configMapGenerator:
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/echo/2/streamflow.yml b/echo/2/streamflow.yml
new file mode 100644
index 0000000..4508afb
--- /dev/null
+++ b/echo/2/streamflow.yml
@@ -0,0 +1,37 @@
+version: v1.0
+workflows:
+ echo-and-uppercase:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /echo
+ target:
+ deployment: echo1
+ service: busybox
+ - step: /uppercase
+ target:
+ deployment: echo2
+ service: busybox
+# NOTE: here we map different steps to different deployments, so Streamflow
+# copies output from source container to target container (run with --debug)
+deployments:
+ echo1:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+ #releaseName: echo1
+ echo2:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+ #releaseName: echo2
diff --git a/echo/2/work.cwl b/echo/2/work.cwl
new file mode 100644
index 0000000..7de5bc5
--- /dev/null
+++ b/echo/2/work.cwl
@@ -0,0 +1,51 @@
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+ message1: string
+
+outputs:
+ out:
+ type: File
+ outputSource: uppercase/uppercase_message
+
+steps:
+ echo:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - echo
+ - -n
+ stdout: output.txt
+ inputs:
+ message:
+ type: string
+ inputBinding: {}
+ outputs:
+ out1:
+ type: File
+ outputBinding:
+ glob: output.txt
+ in:
+ message: message1
+ out: [out1]
+ uppercase:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - awk
+ - >-
+ {print toupper($0);}
+ stdout: output.txt
+ inputs:
+ message:
+ type: File
+ inputBinding: {}
+ outputs:
+ uppercase_message:
+ type: File
+ outputBinding:
+ glob: output.txt
+ in:
+ message: echo/out1
+ out: [uppercase_message]
diff --git a/echo/2a/env b/echo/2a/env
new file mode 100644
index 0000000..1dfed62
--- /dev/null
+++ b/echo/2a/env
@@ -0,0 +1 @@
+JOB_ID=echo-2a
diff --git a/echo/2a/kustomization.yml b/echo/2a/kustomization.yml
new file mode 100644
index 0000000..7c15c46
--- /dev/null
+++ b/echo/2a/kustomization.yml
@@ -0,0 +1,12 @@
+resources:
+- ../2
+
+nameSuffix: "a"
+
+configMapGenerator:
+- name: streamflow-config
+ behavior: merge
+ files:
+ - streamflow.yml
+ envs:
+ - env
diff --git a/echo/2a/streamflow.yml b/echo/2a/streamflow.yml
new file mode 100644
index 0000000..20b4310
--- /dev/null
+++ b/echo/2a/streamflow.yml
@@ -0,0 +1,28 @@
+version: v1.0
+workflows:
+ echo-and-uppercase:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /echo
+ target:
+ deployment: echo1
+ service: busybox
+ - step: /uppercase
+ target:
+ deployment: echo1
+ service: busybox
+# NOTE: here we map different steps to same deployment, so Streamflow
+# doesnt need to copy output from source container to target, just links it (run with --debug)
+deployments:
+ echo1:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+ #releaseName: echo1
diff --git a/echo/3/env b/echo/3/env
new file mode 100644
index 0000000..1d8da4d
--- /dev/null
+++ b/echo/3/env
@@ -0,0 +1 @@
+JOB_ID=echo-3
diff --git a/echo/3/input.yml b/echo/3/input.yml
new file mode 100644
index 0000000..5f72c7b
--- /dev/null
+++ b/echo/3/input.yml
@@ -0,0 +1 @@
+message1: Hello CWL workflow!!!
diff --git a/echo/3/kustomization.yml b/echo/3/kustomization.yml
new file mode 100644
index 0000000..2ef0080
--- /dev/null
+++ b/echo/3/kustomization.yml
@@ -0,0 +1,13 @@
+resources:
+- ../../base/
+
+nameSuffix: "-echo-3"
+
+configMapGenerator:
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/echo/3/streamflow.yml b/echo/3/streamflow.yml
new file mode 100644
index 0000000..25c151e
--- /dev/null
+++ b/echo/3/streamflow.yml
@@ -0,0 +1,25 @@
+version: v1.0
+workflows:
+ echo-and-uppercase:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /echo
+ target:
+ deployment: echo1
+ service: busybox
+ - step: /uppercase
+ target:
+ deployment: echo1
+ service: busybox
+deployments:
+ echo1:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
diff --git a/echo/3/work.cwl b/echo/3/work.cwl
new file mode 100644
index 0000000..a2e18df
--- /dev/null
+++ b/echo/3/work.cwl
@@ -0,0 +1,56 @@
+# An example of using STDIN to feed a command
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+ message1: string
+
+outputs:
+ out:
+ type: File
+ outputSource: uppercase/uppercase_message
+
+steps:
+ echo:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - echo
+ - -n
+ stdout: message.txt
+ inputs:
+ message:
+ type: string
+ inputBinding: {}
+ outputs:
+ out1:
+ type: File
+ outputBinding:
+ glob: message.txt
+ in:
+ message: message1
+ out: [out1]
+ uppercase:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - tr
+ - "[:lower:]"
+ - "[:upper:]"
+ stdout: output.txt
+ # NOTE: pipe this file into stdin ...
+ # https://github.com/common-workflow-language/cwltool/issues/1004
+ stdin: $(inputs.message.path)
+ inputs:
+ message:
+ type: File
+ # Do not bind as input! (because it will pass an additional parameter to our command)
+ #inputBinding: {}
+ outputs:
+ uppercase_message:
+ type: File
+ outputBinding:
+ glob: output.txt
+ in:
+ message: echo/out1
+ out: [uppercase_message]
diff --git a/echo/4/env b/echo/4/env
new file mode 100644
index 0000000..362efba
--- /dev/null
+++ b/echo/4/env
@@ -0,0 +1 @@
+JOB_ID=echo-4
diff --git a/echo/4/input.yml b/echo/4/input.yml
new file mode 100644
index 0000000..0eb57d9
--- /dev/null
+++ b/echo/4/input.yml
@@ -0,0 +1,3 @@
+message1: # type 'File'
+ class: File
+ path: input/hello.txt
diff --git a/echo/4/kustomization.yml b/echo/4/kustomization.yml
new file mode 100644
index 0000000..d5825e9
--- /dev/null
+++ b/echo/4/kustomization.yml
@@ -0,0 +1,13 @@
+resources:
+- ../../base/
+
+nameSuffix: "-echo-4"
+
+configMapGenerator:
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/echo/4/streamflow.yml b/echo/4/streamflow.yml
new file mode 100644
index 0000000..25c151e
--- /dev/null
+++ b/echo/4/streamflow.yml
@@ -0,0 +1,25 @@
+version: v1.0
+workflows:
+ echo-and-uppercase:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /echo
+ target:
+ deployment: echo1
+ service: busybox
+ - step: /uppercase
+ target:
+ deployment: echo1
+ service: busybox
+deployments:
+ echo1:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/busybox
+ chartVersion: '0.1.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
diff --git a/echo/4/work.cwl b/echo/4/work.cwl
new file mode 100644
index 0000000..efa0bdf
--- /dev/null
+++ b/echo/4/work.cwl
@@ -0,0 +1,52 @@
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+ message1:
+ type: File
+
+outputs:
+ out:
+ type: File
+ outputSource: uppercase/uppercase_message
+
+steps:
+ echo:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - cat
+ stdout: output.txt
+ inputs:
+ message:
+ type: File
+ inputBinding: {}
+ outputs:
+ out1:
+ type: File
+ outputBinding:
+ glob: output.txt
+ in:
+ message: message1
+ out: [out1]
+ uppercase:
+ run:
+ class: CommandLineTool
+ baseCommand:
+ - awk
+ - >-
+ {print toupper($0);}
+ stdout: output.txt
+ inputs:
+ message:
+ type: File
+ inputBinding: {}
+ outputs:
+ uppercase_message:
+ type: File
+ outputBinding:
+ glob: output.txt
+ in:
+ message: echo/out1
+ out: [uppercase_message]
+
diff --git a/psd/1/env b/psd/1/env
new file mode 100644
index 0000000..e43c2c7
--- /dev/null
+++ b/psd/1/env
@@ -0,0 +1 @@
+JOB_ID=psd-1
diff --git a/psd/1/input.yml b/psd/1/input.yml
new file mode 100644
index 0000000..5f9ce0f
--- /dev/null
+++ b/psd/1/input.yml
@@ -0,0 +1,7 @@
+input_file:
+ class: File
+ #location: 'https://data-proxy.ebrains.eu/api/v1/permalinks/06b0b2c8-31cb-4108-b6a7-0d275e118339'
+ path: 'input/1.mat'
+channels: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+psd_output_file_name: 'psd.json'
+output_file_name: 'output.png'
diff --git a/psd/1/kustomization.yml b/psd/1/kustomization.yml
new file mode 100644
index 0000000..3f9c11d
--- /dev/null
+++ b/psd/1/kustomization.yml
@@ -0,0 +1,17 @@
+resources:
+- ../../base/
+
+nameSuffix: "-psd-1"
+
+configMapGenerator:
+- name: cwl-defs
+ files:
+ - psd_calc.cwl
+ - psd_vis.cwl
+- name: streamflow-config
+ files:
+ - work.cwl
+ - input.yml
+ - streamflow.yml
+ envs:
+ - env
diff --git a/psd/1/psd_calc.cwl b/psd/1/psd_calc.cwl
new file mode 100644
index 0000000..226f3d5
--- /dev/null
+++ b/psd/1/psd_calc.cwl
@@ -0,0 +1,26 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: psd_calc.py
+hints:
+ DockerRequirement:
+ dockerPull: docker-registry.ebrains.eu/tc/cwl-tools/psd_calc:latest
+inputs:
+ input_file:
+ type: File
+ inputBinding:
+ position: 1
+ output_file_name:
+ type: string
+ inputBinding:
+ prefix: --output_file
+ position: 2
+ channels:
+ type: int[]
+ inputBinding:
+ prefix: --channels
+ position: 3
+outputs:
+ output_file:
+ type: File
+ outputBinding:
+ glob: $(inputs.output_file_name)
diff --git a/psd/1/psd_vis.cwl b/psd/1/psd_vis.cwl
new file mode 100644
index 0000000..42641c3
--- /dev/null
+++ b/psd/1/psd_vis.cwl
@@ -0,0 +1,26 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: psd_vis.py
+hints:
+ DockerRequirement:
+ dockerPull: docker-registry.ebrains.eu/tc/cwl-tools/psd_vis:latest
+inputs:
+ input_file:
+ type: File
+ inputBinding:
+ position: 1
+ output_file_name:
+ type: string
+ inputBinding:
+ prefix: --output_file
+ position: 2
+ channels:
+ type: int[]
+ inputBinding:
+ prefix: --channels
+ position: 3
+outputs:
+ plot:
+ type: File
+ outputBinding:
+ glob: $(inputs.output_file_name)
diff --git a/psd/1/streamflow.yml b/psd/1/streamflow.yml
new file mode 100644
index 0000000..6dc14dd
--- /dev/null
+++ b/psd/1/streamflow.yml
@@ -0,0 +1,33 @@
+version: v1.0
+workflows:
+ calculate-and-visualiza:
+ type: cwl
+ config:
+ file: work.cwl
+ settings: input.yml
+ bindings:
+ - step: /psd_calculation
+ target:
+ deployment: calculation
+ service: debian
+ - step: /visualization
+ target:
+ deployment: visualization
+ service: debian
+deployments:
+ calculation:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/debian
+ chartVersion: '0.0.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/cwl-tools/psd_calc,image.tag=latest,serviceAccount.create=false
+ visualization:
+ type: helm
+ config:
+ inCluster: true
+ chart: opertusmundi/debian
+ chartVersion: '0.0.1'
+ stringValues: >-
+ image.repository=docker-registry.ebrains.eu/tc/cwl-tools/psd_vis,image.tag=latest,serviceAccount.create=false
diff --git a/psd/1/work.cwl b/psd/1/work.cwl
new file mode 100644
index 0000000..4f0c459
--- /dev/null
+++ b/psd/1/work.cwl
@@ -0,0 +1,30 @@
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+ input_file: File
+ channels: int[]
+ psd_output_file_name: string
+ output_file_name: string
+
+outputs:
+ final_output:
+ type: File
+ outputSource: visualization/plot
+
+steps:
+ psd_calculation:
+ run: cwl/psd_calc.cwl
+ in:
+ input_file: input_file
+ output_file_name: psd_output_file_name
+ channels: channels
+ out: [output_file]
+
+ visualization:
+ run: cwl/psd_vis.cwl
+ in:
+ input_file: psd_calculation/output_file
+ output_file_name: output_file_name
+ channels: channels
+ out: [plot]
diff --git a/psd/README b/psd/README
new file mode 100644
index 0000000..3daa589
--- /dev/null
+++ b/psd/README
@@ -0,0 +1 @@
+https://gitlab.ebrains.eu/technical-coordination/project-internal/workflows/cwl-workflows/-/tree/main/Workflows/PSD_workflow_1
--
GitLab