diff --git a/README.md b/README.md index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ac0ebc910a4f21c361a1ad252dfd899f52214676 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,26 @@ +# README + +## 1. Prerequisites + +### 1.1. Prepare a service account for running Streamflow + +Create a serviceaccount for running Streamflow: + + kubectl create serviceaccount streamflow + +Make our serviceaccount capable of installing Helm charts (see also: https://helm.sh/docs/topics/rbac/). For example, assuming that `streamflow` servica account lives inside the `default` namespace: + + kubectl create rolebinding streamflow-edit --clusterrole edit --serviceaccount default:streamflow + +### 1.2. Prepare volume for job data + +Create a PVC named `job-data` and make sure it can bind to a PV (lazily or eagerly). This PVC will be used to store input/output data for workflows. + +For workflows that expect input as files, prepare those files under `JOB_ID/input` subpath of the volume. All output files will be under `JOB_ID/output` subpath. + +## 2. Run a job + +Prepare the configuration files under a kustomization directory `echo/1`. Then, apply: + + kubectl apply -k echo/1 + diff --git a/base/job.yml b/base/job.yml new file mode 100644 index 0000000000000000000000000000000000000000..f27e15bd6f23ce2569308e77b2749148aa451541 --- /dev/null +++ b/base/job.yml @@ -0,0 +1,118 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: streamflow +spec: + backoffLimit: 2 + template: + metadata: + {} + spec: + securityContext: + runAsUser: 1001200000 + runAsGroup: 1001200000 + fsGroup: 1001200000 + serviceAccountName: streamflow + volumes: + - name: config + configMap: + name: streamflow-config + - name: helm-config + configMap: + name: streamflow-helm-config + - name: cwl-defs + configMap: + name: cwl-defs + optional: true + - name: temp + emptyDir: {} + - name: cache + emptyDir: {} + - name: data + persistentVolumeClaim: + claimName: job-data + initContainers: + - name: update-helm-repos + image: docker-registry.ebrains.eu/tc/streamflow:0.2-dev + command: + - helm + - repo + - update + volumeMounts: + - name: helm-config + mountPath: /.config/helm/repositories.yaml + subPath: repositories.yaml + readOnly: true + - name: cache + mountPath: /.cache + - name: generate-kubeconfig + image: docker-registry.ebrains.eu/tc/busybox:1.32 + # NOTE: generate an empty kubeconfig file (otherwise, helm will complain) + command: + - sh + - -c + - >- + touch /.streamflow/kubeconfig && chmod 0600 /.streamflow/kubeconfig + volumeMounts: + - name: temp + mountPath: /.streamflow + containers: + - name: streamflow + image: docker-registry.ebrains.eu/tc/streamflow:0.2-dev + workingDir: /streamflow/project/ + command: + - streamflow + - run + - --debug + - --outdir + - /streamflow/results + - streamflow.yml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: JOB_ID + valueFrom: + configMapKeyRef: + name: streamflow-config + key: JOB_ID + - name: KUBECONFIG + value: /.streamflow/kubeconfig + volumeMounts: + - name: config + mountPath: /streamflow/project/work.cwl + subPath: work.cwl + readOnly: true + - name: config + mountPath: /streamflow/project/input.yml + subPath: input.yml + readOnly: true + - name: config + mountPath: /streamflow/project/streamflow.yml + subPath: streamflow.yml + readOnly: true + - name: helm-config + mountPath: /.config/helm/repositories.yaml + subPath: repositories.yaml + readOnly: true + - name: cwl-defs + mountPath: /streamflow/project/cwl + readOnly: true + - name: temp + mountPath: /.streamflow + - name: cache + mountPath: /.cache + - name: data + mountPath: /streamflow/results + subPathExpr: $(JOB_ID)/output + - name: data + mountPath: /streamflow/project/input + subPathExpr: $(JOB_ID)/input + readOnly: true + resources: + limits: + memory: 1Gi + requests: + memory: 256Mi + restartPolicy: Never diff --git a/base/kustomization.yml b/base/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..cfc3acf7ee3544b2da7f229c885ad22054f5d5bd --- /dev/null +++ b/base/kustomization.yml @@ -0,0 +1,10 @@ +resources: +- job.yml + +configMapGenerator: +- name: streamflow-helm-config + files: + - repositories.yaml + +#generatorOptions: +# disableNameSuffixHash: true diff --git a/base/repositories.yaml b/base/repositories.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87104e2cc099bb8b160fa577363b0dcc40f4112e --- /dev/null +++ b/base/repositories.yaml @@ -0,0 +1,4 @@ +apiVersion: "" +repositories: +- name: opertusmundi + url: https://opertusmundi.github.io/helm-charts/ diff --git a/compile-java/1/env b/compile-java/1/env new file mode 100644 index 0000000000000000000000000000000000000000..bab63afd96116f4e9cde8489cd07f70f35b79c12 --- /dev/null +++ b/compile-java/1/env @@ -0,0 +1 @@ +JOB_ID=compile-java-1 diff --git a/compile-java/1/input.yml b/compile-java/1/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..d43d0e7fc61b5b01e9ece7fb83f4587e37b73657 --- /dev/null +++ b/compile-java/1/input.yml @@ -0,0 +1,4 @@ +tarball: # type 'File' + class: File + path: input/hello.tgz +name_of_file_to_extract: Hello.java # type 'string' diff --git a/compile-java/1/kustomization.yml b/compile-java/1/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..acc3053f104df638f1a0a3e0b17b1831e5b898b7 --- /dev/null +++ b/compile-java/1/kustomization.yml @@ -0,0 +1,13 @@ +resources: +- ../../base/ + +nameSuffix: "-compile-java-1" + +configMapGenerator: +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/compile-java/1/streamflow.yml b/compile-java/1/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..5ef010b77b504ee3e220ed8b0e3122dddaeedac4 --- /dev/null +++ b/compile-java/1/streamflow.yml @@ -0,0 +1,33 @@ +version: v1.0 +workflows: + extract-and-compile-java: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /untar + target: + deployment: busybox + service: busybox + - step: /compile + target: + deployment: openjdk + service: debian +deployments: + busybox: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false + openjdk: + type: helm + config: + inCluster: true + chart: opertusmundi/debian + chartVersion: '0.0.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/openjdk,image.tag=11-jdk,serviceAccount.create=false diff --git a/compile-java/1/work.cwl b/compile-java/1/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..191424d647f77f13be3c626664761804a7c9f9ac --- /dev/null +++ b/compile-java/1/work.cwl @@ -0,0 +1,58 @@ +cwlVersion: v1.2 +class: Workflow + +inputs: + tarball: File + name_of_file_to_extract: string + +outputs: + compiled_class: + type: File + outputSource: compile/classfile + +steps: + untar: + run: + class: CommandLineTool + baseCommand: + - tar + - xvf + inputs: + tarfile: + type: File + inputBinding: + position: 1 + name_of_file_to_extract: + type: string + inputBinding: + position: 2 + outputs: + extracted_file: + type: File + outputBinding: + glob: "*.java" + in: + tarfile: tarball + name_of_file_to_extract: name_of_file_to_extract + out: [extracted_file] + + compile: + run: + class: CommandLineTool + baseCommand: javac + arguments: + - -d + - "$(runtime.outdir)" + inputs: + src: + type: File + inputBinding: + position: 1 + outputs: + classfile: + type: File + outputBinding: + glob: "*.class" + in: + src: untar/extracted_file + out: [classfile] diff --git a/compile-java/2/env b/compile-java/2/env new file mode 100644 index 0000000000000000000000000000000000000000..ffd8077d5ca154a0c0f5dce23d6b74bc7cea6f85 --- /dev/null +++ b/compile-java/2/env @@ -0,0 +1 @@ +JOB_ID=compile-java-2 diff --git a/compile-java/2/input.yml b/compile-java/2/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..64060aebd96640e0577efb04d8b2baaedc215ecb --- /dev/null +++ b/compile-java/2/input.yml @@ -0,0 +1,6 @@ +tarball: # type 'File' + class: File + path: input/hello.tgz +names_of_files_to_extract: # array of type 'string' + - Hello.java + - HelloUrlConnection.java diff --git a/compile-java/2/kustomization.yml b/compile-java/2/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..e816221ad9e47fdf9b71d35c8dae358ee536c624 --- /dev/null +++ b/compile-java/2/kustomization.yml @@ -0,0 +1,13 @@ +resources: +- ../../base/ + +nameSuffix: "-compile-java-2" + +configMapGenerator: +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/compile-java/2/streamflow.yml b/compile-java/2/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..00ec19a94b6f5a932daa8c684640a5c95d6e35e1 --- /dev/null +++ b/compile-java/2/streamflow.yml @@ -0,0 +1,33 @@ +version: v1.0 +workflows: + extract-and-compile-java: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /untar + target: + deployment: openjdk + service: debian + - step: /compile + target: + deployment: openjdk + service: debian +deployments: + busybox: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false + openjdk: + type: helm + config: + inCluster: true + chart: opertusmundi/debian + chartVersion: '0.0.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/openjdk,image.tag=11-jdk,serviceAccount.create=false diff --git a/compile-java/2/work.cwl b/compile-java/2/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..5d4367b394efda152253dc6a73dfcadd3de6cc54 --- /dev/null +++ b/compile-java/2/work.cwl @@ -0,0 +1,61 @@ +cwlVersion: v1.2 +class: Workflow + +inputs: + tarball: File + names_of_files_to_extract: string[] + +outputs: + classfiles: + type: File[] + outputSource: compile/classfiles + +steps: + untar: + run: + class: CommandLineTool + baseCommand: + - tar + - x + - -zvo + stdout: output.txt + inputs: + tarfile: + type: File + inputBinding: + position: 1 + prefix: -f + names_of_files_to_extract: + type: string[] + inputBinding: + position: 2 + outputs: + extracted_files: + type: File[] + outputBinding: + glob: "*.java" + in: + tarfile: tarball + names_of_files_to_extract: names_of_files_to_extract + out: [extracted_files] + + compile: + run: + class: CommandLineTool + baseCommand: javac + arguments: + - -d + - "$(runtime.outdir)" + inputs: + src: + type: File[] + inputBinding: + position: 1 + outputs: + classfiles: + type: File[] + outputBinding: + glob: "*.class" + in: + src: untar/extracted_files + out: [classfiles] diff --git a/echo/1/env b/echo/1/env new file mode 100644 index 0000000000000000000000000000000000000000..cbcbfcad36d70629396cf630fbcd512b62bcfb4d --- /dev/null +++ b/echo/1/env @@ -0,0 +1 @@ +JOB_ID=echo-1 diff --git a/echo/1/input.yml b/echo/1/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..146103bc68b494ccedc0868cb620ecf17cbe043a --- /dev/null +++ b/echo/1/input.yml @@ -0,0 +1 @@ +message1: Hello CWL workflow! diff --git a/echo/1/kustomization.yml b/echo/1/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..814642dac7984311cdc509df3591afcabe850601 --- /dev/null +++ b/echo/1/kustomization.yml @@ -0,0 +1,13 @@ +resources: +- ../../base/ + +nameSuffix: "-echo-1" + +configMapGenerator: +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/echo/1/streamflow.yml b/echo/1/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..38cf4e49f60e7ec4d75d31a409575432f624cf7a --- /dev/null +++ b/echo/1/streamflow.yml @@ -0,0 +1,25 @@ +version: v1.0 +workflows: + echo-and-uppercase: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /echo + target: + deployment: echo + service: busybox +deployments: + echo: + type: helm + config: + inCluster: true + #chart: https://opertusmundi.github.io/helm-charts/busybox-0.1.0.tgz + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox + #releaseName: echo1 + #namespace: workflows-1 + timeout: '30s' diff --git a/echo/1/work.cwl b/echo/1/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..3124814a124bd95d610d77800a9cd3fbbaaf46b2 --- /dev/null +++ b/echo/1/work.cwl @@ -0,0 +1,50 @@ +cwlVersion: v1.2 +class: Workflow + +requirements: + InlineJavascriptRequirement: {} + +inputs: + message1: string + +outputs: + out: + type: string + outputSource: uppercase/uppercase_message + +steps: + echo: + run: + class: CommandLineTool + baseCommand: + - echo + - -n + stdout: output.txt + inputs: + message: + type: string + inputBinding: {} + outputs: + out1: + type: string + outputBinding: + glob: output.txt + loadContents: true + outputEval: $(self[0].contents) + in: + message: message1 + out: [out1] + uppercase: + run: + class: ExpressionTool + requirements: + InlineJavascriptRequirement: {} + inputs: + message: string + outputs: + uppercase_message: string + expression: | + ${ return {"uppercase_message": inputs.message.toUpperCase()}; } + in: + message: echo/out1 + out: [uppercase_message] diff --git a/echo/2/env b/echo/2/env new file mode 100644 index 0000000000000000000000000000000000000000..d0ddc61888de03e27b5b113536b3586280a0e540 --- /dev/null +++ b/echo/2/env @@ -0,0 +1 @@ +JOB_ID=echo-2 diff --git a/echo/2/input.yml b/echo/2/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..a896374aa1dee6e1e1b856a4af6eae092d963e3e --- /dev/null +++ b/echo/2/input.yml @@ -0,0 +1 @@ +message1: Hello CWL workflow!! diff --git a/echo/2/kustomization.yml b/echo/2/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..5d15cfb1f07e1926706f417375f8a5c7a8cabd1a --- /dev/null +++ b/echo/2/kustomization.yml @@ -0,0 +1,13 @@ +resources: +- ../../base/ + +nameSuffix: "-echo-2" + +configMapGenerator: +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/echo/2/streamflow.yml b/echo/2/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..4508afb1cfd6a56ac8fdcc65e769aa99d7f1f791 --- /dev/null +++ b/echo/2/streamflow.yml @@ -0,0 +1,37 @@ +version: v1.0 +workflows: + echo-and-uppercase: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /echo + target: + deployment: echo1 + service: busybox + - step: /uppercase + target: + deployment: echo2 + service: busybox +# NOTE: here we map different steps to different deployments, so Streamflow +# copies output from source container to target container (run with --debug) +deployments: + echo1: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false + #releaseName: echo1 + echo2: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false + #releaseName: echo2 diff --git a/echo/2/work.cwl b/echo/2/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..7de5bc5085abc3e8d4ea783e1ab0b67221bb78ae --- /dev/null +++ b/echo/2/work.cwl @@ -0,0 +1,51 @@ +cwlVersion: v1.2 +class: Workflow + +inputs: + message1: string + +outputs: + out: + type: File + outputSource: uppercase/uppercase_message + +steps: + echo: + run: + class: CommandLineTool + baseCommand: + - echo + - -n + stdout: output.txt + inputs: + message: + type: string + inputBinding: {} + outputs: + out1: + type: File + outputBinding: + glob: output.txt + in: + message: message1 + out: [out1] + uppercase: + run: + class: CommandLineTool + baseCommand: + - awk + - >- + {print toupper($0);} + stdout: output.txt + inputs: + message: + type: File + inputBinding: {} + outputs: + uppercase_message: + type: File + outputBinding: + glob: output.txt + in: + message: echo/out1 + out: [uppercase_message] diff --git a/echo/2a/env b/echo/2a/env new file mode 100644 index 0000000000000000000000000000000000000000..1dfed62a17e5652cd7ab282bc481a8987e8d7f01 --- /dev/null +++ b/echo/2a/env @@ -0,0 +1 @@ +JOB_ID=echo-2a diff --git a/echo/2a/kustomization.yml b/echo/2a/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..7c15c4635a16bd6aa6dfd3562e99e493fcff5790 --- /dev/null +++ b/echo/2a/kustomization.yml @@ -0,0 +1,12 @@ +resources: +- ../2 + +nameSuffix: "a" + +configMapGenerator: +- name: streamflow-config + behavior: merge + files: + - streamflow.yml + envs: + - env diff --git a/echo/2a/streamflow.yml b/echo/2a/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..20b43105c4a6c1676ad8e702a9462590757b2cc6 --- /dev/null +++ b/echo/2a/streamflow.yml @@ -0,0 +1,28 @@ +version: v1.0 +workflows: + echo-and-uppercase: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /echo + target: + deployment: echo1 + service: busybox + - step: /uppercase + target: + deployment: echo1 + service: busybox +# NOTE: here we map different steps to same deployment, so Streamflow +# doesnt need to copy output from source container to target, just links it (run with --debug) +deployments: + echo1: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false + #releaseName: echo1 diff --git a/echo/3/env b/echo/3/env new file mode 100644 index 0000000000000000000000000000000000000000..1d8da4dc4869cedf23b6bde91b2f516a3eaf0c56 --- /dev/null +++ b/echo/3/env @@ -0,0 +1 @@ +JOB_ID=echo-3 diff --git a/echo/3/input.yml b/echo/3/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..5f72c7b7a5b32084863ea10dbf3f0e7bf5e8df7a --- /dev/null +++ b/echo/3/input.yml @@ -0,0 +1 @@ +message1: Hello CWL workflow!!! diff --git a/echo/3/kustomization.yml b/echo/3/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..2ef008045aace5780a2d54a0ffd44ee16250e6f6 --- /dev/null +++ b/echo/3/kustomization.yml @@ -0,0 +1,13 @@ +resources: +- ../../base/ + +nameSuffix: "-echo-3" + +configMapGenerator: +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/echo/3/streamflow.yml b/echo/3/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..25c151e11c45a3ec3342121b5213ef55a3a41ae5 --- /dev/null +++ b/echo/3/streamflow.yml @@ -0,0 +1,25 @@ +version: v1.0 +workflows: + echo-and-uppercase: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /echo + target: + deployment: echo1 + service: busybox + - step: /uppercase + target: + deployment: echo1 + service: busybox +deployments: + echo1: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false diff --git a/echo/3/work.cwl b/echo/3/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..a2e18dfd3dc88eb9135f88467ae99d439945806e --- /dev/null +++ b/echo/3/work.cwl @@ -0,0 +1,56 @@ +# An example of using STDIN to feed a command +cwlVersion: v1.2 +class: Workflow + +inputs: + message1: string + +outputs: + out: + type: File + outputSource: uppercase/uppercase_message + +steps: + echo: + run: + class: CommandLineTool + baseCommand: + - echo + - -n + stdout: message.txt + inputs: + message: + type: string + inputBinding: {} + outputs: + out1: + type: File + outputBinding: + glob: message.txt + in: + message: message1 + out: [out1] + uppercase: + run: + class: CommandLineTool + baseCommand: + - tr + - "[:lower:]" + - "[:upper:]" + stdout: output.txt + # NOTE: pipe this file into stdin ... + # https://github.com/common-workflow-language/cwltool/issues/1004 + stdin: $(inputs.message.path) + inputs: + message: + type: File + # Do not bind as input! (because it will pass an additional parameter to our command) + #inputBinding: {} + outputs: + uppercase_message: + type: File + outputBinding: + glob: output.txt + in: + message: echo/out1 + out: [uppercase_message] diff --git a/echo/4/env b/echo/4/env new file mode 100644 index 0000000000000000000000000000000000000000..362efba1365fa11d7eb8de5b899b174c0e0a2b9e --- /dev/null +++ b/echo/4/env @@ -0,0 +1 @@ +JOB_ID=echo-4 diff --git a/echo/4/input.yml b/echo/4/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..0eb57d9796b01f804354e8050cd6be1f210d98ad --- /dev/null +++ b/echo/4/input.yml @@ -0,0 +1,3 @@ +message1: # type 'File' + class: File + path: input/hello.txt diff --git a/echo/4/kustomization.yml b/echo/4/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..d5825e9079732afeda052914cc6e457b58a2936f --- /dev/null +++ b/echo/4/kustomization.yml @@ -0,0 +1,13 @@ +resources: +- ../../base/ + +nameSuffix: "-echo-4" + +configMapGenerator: +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/echo/4/streamflow.yml b/echo/4/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..25c151e11c45a3ec3342121b5213ef55a3a41ae5 --- /dev/null +++ b/echo/4/streamflow.yml @@ -0,0 +1,25 @@ +version: v1.0 +workflows: + echo-and-uppercase: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /echo + target: + deployment: echo1 + service: busybox + - step: /uppercase + target: + deployment: echo1 + service: busybox +deployments: + echo1: + type: helm + config: + inCluster: true + chart: opertusmundi/busybox + chartVersion: '0.1.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false diff --git a/echo/4/work.cwl b/echo/4/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..efa0bdfe630580e4141946b619af7f6db789ba91 --- /dev/null +++ b/echo/4/work.cwl @@ -0,0 +1,52 @@ +cwlVersion: v1.2 +class: Workflow + +inputs: + message1: + type: File + +outputs: + out: + type: File + outputSource: uppercase/uppercase_message + +steps: + echo: + run: + class: CommandLineTool + baseCommand: + - cat + stdout: output.txt + inputs: + message: + type: File + inputBinding: {} + outputs: + out1: + type: File + outputBinding: + glob: output.txt + in: + message: message1 + out: [out1] + uppercase: + run: + class: CommandLineTool + baseCommand: + - awk + - >- + {print toupper($0);} + stdout: output.txt + inputs: + message: + type: File + inputBinding: {} + outputs: + uppercase_message: + type: File + outputBinding: + glob: output.txt + in: + message: echo/out1 + out: [uppercase_message] + diff --git a/psd/1/env b/psd/1/env new file mode 100644 index 0000000000000000000000000000000000000000..e43c2c74f596920402889b2bf95e68ef160c07d2 --- /dev/null +++ b/psd/1/env @@ -0,0 +1 @@ +JOB_ID=psd-1 diff --git a/psd/1/input.yml b/psd/1/input.yml new file mode 100644 index 0000000000000000000000000000000000000000..5f9ce0fc5959cf73e50570fad5d594962535ddc2 --- /dev/null +++ b/psd/1/input.yml @@ -0,0 +1,7 @@ +input_file: + class: File + #location: 'https://data-proxy.ebrains.eu/api/v1/permalinks/06b0b2c8-31cb-4108-b6a7-0d275e118339' + path: 'input/1.mat' +channels: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +psd_output_file_name: 'psd.json' +output_file_name: 'output.png' diff --git a/psd/1/kustomization.yml b/psd/1/kustomization.yml new file mode 100644 index 0000000000000000000000000000000000000000..3f9c11dabc3915241e2bda4a4b5643ae05f916dc --- /dev/null +++ b/psd/1/kustomization.yml @@ -0,0 +1,17 @@ +resources: +- ../../base/ + +nameSuffix: "-psd-1" + +configMapGenerator: +- name: cwl-defs + files: + - psd_calc.cwl + - psd_vis.cwl +- name: streamflow-config + files: + - work.cwl + - input.yml + - streamflow.yml + envs: + - env diff --git a/psd/1/psd_calc.cwl b/psd/1/psd_calc.cwl new file mode 100644 index 0000000000000000000000000000000000000000..226f3d5b6eee4ac5c69a936f08139771947b4f8e --- /dev/null +++ b/psd/1/psd_calc.cwl @@ -0,0 +1,26 @@ +cwlVersion: v1.0 +class: CommandLineTool +baseCommand: psd_calc.py +hints: + DockerRequirement: + dockerPull: docker-registry.ebrains.eu/tc/cwl-tools/psd_calc:latest +inputs: + input_file: + type: File + inputBinding: + position: 1 + output_file_name: + type: string + inputBinding: + prefix: --output_file + position: 2 + channels: + type: int[] + inputBinding: + prefix: --channels + position: 3 +outputs: + output_file: + type: File + outputBinding: + glob: $(inputs.output_file_name) diff --git a/psd/1/psd_vis.cwl b/psd/1/psd_vis.cwl new file mode 100644 index 0000000000000000000000000000000000000000..42641c3e609ae8f81a350e1fb1ceef65ddfa65d3 --- /dev/null +++ b/psd/1/psd_vis.cwl @@ -0,0 +1,26 @@ +cwlVersion: v1.0 +class: CommandLineTool +baseCommand: psd_vis.py +hints: + DockerRequirement: + dockerPull: docker-registry.ebrains.eu/tc/cwl-tools/psd_vis:latest +inputs: + input_file: + type: File + inputBinding: + position: 1 + output_file_name: + type: string + inputBinding: + prefix: --output_file + position: 2 + channels: + type: int[] + inputBinding: + prefix: --channels + position: 3 +outputs: + plot: + type: File + outputBinding: + glob: $(inputs.output_file_name) diff --git a/psd/1/streamflow.yml b/psd/1/streamflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..6dc14dde70d10cf3ccdd667fdc80f681c7876765 --- /dev/null +++ b/psd/1/streamflow.yml @@ -0,0 +1,33 @@ +version: v1.0 +workflows: + calculate-and-visualiza: + type: cwl + config: + file: work.cwl + settings: input.yml + bindings: + - step: /psd_calculation + target: + deployment: calculation + service: debian + - step: /visualization + target: + deployment: visualization + service: debian +deployments: + calculation: + type: helm + config: + inCluster: true + chart: opertusmundi/debian + chartVersion: '0.0.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/cwl-tools/psd_calc,image.tag=latest,serviceAccount.create=false + visualization: + type: helm + config: + inCluster: true + chart: opertusmundi/debian + chartVersion: '0.0.1' + stringValues: >- + image.repository=docker-registry.ebrains.eu/tc/cwl-tools/psd_vis,image.tag=latest,serviceAccount.create=false diff --git a/psd/1/work.cwl b/psd/1/work.cwl new file mode 100644 index 0000000000000000000000000000000000000000..4f0c459797f3160e771da7bdf79b49f54f086cf8 --- /dev/null +++ b/psd/1/work.cwl @@ -0,0 +1,30 @@ +cwlVersion: v1.0 +class: Workflow + +inputs: + input_file: File + channels: int[] + psd_output_file_name: string + output_file_name: string + +outputs: + final_output: + type: File + outputSource: visualization/plot + +steps: + psd_calculation: + run: cwl/psd_calc.cwl + in: + input_file: input_file + output_file_name: psd_output_file_name + channels: channels + out: [output_file] + + visualization: + run: cwl/psd_vis.cwl + in: + input_file: psd_calculation/output_file + output_file_name: output_file_name + channels: channels + out: [plot] diff --git a/psd/README b/psd/README new file mode 100644 index 0000000000000000000000000000000000000000..3daa589f823c543f78969429e35cf4a2e162d179 --- /dev/null +++ b/psd/README @@ -0,0 +1 @@ +https://gitlab.ebrains.eu/technical-coordination/project-internal/workflows/cwl-workflows/-/tree/main/Workflows/PSD_workflow_1