Add some basic examples of workflows

README.md

+# README
+
+## 1. Prerequisites
+
+### 1.1. Prepare a service account for running Streamflow
+
+Create a serviceaccount for running Streamflow:
+
+    kubectl create serviceaccount streamflow
+
+Make our serviceaccount capable of installing Helm charts (see also: https://helm.sh/docs/topics/rbac/). For example, assuming that `streamflow` servica account lives inside the `default` namespace:
+
+    kubectl create rolebinding streamflow-edit --clusterrole edit --serviceaccount default:streamflow
+
+### 1.2. Prepare volume for job data
+
+Create a PVC named `job-data` and make sure it can bind to a PV (lazily or eagerly). This PVC will be used to store input/output data for workflows.
+
+For workflows that expect input as files, prepare those files under `JOB_ID/input` subpath of the volume. All output files will be under `JOB_ID/output` subpath.
+
+## 2. Run a job
+
+Prepare the configuration files under a kustomization directory `echo/1`. Then, apply:
+
+    kubectl apply -k echo/1
+

base/job.yml

+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: streamflow
+spec:
+  backoffLimit: 2
+  template:
+    metadata:
+      {}
+    spec:
+      securityContext:
+        runAsUser: 1001200000
+        runAsGroup: 1001200000 
+        fsGroup: 1001200000
+      serviceAccountName: streamflow
+      volumes:
+      - name: config
+        configMap:
+          name: streamflow-config
+      - name: helm-config
+        configMap:
+          name: streamflow-helm-config
+      - name: cwl-defs
+        configMap:
+          name: cwl-defs
+          optional: true
+      - name: temp
+        emptyDir: {}
+      - name: cache
+        emptyDir: {}
+      - name: data
+        persistentVolumeClaim:
+          claimName: job-data
+      initContainers:
+      - name: update-helm-repos
+        image: docker-registry.ebrains.eu/tc/streamflow:0.2-dev
+        command:
+        - helm
+        - repo
+        - update
+        volumeMounts:
+        - name: helm-config
+          mountPath: /.config/helm/repositories.yaml
+          subPath: repositories.yaml
+          readOnly: true
+        - name: cache
+          mountPath: /.cache
+      - name: generate-kubeconfig
+        image: docker-registry.ebrains.eu/tc/busybox:1.32
+        # NOTE: generate an empty kubeconfig file (otherwise, helm will complain)
+        command:
+        - sh
+        - -c
+        - >-
+          touch /.streamflow/kubeconfig && chmod 0600 /.streamflow/kubeconfig
+        volumeMounts:
+        - name: temp
+          mountPath: /.streamflow
+      containers:
+      - name: streamflow
+        image: docker-registry.ebrains.eu/tc/streamflow:0.2-dev
+        workingDir: /streamflow/project/
+        command:
+        - streamflow
+        - run
+        - --debug
+        - --outdir
+        - /streamflow/results
+        - streamflow.yml
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: JOB_ID
+          valueFrom:
+            configMapKeyRef:
+              name: streamflow-config
+              key: JOB_ID
+        - name: KUBECONFIG
+          value: /.streamflow/kubeconfig
+        volumeMounts:
+        - name: config
+          mountPath: /streamflow/project/work.cwl
+          subPath: work.cwl
+          readOnly: true
+        - name: config
+          mountPath: /streamflow/project/input.yml
+          subPath: input.yml
+          readOnly: true
+        - name: config
+          mountPath: /streamflow/project/streamflow.yml
+          subPath: streamflow.yml
+          readOnly: true
+        - name: helm-config
+          mountPath: /.config/helm/repositories.yaml
+          subPath: repositories.yaml
+          readOnly: true
+        - name: cwl-defs
+          mountPath: /streamflow/project/cwl
+          readOnly: true
+        - name: temp
+          mountPath: /.streamflow
+        - name: cache
+          mountPath: /.cache
+        - name: data
+          mountPath: /streamflow/results
+          subPathExpr: $(JOB_ID)/output
+        - name: data
+          mountPath: /streamflow/project/input
+          subPathExpr: $(JOB_ID)/input
+          readOnly: true
+        resources:
+          limits:
+            memory: 1Gi
+          requests:
+            memory: 256Mi
+      restartPolicy: Never

base/kustomization.yml

+resources:
+- job.yml
+
+configMapGenerator:
+- name: streamflow-helm-config
+  files:
+  - repositories.yaml
+
+#generatorOptions:
+#  disableNameSuffixHash: true

base/repositories.yaml

+apiVersion: ""
+repositories:
+- name: opertusmundi
+  url: https://opertusmundi.github.io/helm-charts/

compile-java/1/env

+JOB_ID=compile-java-1

compile-java/1/input.yml

+tarball:  # type 'File'
+    class: File
+    path: input/hello.tgz
+name_of_file_to_extract: Hello.java  # type 'string'

compile-java/1/kustomization.yml

+resources:
+- ../../base/
+
+nameSuffix: "-compile-java-1"
+
+configMapGenerator:
+- name: streamflow-config
+  files:
+  - work.cwl
+  - input.yml
+  - streamflow.yml
+  envs:
+  - env

compile-java/1/streamflow.yml

+version: v1.0
+workflows:
+  extract-and-compile-java:
+    type: cwl
+    config:
+      file: work.cwl
+      settings: input.yml
+    bindings:
+    - step: /untar
+      target:
+        deployment: busybox
+        service: busybox
+    - step: /compile
+      target:
+        deployment: openjdk
+        service: debian
+deployments:
+  busybox:
+    type: helm
+    config:
+      inCluster: true
+      chart: opertusmundi/busybox
+      chartVersion: '0.1.1'
+      stringValues: >-
+        image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+  openjdk:
+    type: helm
+    config:
+      inCluster: true
+      chart: opertusmundi/debian
+      chartVersion: '0.0.1'
+      stringValues: >-
+        image.repository=docker-registry.ebrains.eu/tc/openjdk,image.tag=11-jdk,serviceAccount.create=false

compile-java/1/work.cwl

+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+  tarball: File
+  name_of_file_to_extract: string
+
+outputs:
+  compiled_class:
+    type: File
+    outputSource: compile/classfile
+
+steps:
+  untar:
+    run:
+      class: CommandLineTool
+      baseCommand:
+      - tar
+      - xvf
+      inputs:
+        tarfile:
+          type: File
+          inputBinding:
+            position: 1
+        name_of_file_to_extract:
+          type: string
+          inputBinding:
+            position: 2
+      outputs:
+        extracted_file:
+          type: File
+          outputBinding:
+            glob: "*.java"
+    in:
+      tarfile: tarball
+      name_of_file_to_extract: name_of_file_to_extract
+    out: [extracted_file]
+
+  compile:
+    run:
+      class: CommandLineTool
+      baseCommand: javac
+      arguments:
+      - -d
+      - "$(runtime.outdir)"
+      inputs:
+        src:
+          type: File
+          inputBinding:
+            position: 1
+      outputs:
+        classfile:
+          type: File
+          outputBinding:
+            glob: "*.class"
+    in:
+      src: untar/extracted_file
+    out: [classfile]

compile-java/2/env

+JOB_ID=compile-java-2

compile-java/2/input.yml

+tarball:  # type 'File'
+    class: File
+    path: input/hello.tgz
+names_of_files_to_extract:  # array of type 'string'
+  - Hello.java
+  - HelloUrlConnection.java

compile-java/2/kustomization.yml

+resources:
+- ../../base/
+
+nameSuffix: "-compile-java-2"
+
+configMapGenerator:
+- name: streamflow-config
+  files:
+  - work.cwl
+  - input.yml
+  - streamflow.yml
+  envs:
+  - env

compile-java/2/streamflow.yml

+version: v1.0
+workflows:
+  extract-and-compile-java:
+    type: cwl
+    config:
+      file: work.cwl
+      settings: input.yml
+    bindings:
+    - step: /untar
+      target:
+        deployment: openjdk
+        service: debian
+    - step: /compile
+      target:
+        deployment: openjdk
+        service: debian
+deployments:
+  busybox:
+    type: helm
+    config:
+      inCluster: true
+      chart: opertusmundi/busybox
+      chartVersion: '0.1.1'
+      stringValues: >-
+        image.repository=docker-registry.ebrains.eu/tc/busybox,serviceAccount.create=false
+  openjdk:
+    type: helm
+    config:
+      inCluster: true
+      chart: opertusmundi/debian
+      chartVersion: '0.0.1'
+      stringValues: >-
+        image.repository=docker-registry.ebrains.eu/tc/openjdk,image.tag=11-jdk,serviceAccount.create=false

compile-java/2/work.cwl

+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+  tarball: File
+  names_of_files_to_extract: string[]
+
+outputs:
+  classfiles:
+    type: File[]
+    outputSource: compile/classfiles
+
+steps:
+  untar:
+    run:
+      class: CommandLineTool
+      baseCommand:
+      - tar
+      - x
+      - -zvo
+      stdout: output.txt
+      inputs:
+        tarfile:
+          type: File
+          inputBinding:
+            position: 1
+            prefix: -f
+        names_of_files_to_extract:
+          type: string[]
+          inputBinding:
+            position: 2
+      outputs:
+        extracted_files:
+          type: File[]
+          outputBinding:
+            glob: "*.java"
+    in:
+      tarfile: tarball
+      names_of_files_to_extract: names_of_files_to_extract
+    out: [extracted_files]
+
+  compile:
+    run:
+      class: CommandLineTool
+      baseCommand: javac
+      arguments:
+      - -d
+      - "$(runtime.outdir)"
+      inputs:
+        src:
+          type: File[]
+          inputBinding:
+            position: 1
+      outputs:
+        classfiles:
+          type: File[]
+          outputBinding:
+            glob: "*.class"
+    in:
+      src: untar/extracted_files
+    out: [classfiles]

echo/1/env

+JOB_ID=echo-1

echo/1/input.yml

+message1: Hello CWL workflow!

echo/1/kustomization.yml

+resources:
+- ../../base/
+
+nameSuffix: "-echo-1"
+
+configMapGenerator:
+- name: streamflow-config
+  files:
+  - work.cwl
+  - input.yml
+  - streamflow.yml
+  envs:
+  - env

echo/1/streamflow.yml

+version: v1.0
+workflows:
+  echo-and-uppercase:
+    type: cwl
+    config:
+      file: work.cwl
+      settings: input.yml
+    bindings:
+    - step: /echo
+      target:
+        deployment: echo
+        service: busybox
+deployments:
+  echo:
+    type: helm
+    config:
+      inCluster: true
+      #chart: https://opertusmundi.github.io/helm-charts/busybox-0.1.0.tgz
+      chart: opertusmundi/busybox
+      chartVersion: '0.1.1'
+      stringValues: >-
+        image.repository=docker-registry.ebrains.eu/tc/busybox
+      #releaseName: echo1
+      #namespace: workflows-1
+      timeout: '30s'

echo/1/work.cwl

+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  InlineJavascriptRequirement: {}
+
+inputs:
+  message1: string
+
+outputs:
+  out:
+    type: string
+    outputSource: uppercase/uppercase_message
+
+steps:
+  echo:
+    run:
+      class: CommandLineTool
+      baseCommand:
+      - echo
+      - -n
+      stdout: output.txt
+      inputs:
+        message:
+          type: string
+          inputBinding: {}
+      outputs:
+        out1:
+          type: string
+          outputBinding:
+            glob: output.txt
+            loadContents: true
+            outputEval: $(self[0].contents)
+    in:
+      message: message1
+    out: [out1]
+  uppercase:
+    run:
+      class: ExpressionTool
+      requirements:
+        InlineJavascriptRequirement: {}
+      inputs:
+        message: string
+      outputs:
+        uppercase_message: string
+      expression: |
+        ${ return {"uppercase_message": inputs.message.toUpperCase()}; }
+    in:
+      message: echo/out1
+    out: [uppercase_message]

echo/2/env

+JOB_ID=echo-2