First effort to copy files directly from gitlab runner to the OpenShift job pod

.gitlab-ci.yml

@@ -9,21 +9,24 @@ variables:
   #SWIFT_FILE: ebrains-spack-builds${CI_PIPELINE_ID}.tar.gz
   SWIFT_FILE: ebrains-spack-builds.tar.gz
 
-copy-files-to-swift:
-  stage: upload
-  script:
-    - 'curl -H "X-Auth-Token: ${OS_TOKEN}" -X DELETE ${OBJSTORE}ebrains-spack-builds.tar.gz > curl_log.txt'
-    - if [ $(cat curl_log.txt | grep "Unauthorized" |wc -l) -gt 0 ]; then echo "You need to refresh OpenStack token"; exit 1;fi;
-    - tar czf ${SWIFT_FILE} packages/ repo.yaml spack.yaml
-    - 'curl -H "X-Auth-Token: ${OS_TOKEN}" -i ${OBJSTORE} -X PUT -T ${SWIFT_FILE}'
-  tags:
-    - shell-runner
+#copy-files-to-swift:
+#  stage: upload
+#  script:
+#    - 'curl -H "X-Auth-Token: ${OS_TOKEN}" -X DELETE ${OBJSTORE}ebrains-spack-builds.tar.gz > curl_log.txt'
+#    - if [ $(cat curl_log.txt | grep "Unauthorized" |wc -l) -gt 0 ]; then echo "You need to refresh OpenStack token"; exit 1;fi;
+#    - tar czf ${SWIFT_FILE} packages/ repo.yaml spack.yaml
+#    - 'curl -H "X-Auth-Token: ${OS_TOKEN}" -i ${OBJSTORE} -X PUT -T ${SWIFT_FILE}'
+#  tags:
+#    - shell-runner
 
 deploy-build-environment:
   stage: deploy
-  needs: [copy-files-to-swift]
+  #needs: [copy-files-to-swift]
   before_script:
     - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN"
+    - tar czf ${SWIFT_FILE} packages/ repo.yaml spack.yaml
+    - mkdir copy_folder
+    - mv ${SWIFT_FILE} copy_folder
   script:
     # create job description file
     - chmod a+x create_job.sh
@@ -33,15 +36,20 @@ deploy-build-environment:
     - oc project jupyterhub-int
     # start the deploy job
     - oc create -f simplejob.yml
-    # wait for job to finish https://stackoverflow.com/questions/5073453wait-for-kubernetes-job-to-complete-on-either-failure-success-using-command-line
+    ## wait for job to finish https://stackoverflow.com/questions/5073453wait-for-kubernetes-job-to-complete-on-either-failure-success-using-command-line
     - oc get job/simplejob${CI_PIPELINE_ID} -o=jsonpath='{.status}' -w && oc get job/simplejob${CI_PIPELINE_ID} -o=jsonpath='{.status.conditions[*].type}' | grep -i -E 'failed|complete' || echo 'Failed'
+    # wait for job's pod to become available so as to copy from the gitlab runner to the OpenShift pod 
+    # the necessary files that define the environment that spack needs to build
+    - while true; do x=$(oc get pods |grep simplejob${CI_PIPELINE_ID}|awk '{ print $3}');if [ $x == "Running" ]; then break; fi; sleep 1; done
+    - pod=$(oc get pods |grep simplejob${CI_PIPELINE_ID}|awk '{ print $1}')
+    - oc rsync copy_folder $pod:/opt/app-root/src
     - while true; do x=$(oc get pods |grep simplejob${CI_PIPELINE_ID}|awk '{ print $3}');if [ $x != "Running" ]; then break; fi; sleep 10; done 
     # when build job finishes get the logs
     - oc logs jobs/simplejob${CI_PIPELINE_ID} | tee log.txt
     # if spack install has failed, fail the pipeline
     - if [ $(cat log.txt |grep "Error:"|wc -l) -gt 1 ]; then exit 1;fi;
     # delete the job from OpenShift as we have the logs here
-    - oc delete job simplejob${CI_PIPELINE_ID} || true
+    #- oc delete job simplejob${CI_PIPELINE_ID} || true
   tags:
     - shell-runner