Refactor(spack): Move spack package/view settings into per-flavor files

Change-Id: I4b3529c33f7969697dbd7e5c76c72295bcf7c14e

.ci/commons.sh

@@ -3,6 +3,8 @@
 set -euo pipefail
 shopt -s inherit_errexit 2>/dev/null || true
 
+SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
+
 JENKINS_ENV_FILE_INSIDE="/tmp/spack/jenkins.env"
 if [ -n "${WORKSPACE:-}" ]; then
     # we are not in container
@@ -201,25 +203,7 @@ get_pinned_deps() {
 # the version of dev tools we want in our view
 SPEC_VIEW_VISIONARY_DEV_TOOLS="visionary-dev-tools ^${DEPENDENCY_PYTHON3} $(get_pinned_deps dev) %${VISIONARY_GCC}"
 
-# All spack packages that should be fetched/installed in the container
-spack_packages=(
-    "${SPEC_VIEW_VISIONARY_DEV_TOOLS}"
-    "visionary-simulation~dev ^${DEPENDENCY_PYTHON} $(get_pinned_deps simulation) %${VISIONARY_GCC}"
-    "visionary-simulation ^${DEPENDENCY_PYTHON} $(get_pinned_deps simulation) %${VISIONARY_GCC}"
-    "visionary-wafer~dev ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
-    "visionary-wafer ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
-    "visionary-wafer ~dev+gccxml^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
-    "visionary-wafer+gccxml ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
-    "visionary-wafer-visu ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer-visu) %${VISIONARY_GCC}"
-    # START python 3 packages
-    "visionary-clusterservices ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
-    "visionary-dls~dev ^${DEPENDENCY_PYTHON3} $(get_pinned_deps dls) %${VISIONARY_GCC}"
-    "visionary-dls ^${DEPENDENCY_PYTHON3} $(get_pinned_deps dls) %${VISIONARY_GCC}"
-    "py-jupyterhub ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
-    "py-jupyterhub-dummyauthenticator ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
-    "py-jupyterhub-simplespawner ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
-    # END python 3 packages
-)
+source "${SOURCE_DIR}/${CONTAINER_STYLE}_spack_collection.sh"
 
 # Control verbosity etc of commands
 SPACK_ARGS_INSTALL=()
@@ -248,22 +232,6 @@ spack_bootstrap_dependencies=(
 # Views are put under /opt/spack_views/visionary-xy
 # The app names are then just xy for smaller terminal lines.
 
-spack_views=(\
-        visionary-dev-tools
-        visionary-dls-core
-        visionary-dls
-        visionary-dls-nodev
-        visionary-simulation
-        visionary-simulation-nodev
-        visionary-slurmviz
-        visionary-wafer
-        visionary-wafer-nodev
-    )
-
-spack_views_no_default_gcc=(
-    "visionary-nux" # currenlty visionary-nux is no view, but serves as example
-)
-
 # associative array: spec to add -> view names seperated by spaces
 declare -A spack_add_to_view
 # associative array: spec to add -> "yes" for when dependencies should be added

.ci/complete_spack_install_routine_called_in_post_as_root.sh

@@ -11,7 +11,7 @@ shopt -s inherit_errexit 2>/dev/null || true
 SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
 source "${SOURCE_DIR}/commons.sh"
 
-"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_prepare_spack_as_root.sh"
+"${SPACK_INSTALL_SCRIPTS}/prepare_spack_as_root.sh"
 sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/bootstrap_spack.sh"
 
 sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_spack.sh" && \
@@ -20,7 +20,7 @@ sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings.sh" && \
 # remove temporary cache folder
 rm -rfv /opt/spack/.spack
 
-"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_restore_spack_user_settings_as_root.sh" && \
+"${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings_as_root.sh" && \
 "${SPACK_INSTALL_SCRIPTS}/generate_modules.sh" || \
 (
 sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/preserve_built_spack_packages.sh" &&

.ci/install_spack.sh

@@ -48,35 +48,7 @@ umask 000
 # several spack packages get linked into the same view and the random order of
 # execution in a parallel context, builds might become unstable otherwise.
 {
-cat <<EOF
-####################################
-# Packages still plagued by gccxml #
-####################################
-
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-wafer $(get_latest_hash visionary-wafer+dev~gccxml)
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-wafer-nodev $(get_latest_hash visionary-wafer~dev~gccxml)
-
-##################################################
-# Strong independent packages who need no gccxml #
-##################################################
-
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-simulation $(get_latest_hash "visionary-simulation+dev")
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-simulation-nodev $(get_latest_hash "visionary-simulation~dev")
-
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dls-core $(get_latest_hash visionary-dls-core "^${DEPENDENCY_PYTHON3}")
-
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dls $(get_latest_hash visionary-dls+dev "^${DEPENDENCY_PYTHON3}")
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dls-nodev $(get_latest_hash visionary-dls~dev "^${DEPENDENCY_PYTHON3}")
-
-# slurvmiz needs no dev-tools because it is not for end-users
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-slurmviz $(get_latest_hash "visionary-slurmviz %${VISIONARY_GCC}")
-
-#############
-# dev tools #
-#############
-
-${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dev-tools $(get_latest_hash "${SPEC_VIEW_VISIONARY_DEV_TOOLS}")
-EOF
+source ${SPACK_INSTALL_SCRIPTS}/${CONTAINER_STYLE}_spack_custom_view.sh
 } | parallel_cmds
 
 # Perform the remaining additions to the views defined in commons.

.ci/visionary_prepare_spack_as_root.sh → .ci/prepare_spack_as_root.sh

@@ -10,22 +10,22 @@ source "${SOURCE_DIR}/commons.sh"
 
 # spack stuff
 # home has to exist, so we simply point ot /opt/spack
-adduser spack --uid 888 --no-create-home --home /opt/spack --disabled-password --system --shell /bin/bash
-chown spack:nogroup /opt
+spack_create_user_cmd
+chown spack:$spack_gid /opt
 mkdir /opt/spack_views
-chown spack:nogroup /opt/spack_views
+chown spack:$spack_gid /opt/spack_views
 mkdir -p "${SPEC_FOLDER_IN_CONTAINER}"
-chown spack:nogroup "${SPEC_FOLDER_IN_CONTAINER}"
-chown spack:nogroup "${BUILD_CACHE_INSIDE}"
+chown spack:$spack_gid "${SPEC_FOLDER_IN_CONTAINER}"
+chown spack:$spack_gid "${BUILD_CACHE_INSIDE}"
 chmod go=rwx /opt
 # in the final image /opt/spack* should be owned by the spack user.
 # Therefore: chown everything to the spack user except for var/cache (contains
 # hardlinks to vis_jenkins-owned files)
 find "/opt/spack" \
     \( -type d -wholename "/opt/spack/var/spack/cache" -prune \
-    \) -o -exec chown spack:nogroup '{}' \;
+    \) -o -exec chown spack:$spack_gid '{}' \;
 chmod +x /opt/spack_install_scripts/*.sh
 # have a convenience folder to easily execute other shells for user
 # sessions independent of any app
 mkdir /opt/shell
-chown spack:nogroup /opt/shell
+chown spack:$spack_gid /opt/shell

.ci/visionary_restore_spack_user_settings_as_root.sh → .ci/restore_spack_user_settings_as_root.sh

@@ -12,4 +12,4 @@ source "${sourcedir}/commons.sh"
 
 # shrink image: remove download cache (owned by vis_jenkins)
 rm -rf "${MY_SPACK_FOLDER}"/var/spack/cache/*
-chown spack:nogroup "${MY_SPACK_FOLDER}"/var/spack/cache
+chown spack:$spack_gid "${MY_SPACK_FOLDER}"/var/spack/cache

.ci/visionary_spack_collection.sh

+# All spack packages that should be fetched/installed in the container
+spack_packages=(
+    "${SPEC_VIEW_VISIONARY_DEV_TOOLS}"
+    "visionary-simulation~dev ^${DEPENDENCY_PYTHON} $(get_pinned_deps simulation) %${VISIONARY_GCC}"
+    "visionary-simulation ^${DEPENDENCY_PYTHON} $(get_pinned_deps simulation) %${VISIONARY_GCC}"
+    "visionary-wafer~dev ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
+    "visionary-wafer ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
+    "visionary-wafer ~dev+gccxml^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
+    "visionary-wafer+gccxml ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer) %${VISIONARY_GCC}"
+    "visionary-wafer-visu ^${DEPENDENCY_PYTHON} $(get_pinned_deps wafer-visu) %${VISIONARY_GCC}"
+    # START python 3 packages
+    "visionary-clusterservices ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
+    "visionary-dls~dev ^${DEPENDENCY_PYTHON3} $(get_pinned_deps dls) %${VISIONARY_GCC}"
+    "visionary-dls ^${DEPENDENCY_PYTHON3} $(get_pinned_deps dls) %${VISIONARY_GCC}"
+    "py-jupyterhub ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
+    "py-jupyterhub-dummyauthenticator ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
+    "py-jupyterhub-simplespawner ^${DEPENDENCY_PYTHON3} %${VISIONARY_GCC}"
+    # END python 3 packages
+)
+
+spack_views=(\
+    visionary-dev-tools
+    visionary-dls-core
+    visionary-dls
+    visionary-dls-nodev
+    visionary-simulation
+    visionary-simulation-nodev
+    visionary-slurmviz
+    visionary-wafer
+    visionary-wafer-nodev
+)
+
+spack_views_no_default_gcc=(
+    "visionary-nux" # currenlty visionary-nux is no view, but serves as example
+)
+
+
+
+spack_gid="nogroup"
+
+spack_create_user_cmd() {
+    adduser spack --uid 888 --no-create-home --home /opt/spack --disabled-password --system --shell /bin/bash
+}

.ci/visionary_spack_custom_view.sh

+cat <<EOF
+####################################
+# Packages still plagued by gccxml #
+####################################
+
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-wafer $(get_latest_hash visionary-wafer+dev~gccxml)
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-wafer-nodev $(get_latest_hash visionary-wafer~dev~gccxml)
+
+##################################################
+# Strong independent packages who need no gccxml #
+##################################################
+
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-simulation $(get_latest_hash "visionary-simulation+dev")
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-simulation-nodev $(get_latest_hash "visionary-simulation~dev")
+
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dls-core $(get_latest_hash visionary-dls-core "^${DEPENDENCY_PYTHON3}")
+
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dls $(get_latest_hash visionary-dls+dev "^${DEPENDENCY_PYTHON3}")
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dls-nodev $(get_latest_hash visionary-dls~dev "^${DEPENDENCY_PYTHON3}")
+
+# slurvmiz needs no dev-tools because it is not for end-users
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-slurmviz $(get_latest_hash "visionary-slurmviz %${VISIONARY_GCC}")
+
+#############
+# dev tools #
+#############
+
+${MY_SPACK_BIN} ${SPACK_ARGS_VIEW[@]+"${SPACK_ARGS_VIEW[@]}"} view -d yes symlink -i ${MY_SPACK_VIEW_PREFIX}/visionary-dev-tools $(get_latest_hash "${SPEC_VIEW_VISIONARY_DEV_TOOLS}")
+EOF