From 19ebaa68f4b40091fff92054b3255df683689ae9 Mon Sep 17 00:00:00 2001
From: Ben Cumming <bcumming@cscs.ch>
Date: Fri, 23 Apr 2021 09:14:27 +0200
Subject: [PATCH] Remove codecov from gitlab tests (#1499)
In response to:
https://gizmodo.com/u-s-federal-investigators-are-reportedly-looking-into-1846707144
Rationale:
* CodeCov didn't detect the breach for two months
* They took a further 2 weeks to alert users after that
* Their efforts to address the issue after that have been painful to watch
* https://github.com/codecov/codecov-action/issues/281#issuecomment-823407167
---
README.md | 1 -
ci/codecov/build.Dockerfile | 55 ---------------------
ci/codecov/deploy.Dockerfile | 92 ------------------------------------
ci/codecov_post | 19 --------
ci/codecov_pre | 10 ----
ci/gitlab-cscs.yml | 73 ----------------------------
ci/upload_codecov | 11 -----
7 files changed, 261 deletions(-)
delete mode 100644 ci/codecov/build.Dockerfile
delete mode 100644 ci/codecov/deploy.Dockerfile
delete mode 100755 ci/codecov_post
delete mode 100755 ci/codecov_pre
delete mode 100755 ci/upload_codecov
diff --git a/README.md b/README.md
index fdcf23ae..eb0437df 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,5 @@
[](https://gitlab.com/cscs-ci/arbor-sim/arbor/-/commits/master)
[](https://github.com/arbor-sim/arbor/actions?query=workflow%3A%22Basic+Tests+and+Documentation%22)
-[](https://codecov.io/gl/cscs-ci:arbor-sim/arbor)
[](https://gitpod.io/#https://github.com/arbor-sim/arbor)
[](https://arbor.readthedocs.io/en/latest/)
diff --git a/ci/codecov/build.Dockerfile b/ci/codecov/build.Dockerfile
deleted file mode 100644
index b5e9d5a8..00000000
--- a/ci/codecov/build.Dockerfile
+++ /dev/null
@@ -1,55 +0,0 @@
-FROM nvidia/cuda:10.2-devel-ubuntu18.04
-
-WORKDIR /root
-
-ARG MPICH_VERSION=3.3.2
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV FORCE_UNSAFE_CONFIGURE 1
-ENV MPICH_VERSION ${MPICH_VERSION}
-
-# Install basic tools
-RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \
- python3 \
- git tar wget curl \
- gcc-8 g++-8 make && \
- update-alternatives \
- --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 \
- --slave /usr/bin/g++ g++ /usr/bin/g++-8 \
- --slave /usr/bin/gcov gcov /usr/bin/gcov-8 && \
- update-alternatives --config gcc && \
- rm -rf /var/lib/apt/lists/*
-
-RUN cd /usr/local/bin && \
- curl -Ls https://codecov.io/bash > codecov.sh && \
- echo "89c658e261d5f25533598a222fd96cf17a5fa0eb3772f2defac754d9970b2ec8 codecov.sh" | sha256sum --check --quiet && \
- chmod +x codecov.sh
-
-RUN wget -q "https://github.com/linux-test-project/lcov/archive/v1.15.tar.gz" && \
- echo "d88b0718f59815862785ac379aed56974b9edd8037567347ae70081cd4a3542a v1.15.tar.gz" | sha256sum --check --quiet && \
- tar -xzf v1.15.tar.gz && \
- cd lcov-1.15 && \
- make install -j$(nproc) && \
- rm -rf lcov-1.15 v1.15.tar.gz
-
-# Install MPICH ABI compatible with Cray's lib on Piz Daint
-RUN wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz -O mpich.tar.gz && \
- echo "4bfaf8837a54771d3e4922c84071ef80ffebddbb6971a006038d91ee7ef959b9 mpich.tar.gz" | sha256sum --check --quiet && \
- tar -xzf mpich.tar.gz && \
- cd mpich-${MPICH_VERSION} && \
- ./configure --disable-fortran && \
- make install -j$(nproc) && \
- rm -rf mpich.tar.gz mpich-${MPICH_VERSION}
-
-# Install cmake
-RUN wget -q "https://github.com/Kitware/CMake/releases/download/v3.12.4/cmake-3.12.4-Linux-x86_64.tar.gz" -O cmake.tar.gz && \
- echo "486edd6710b5250946b4b199406ccbf8f567ef0e23cfe38f7938b8c78a2ffa5f cmake.tar.gz" | sha256sum --check --quiet && \
- tar --strip-components=1 -xzf cmake.tar.gz -C /usr/local && \
- rm -rf cmake.tar.gz
-
-# Install bundle tooling for creating small Docker images
-RUN wget -q https://github.com/haampie/libtree/releases/download/v1.2.0/libtree_x86_64.tar.gz && \
- echo "4316a52aed7c8d2f7d2736c935bbda952204be92e56948110a143283764c427c libtree_x86_64.tar.gz" | sha256sum --check --quiet && \
- tar -xzf libtree_x86_64.tar.gz && \
- rm libtree_x86_64.tar.gz && \
- ln -s /root/libtree/libtree /usr/local/bin/libtree
diff --git a/ci/codecov/deploy.Dockerfile b/ci/codecov/deploy.Dockerfile
deleted file mode 100644
index caf6a6e1..00000000
--- a/ci/codecov/deploy.Dockerfile
+++ /dev/null
@@ -1,92 +0,0 @@
-# Multistage build: here we import the current source code
-# into build environment image, build the project, bundle it
-# and then extract it into a small image that just contains
-# the binaries we need to run
-
-ARG BUILD_ENV
-
-ARG SOURCE_DIR=/arbor-source
-ARG BUILD_DIR=/arbor-build
-ARG BUNDLE_DIR=/root/arbor.bundle
-
-FROM $BUILD_ENV as builder
-
-ARG SOURCE_DIR
-ARG BUILD_DIR
-ARG BUNDLE_DIR
-
-# Build arbor
-COPY . ${SOURCE_DIR}
-
-# Build and bundle binaries
-RUN mkdir ${BUILD_DIR} && cd ${BUILD_DIR} && \
- CC=mpicc CXX=mpicxx cmake ${SOURCE_DIR} \
- -DARB_VECTORIZE=ON \
- -DARB_ARCH=broadwell \
- -DARB_WITH_PYTHON=OFF \
- -DARB_WITH_MPI=ON \
- -DARB_GPU=cuda \
- -DARB_USE_BUNDLED_LIBS=ON \
- -DCMAKE_BUILD_TYPE=Debug \
- -DCMAKE_CXX_FLAGS="-g -O0 --coverage" \
- -DCMAKE_EXE_LINKER_FLAGS="--coverage" \
- -DCMAKE_INSTALL_PREFIX=/usr && \
- make -j$(nproc) tests && \
- libtree --chrpath \
- -d ${BUNDLE_DIR} \
- ${BUILD_DIR}/bin/modcc \
- ${BUILD_DIR}/bin/unit \
- ${BUILD_DIR}/bin/unit-local \
- ${BUILD_DIR}/bin/unit-modcc \
- ${BUILD_DIR}/bin/unit-mpi
-
-# Install some code cov related executables
-RUN libtree -d ${BUNDLE_DIR} $(which gcov) && \
- cp -L ${SOURCE_DIR}/ci/codecov_pre ${SOURCE_DIR}/ci/codecov_post ${SOURCE_DIR}/ci/upload_codecov ${BUNDLE_DIR}/usr/bin && \
- cp -L $(which lcov geninfo) ${BUNDLE_DIR}/usr/bin && \
- cp -L /usr/local/bin/codecov.sh ${BUNDLE_DIR}/usr/bin
-
-# In the build dir, remove everything except for gcno coverage files
-RUN mv ${BUILD_DIR} ${BUILD_DIR}-tmp && \
- mkdir ${BUILD_DIR} && \
- cd ${BUILD_DIR}-tmp && \
- find -iname "*.gcno" -exec cp --parent \{\} ${BUILD_DIR} \; && \
- rm -rf ${BUILD_DIR}-tmp
-
-# Only keep the sources for tests, not the git history
-RUN rm -rf ${SOURCE_DIR}/.git
-
-FROM ubuntu:18.04
-
-ARG SOURCE_DIR
-ARG BUILD_DIR
-ARG BUNDLE_DIR
-
-ENV SOURCE_DIR=$SOURCE_DIR
-ENV BUILD_DIR=$BUILD_DIR
-ENV BUNDLE_DIR=$BUNDLE_DIR
-
-# This is the only thing necessary really from nvidia/cuda's ubuntu18.04 runtime image
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"
-
-# Install perl to make lcov happy
-RUN apt-get update -qq && \
- apt-get install --no-install-recommends -qq perl curl ca-certificates && \
- rm -rf /var/lib/apt/lists/*
-
-COPY --from=builder ${BUNDLE_DIR} ${BUNDLE_DIR}
-COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR}
-COPY --from=builder ${BUILD_DIR} ${BUILD_DIR}
-
-# Make it easy to call our binaries.
-ENV PATH="${BUNDLE_DIR}/usr/bin:$PATH"
-
-# Automatically print stacktraces on segfault
-ENV LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
-
-RUN echo "${BUNDLE_DIR}/usr/lib/" > /etc/ld.so.conf.d/arbor.conf && ldconfig
-
-WORKDIR ${BUNDLE_DIR}/usr/bin
-
diff --git a/ci/codecov_post b/ci/codecov_post
deleted file mode 100755
index f5c4f7f6..00000000
--- a/ci/codecov_post
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# In case of MPI tests running on a shared file system, we run into race conditions writing files
-# so here we generate some unique names for the codecov files.
-
-LOCAL_REPORTS="/codecov-reports"
-SHARED_REPORTS="$CI_PROJECT_DIR/codecov-reports"
-REPORT_NAME=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1`
-mkdir -p "$SHARED_REPORTS"
-
-# Create coverage reports for code run
-echo "Combining reports"
-lcov --exclude "*/ext/*" --exclude "*/test/*" --exclude "*/mechanisms/*" --no-external --capture --base-directory $SOURCE_DIR --directory $BUILD_DIR --output-file "$LOCAL_REPORTS/run.info"
-lcov --add-tracefile "$LOCAL_REPORTS/baseline-codecov.info" --add-tracefile "$LOCAL_REPORTS/run.info" --output-file "$LOCAL_REPORTS/combined.info"
-
-# Only keep our own source
-lcov --extract "$LOCAL_REPORTS/combined.info" "$SOURCE_DIR/*" --output-file "$LOCAL_REPORTS/combined.info"
-
-cp "$LOCAL_REPORTS/combined.info" "$SHARED_REPORTS/codecov-$REPORT_NAME.info"
diff --git a/ci/codecov_pre b/ci/codecov_pre
deleted file mode 100755
index 2c56e40a..00000000
--- a/ci/codecov_pre
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# In case of MPI tests running on a shared file system, we run into race conditions writing files
-# so here we generate some unique names for the codecov files.
-
-LOCAL_REPORTS="/codecov-reports"
-mkdir -p "$LOCAL_REPORTS"
-
-echo "Generating baseline codecov report"
-lcov --exclude "*/ext/*" --exclude "*/test/*" --exclude "*/mechanisms/*" --no-external --capture --initial --base-directory $SOURCE_DIR --directory $BUILD_DIR --output-file "$LOCAL_REPORTS/baseline-codecov.info"
diff --git a/ci/gitlab-cscs.yml b/ci/gitlab-cscs.yml
index e41591e5..e8d46151 100644
--- a/ci/gitlab-cscs.yml
+++ b/ci/gitlab-cscs.yml
@@ -30,14 +30,6 @@ build release:
DEPLOY_DOCKERFILE: ci/release/deploy.Dockerfile
DEPLOY_IMAGE: $CSCS_REGISTRY_IMAGE/release/deploy:$CI_COMMIT_SHA
-build codecov:
- extends: .build_docker_images
- variables:
- BUILD_DOCKERFILE: ci/codecov/build.Dockerfile
- BUILD_IMAGE: $CSCS_REGISTRY_IMAGE/codecov/build:v2
- DEPLOY_DOCKERFILE: ci/codecov/deploy.Dockerfile
- DEPLOY_IMAGE: $CSCS_REGISTRY_IMAGE/codecov/deploy:$CI_COMMIT_SHA
-
notify_github_start:
stage: build
allow_failure: true
@@ -103,71 +95,6 @@ deallocate release:
variables:
ALLOCATION_NAME: arbor-ci-release-$CI_PIPELINE_ID
-### Codecov tests ###
-allocate codecov:
- stage: allocate
- only: ['master', 'staging', 'trying']
- image: $CSCS_REGISTRY_IMAGE/codecov/deploy:$CI_COMMIT_SHA
- extends: .daint_alloc
- variables:
- PULL_IMAGE: 'YES'
- ALLOCATION_NAME: arbor-ci-codecov-$CI_PIPELINE_ID
-
-single node codecov:
- extends: .daint
- only: ['master', 'staging', 'trying']
- image: $CSCS_REGISTRY_IMAGE/codecov/deploy:$CI_COMMIT_SHA
- stage: test
- script:
- - codecov_pre
- - unit
- - unit-local
- - unit-modcc
- - codecov_post
- variables:
- SLURM_JOB_NUM_NODES: 1
- SLURM_NTASKS: 1
- ALLOCATION_NAME: arbor-ci-codecov-$CI_PIPELINE_ID
- artifacts:
- paths:
- - codecov-reports/
-
-multi node codecov:
- extends: .daint
- only: ['master', 'staging', 'trying']
- image: $CSCS_REGISTRY_IMAGE/codecov/deploy:$CI_COMMIT_SHA
- stage: test
- script:
- - codecov_pre
- - unit-mpi
- - codecov_post
- variables:
- SLURM_JOB_NUM_NODES: 2
- SLURM_NTASKS: 2
- ALLOCATION_NAME: arbor-ci-codecov-$CI_PIPELINE_ID
- artifacts:
- paths:
- - codecov-reports/
-
-upload codecov reports:
- extends: .daint
- only: ['master', 'staging', 'trying']
- image: $CSCS_REGISTRY_IMAGE/codecov/deploy:$CI_COMMIT_SHA
- stage: upload_reports
- variables:
- SLURM_JOB_NUM_NODES: 1
- SLURM_NTASKS: 1
- ALLOCATION_NAME: arbor-ci-codecov-$CI_PIPELINE_ID
- script: upload_codecov
-
-deallocate codecov:
- only: ['master', 'staging', 'trying']
- image: $CSCS_REGISTRY_IMAGE/codecov/deploy:$CI_COMMIT_SHA
- stage: cleanup
- extends: .daint_dealloc
- variables:
- ALLOCATION_NAME: arbor-ci-codecov-$CI_PIPELINE_ID
-
notify_github_success:
stage: cleanup
when: on_success
diff --git a/ci/upload_codecov b/ci/upload_codecov
deleted file mode 100755
index 343f157d..00000000
--- a/ci/upload_codecov
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# Combine all reports into a single one
-SHARED_REPORTS="$CI_PROJECT_DIR/codecov-reports"
-TRACE_FILES_ARGS=`find "$SHARED_REPORTS" -type f -iname '*.info' -exec sh -c "echo --add-tracefile {}" \;`
-lcov ${TRACE_FILES_ARGS} --output-file "$SHARED_REPORTS/combined.info"
-
-pushd $SOURCE_DIR
-codecov.sh -f "$SHARED_REPORTS/combined.info" -t $CODECOV_TOKEN_GITHUB
-codecov.sh -f "$SHARED_REPORTS/combined.info" -t $CODECOV_TOKEN_GITLAB
-popd
\ No newline at end of file
--
GitLab