Add spike counts to pre-commit tests. (#1965)

- Enhance `run_cpp_examples.sh` to do some very basic checking of spikes counts
- Add a script to check all releases
- The former must succeed for all commits to master

Here's the expected spike counts as of 0.7
- brunel 6998
- bench 972
- ring 94 (19 pre-v0.7)
- gap_junctions 30

doc/contrib/release.rst

@@ -75,6 +75,8 @@ Release
    
 #. Change ``VERSION``. Make sure does not end with ``-rc`` or ``-dev``.
 
+#. Update ``scripts/check-all-tags.sh`` to check the current tag.
+
 #. Tag
 
    - commit and open a PR for the above changes.

scripts/check-all-tags.sh

+#!/usr/bin/env bash
+# Runs all C++ examples
+
+set -Eeuo pipefail
+
+if [[ "$#" -gt 1 ]]; then
+    echo "usage: run_cpp_examples.sh <prefix, e.g., mpirun -n 4 -oversubscribe>"
+	exit 1
+fi
+
+PREFIX="${1:-} `pwd`/build/bin"
+
+cxx=/usr/local/opt/llvm/bin/clang++
+cc=/usr/local/opt/llvm/bin/clang
+
+for tag in v0.4 v0.5.2 v0.6 v0.7
+do
+    echo "Version=$tag"
+    rm -rf ext/*
+    git checkout $tag
+    git checkout $tag -- ext/
+    git submodule update --init
+    for simd in ON OFF
+    do
+        echo " * simd=$simd"
+        out=results/$tag-`git rev-parse --short HEAD`/cpp/simd=$simd
+        cd build
+        cmake .. -DARB_USE_BUNDLED_LIBS=ON -DCMAKE_CXX_COMPILER=$cxx -DCMAKE_C_COMPILER=$cc -DCMAKE_BUILD_TYPE=release -DARB_VECTORIZE=$simd -DARB_ARCH=native
+        ninja install examples
+        cd -
+        for ex in bench brunel gap_junctions generators lfp ring single-cell "probe-demo v"
+        do
+            echo "   - $ex"
+            dir=`echo $ex | tr ' ' '_'`
+            mkdir -p $out/$dir
+            cd $out/$dir
+            $PREFIX/$ex > stdout.txt 2> stderr.txt
+            cd -
+        done
+    done
+done
+
+ok=0
+check () {
+    prog=$1
+    expected="$2 spikes"
+    actual=$(/usr/bin/grep -Eo '\d+ spikes' results/$tag/cpp/SIMD=$simd/$prog/stdout.txt || echo "N/A")
+    if [ "$expected" == "$actual" ]
+    then
+        echo "   - $prog: OK"
+    else
+        echo "   - $prog: ERROR wrong number of spikes: $expected ./. $actual"
+        ok=1
+    fi
+}
+
+for tag in "v0.4-79855b66" "v0.5.2-51e35898" "v0.6-930c23eb" "v0.7-d0e424b4"
+do
+    echo "Version=$tag"
+    for simd in ON OFF
+    do
+        echo " * SIMD=$simd"
+        check brunel 6998
+        check bench 972
+        if [[ "$tag" < "v0.7" ]]
+        then
+            check ring 19
+        else
+            check ring 94
+        fi
+        check gap_junctions 30
+    done
+done
+exit $ok

scripts/run_cpp_examples.sh

@@ -8,14 +8,38 @@ if [[ "$#" -gt 1 ]]; then
 	exit 1
 fi
 
-PREFIX=${1:-}
+PREFIX="${1:-} `pwd`/build/bin"
+tag=dev-`git rev-parse --short HEAD`
+out="results/$tag/cpp/"
 
-$PREFIX build/bin/bench
-$PREFIX build/bin/brunel
-$PREFIX build/bin/dryrun
-$PREFIX build/bin/gap_junctions
-$PREFIX build/bin/generators
-$PREFIX build/bin/lfp
-$PREFIX build/bin/probe-demo v
-$PREFIX build/bin/ring
-$PREFIX build/bin/single-cell
+ok=0
+check () {
+    prog=$1
+    expected="$2 spikes"
+    actual=$(grep -Eo '[0-9]+ spikes' $out/$prog/stdout.txt || echo "N/A")
+    if [ "$expected" == "$actual" ]
+    then
+        echo "   - $prog: OK"
+    else
+        echo "   - $prog: ERROR wrong number of spikes: $expected ./. $actual"
+        ok=1
+    fi
+}
+
+for ex in bench brunel gap_junctions generators lfp ring single-cell "probe-demo v"
+do
+    echo "   - $ex"
+    dir=`echo $ex | tr ' ' '_'`
+    mkdir -p $out/$dir
+    cd $out/$dir
+    $PREFIX/$ex > stdout.txt 2> stderr.txt
+    cd -
+done
+
+# Do some sanity checks.
+check brunel 6998
+check bench 972
+check ring 94
+check gap_junctions 30
+
+exit $ok