v0.6-rc (#1814)

* bump version to v0.6-rc
* Update `MANIFEST.in` and `scripts/run_python_examples.sh`
* Update Release instructions `docs/contrib/release.rst`
* Fix missing quotes in `python/example/single_cell_nml.py`
* Add testing to sdist creation in `ciwheel.yml`
* Rename `basic.yml` to `test-everything.yml`
   * Remove Python examples step from matrix (mostly because plotting takes much CPU on macos)
   * change to build /w nml
   * add test of pip builds (`pip install` in addition to CMake builds)
      * Python examples are tested here, once is enough.

.github/workflows/ciwheel.yml

@@ -2,7 +2,9 @@ name: Arbor on Wheels
 
 on:
   push:
-    branches: [ ciwheel ]
+    branches: 
+      - ciwheel
+      - v*rc
     tags:
       - v*
 
@@ -72,6 +74,12 @@ jobs:
           submodules: recursive
       - name: Make sdist
         run:  python setup.py sdist
+      - name: Install sdist
+        run:  python -m pip install dist/arbor*.tar.gz
+      - name: Run Python tests
+        run: python3 -m unittest discover -v -s python
+      - name: Run Python examples
+        run: scripts/run_python_examples.sh
       - uses: actions/upload-artifact@v2
         with:
           name: dist

.github/workflows/basic.yml → .github/workflows/test-everything.yml

-name: Arbor
+name: Test Everything
 
 on:
   push:
@@ -7,8 +7,8 @@ on:
     branches: [ master ]
 
 jobs:
-  build:
-    name: "Default"
+  testallconfigs:
+    name: "Tests across OSes, versions, compilers, and build configs."
     runs-on: ${{ matrix.config.os }}
     strategy:
       matrix:
@@ -92,11 +92,15 @@ jobs:
         # See https://github.com/open-mpi/ompi/issues/6518
         OMPI_MCA_btl: "self,tcp"
     steps:
-      - name: Get clang/gcc 8
+      - name: "Linux: get clang/gcc 8, libxml2"
         if: ${{ startsWith(matrix.config.os, 'ubuntu') }}
         run: |
           sudo apt-get update
-          sudo apt-get install -y "clang-8" "lldb-8" "lld-8" "clang-format-8" g++-8
+          sudo apt-get install -y "clang-8" "lldb-8" "lld-8" "clang-format-8" g++-8 libxml2-dev
+      - name: "MacOS: get libxml2"
+        if: ${{ startsWith(matrix.config.os, 'macos') }}
+        run: |
+          brew install libxml2 
       - name: Set up cmake
         uses: jwlawson/actions-setup-cmake@v1.12
         with:
@@ -149,7 +153,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake .. -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_C_COMPILER=$CC -DARB_WITH_PYTHON=ON -DARB_VECTORIZE=${{ matrix.config.simd }} -DPython3_EXECUTABLE=`which python` -DARB_WITH_MPI=${{ matrix.config.mpi }} -DARB_USE_BUNDLED_LIBS=ON
+          cmake .. -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_C_COMPILER=$CC -DARB_WITH_PYTHON=ON -DARB_VECTORIZE=${{ matrix.config.simd }} -DPython3_EXECUTABLE=`which python` -DARB_WITH_MPI=${{ matrix.config.mpi }} -DARB_USE_BUNDLED_LIBS=ON -DARB_WITH_NEUROML=ON
           make -j4 tests examples pyarb html
           cd -
       - name: Install arbor
@@ -176,9 +180,28 @@ jobs:
       - if:   ${{ matrix.config.mpi == 'ON' }}
         name: Run python+MPI tests
         run:  mpirun -n 4 -oversubscribe python3 -m unittest discover -v -s python
-      - name: Run Python examples
-        run: scripts/run_python_examples.sh
+      # - name: Run Python examples (plotting leads to time outs on macos, the step under testpip is enough)
+      #   run: scripts/run_python_examples.sh
       - name: Build and test a catalogue
         run: |
           arbor-build-catalogue -v default mechanisms/default
           ./scripts/test-catalogue.py ./default-catalogue.so
+  testpip:
+    name: "Pip build test + Python examples test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Python packages
+        run:  pip install numpy setuptools scikit-build ninja cmake
+      - name: Clone w/ submodules
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Build and install Arbor using pip + build flags
+        run: python3 -m pip install --verbose --install-option="-DARB_VECTORIZE=ON" --install-option="-DARB_ARCH=native" .
+      - name: Check that build flags match
+        run: |
+          python3 -c "import arbor; print(arbor.config())" | grep -q "'arch': 'native'"
+      - name: Run Python tests
+        run: python3 -m unittest discover -v -s python
+      - name: Run Python examples
+        run: scripts/run_python_examples.sh

MANIFEST.in

@@ -3,15 +3,20 @@ include CMakeLists.txt
 
 graft arbor
 graft arborenv
+graft arborio
+# graft ci
 graft cmake
 graft doc
 graft example
 graft ext
+graft lmorpho
 graft mechanisms
 graft modcc
 graft python
+graft scripts
+# graft spack
 graft sup
 graft test
-graft arborio
+# graft validation
 
 prune ext/google-benchmark

VERSION

-0.5.3-dev
+0.6-rc

doc/contrib/release.rst

@@ -33,54 +33,36 @@ Pre-release
 Update tags/versions and test
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-1. Create new temp-branch ending in ``-rc``. E.g. ``v0.5.1-rc``
+0. Check README.md, ATTRIBUTIONS.md, CONTRIBUTING.md.
+1. Create new temp-branch ending in ``-rc``. E.g. ``v0.6-rc``
 2. Bump the ``VERSION`` file:
    https://github.com/arbor-sim/arbor/blob/master/VERSION
-3. Update Python/pip/PyPi metadata and scripts
-
-   - Update MANIFEST (required for PyPi step later):
-     https://github.com/arbor-sim/arbor/blob/master/MANIFEST.in
-   - also checkout ``setup.cfg`` and ``setup.py``
-
-4. Double check all examples/tutorials/etc not covered by CI
-
-Manual test (deprecated)
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-5.  python setup.py sdist
-6.  python -m venv env
-7.  source env/bin/activate
-8.  move tarball here and extract
-9.  pip install –upgrade pip
-10. pip install numpy
-11. pip install ./arbor-0.5.1 –verbose
-12. python -c ’import arbor; print(arbor.__config__)’
-13. twine upload -r testpypi dist/\* (have some testrepo)
-14. create *another* venv: python -m venv env && source env/bin/activate
-15. pip install numpy
-16. pip install -i https://test.pypi.org/simple/ arbor==0.5.1 –verbose
-17. python -c ’import arbor; print(arbor.__config__)’
-
-Ciwheel/automated test (replaces manual test)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-5. Create/overwrite ``ciwheel`` branch with the above branch, and push
-   to Github.
-6. Collect artifact from GA run.
-7. twine upload -r testpypi dist/\*
-8. Ask users to test the above, e.g.:
+3. Run all tests.
+   - ``ciwheel.yml`` triggers when you push a branch called ``v*rc``, ON YOUR OWN REPO (so check ``github.com/$yourname/arbor/actions``). Make sure the tests pass.
+   - This should catch many problems. For a manual check:
+   - Verify MANIFEST.in (required for PyPI sdist)
+   - Check Python/pip/PyPi metadata and scripts, e.g. ``setup.py``
+   - Double check that all examples/tutorials/etc are covered by CI
+
+Test the RC
+~~~~~~~~~~~
+
+4. Collect artifact from the above GA run.
+   In case you want to manually want to trigger ``ciwheel.yml`` GA, overwrite the ``ciwheel`` branch with the commit of your choosing and force push to Github.
+5. ``twine upload -r testpypi dist/*``
+6. Ask users to test the above, e.g.:
 
 .. code-block:: bash
 
    python -m venv env && source env/bin/activate
-   pip install numpy pip install -i https://test.pypi.org/simple/ arbor==0.5.1
+   pip install numpy
+   pip install -i https://test.pypi.org/simple/ arbor==0.6-rc
    python -c ’import arbor; print(arbor.__config__)’
 
 Release
 -------
 
-0. Make sure ciwheel passes tests, produced working wheels. Make sure
-   tests on master also passed, and master == ciwheel
+0. Make sure ``ciwheel.yml`` passes tests, produced working wheels, and nobody reported problems testing the RC.
 1. Tag and release: https://github.com/arbor-sim/arbor/releases
 
    -  on cmdline: git tag -a TAGNAME
@@ -104,7 +86,7 @@ Release
 
 6. Upload to pypi
 
-   -  twine upload -r arborpypi dist/\*
+   -  twine upload -r arborpypi dist\*
 
 7. Verify
 
@@ -120,5 +102,20 @@ Post release
    -  first, update ``spack/package.py``. The checksum of the targz is the sha256sum.
    -  Then, use the file to `make PR here <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/>`_
 
+2. In the same PR with the update to `spack/package.py`, might as well bump `VERSION` file.
+3. Announce on our website
+4. Add release for citation on Zenodo, add new ID to docs
+5. Add tagged version of docs on ReadTheDocs (should happen automatically)
+6. HBP internal admin
+
+  - [Plus](https://plus.humanbrainproject.eu/components/2691/)
+  - [TC Wiki](https://wiki.ebrains.eu/bin/view/Collabs/technical-coordination/EBRAINS%20components/Arbor/)
+  - [KG](https://kg.ebrains.eu/search/instances/Software/80d205a9-ffb9-4afe-90b8-2f12819950ec) - [Update howto](https://github.com/bweyers/HBPVisCatalogue/wiki/How-to-start-software-meta-data-curation%3F#update-curated-software).
+    - Supported file formats (ie [ContentTypes](https://humanbrainproject.github.io/openMINDS/v3/core/v4/data/contentType.html)), [details](https://github.com/HumanBrainProject/openMINDS_core/tree/v3/instances/data/contentTypes)
+  - Send an update to the folk in charge of HBP Twitter if we want to shout about it
+7. FZJ admin
+
+  - https://juser.fz-juelich.de/submit
+
 .. _GH tags: https://github.com/arbor-sim/arbor/tags
 .. _AUTOMATED: https://github.com/arbor-sim/arbor/blob/master/.github/workflows/ebrains.yml 

python/example/example_requirements.txt

 # pip requirements file
 LFPykit>=0.3
+pandas
+seaborn

python/example/single_cell_nml.py

@@ -74,7 +74,7 @@ decor.discretization(policy)
 # Combine morphology with region and locset definitions to make a cable cell.
 cell = arbor.cable_cell(morpho, labels, decor)
 
-print(cell.locations('axon_end'))
+print(cell.locations('"axon_end"'))
 
 # Make single cell model.
 m = arbor.single_cell_model(cell)

scripts/run_python_examples.sh

@@ -12,14 +12,19 @@ PREFIX=${1:-}
 
 $PREFIX python -m pip install -r python/example/example_requirements.txt
 
+$PREFIX python python/example/brunel.py -n 400 -m 100 -e 20 -p 0.1 -w 1.2 -d 1 -g 0.5 -l 5 -t 100 -s 1 -G 50 -S 123
+# $PREFIX python python/dynamic-catalogue.py # arbor-build-catalog is a test already
+$PREFIX python python/example/gap_junctions.py
+# $PREFIX python python/example/network_ring_mpi_plot.py # requires MPI
+# $PREFIX python python/example/network_ring_mpi.py # requires MPI
 $PREFIX python python/example/network_ring.py
+$PREFIX python python/example/single_cell_cable.py
+$PREFIX python python/example/single_cell_detailed_recipe.py python/example/single_cell_detailed.swc
+$PREFIX python python/example/single_cell_detailed.py python/example/single_cell_detailed.swc
+$PREFIX python python/example/single_cell_extracellular_potentials.py python/example/single_cell_detailed.swc
 $PREFIX python python/example/single_cell_model.py
+$PREFIX python python/example/single_cell_nml.py python/example/morph.nml
 $PREFIX python python/example/single_cell_recipe.py
 $PREFIX python python/example/single_cell_stdp.py
-$PREFIX python python/example/brunel.py -n 400 -m 100 -e 20 -p 0.1 -w 1.2 -d 1 -g 0.5 -l 5 -t 100 -s 1 -G 50 -S 123
 $PREFIX python python/example/single_cell_swc.py python/example/single_cell_detailed.swc
-$PREFIX python python/example/single_cell_detailed.py python/example/single_cell_detailed.swc
-$PREFIX python python/example/single_cell_detailed_recipe.py python/example/single_cell_detailed.swc
-$PREFIX python python/example/single_cell_extracellular_potentials.py python/example/single_cell_detailed.swc
-$PREFIX python python/example/single_cell_cable.py
 $PREFIX python python/example/two_cell_gap_junctions.py