From 16362091634518cf51121e54a03ed5cb16189a51 Mon Sep 17 00:00:00 2001
From: Brent Huisman <brenthuisman@users.noreply.github.com>
Date: Thu, 2 Jun 2022 12:32:24 +0200
Subject: [PATCH] Have dependency version policy (#1865)
As per #1731, this PR adds a dependencies version policy to the docs. It also includes a file containing a list of all configure-time deps.
Obsoletes wiki page.
---
CMakeGraphVizOptions.cmake | 2 +-
doc/conf.py | 2 +-
doc/contrib/dependency-management.rst | 75 ++++++++++++++++++++++
doc/dependencies.csv | 30 +++++++++
doc/dev/index.rst | 2 +-
doc/index.rst | 1 +
doc/install/build_install.rst | 92 ++++++++++++---------------
spack/package.py | 2 +-
sup/README.md | 3 +
9 files changed, 153 insertions(+), 56 deletions(-)
create mode 100644 doc/contrib/dependency-management.rst
create mode 100644 doc/dependencies.csv
create mode 100644 sup/README.md
diff --git a/CMakeGraphVizOptions.cmake b/CMakeGraphVizOptions.cmake
index d121a609..33d774b0 100644
--- a/CMakeGraphVizOptions.cmake
+++ b/CMakeGraphVizOptions.cmake
@@ -1 +1 @@
-set(GRAPHVIZ_IGNORE_TARGETS "bench;brunel;drybench;dryrun;gap_junctions;generators;lfp;probe-demo;ring;single;unit")
+set(GRAPHVIZ_IGNORE_TARGETS "bench;brunel;drybench;dryrun;gap_junctions;generators;lfp;probe-demo;ring;single;unit;lmorpho")
diff --git a/doc/conf.py b/doc/conf.py
index 45f33b66..7f56382a 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -36,7 +36,7 @@ intersphinx_mapping = {
}
project = 'Arbor'
-copyright = '2017-2021, ETHZ & FZJ'
+copyright = '2017-2022, ETHZ & FZJ'
author = 'ETHZ & FZJ'
todo_include_todos = True
diff --git a/doc/contrib/dependency-management.rst b/doc/contrib/dependency-management.rst
new file mode 100644
index 00000000..e36ad2e2
--- /dev/null
+++ b/doc/contrib/dependency-management.rst
@@ -0,0 +1,75 @@
+.. _contribdepverman:
+
+Dependency management
+=====================
+
+Arbor relies on a (small) number of dependencies. We can distinguish three kinds of deps:
+
+0. Source management dependencies: Git.
+1. Build dependencies. E.g. CMake, compilers like GCC or CUDA.
+2. Linking dependencies. E.g. MPI, libxml2.
+3. Source dependencies. These are present as `git submodules <https://git-scm.com/docs/git-submodule>`_ or as copy in ``ext/``. Their use is optional: users who need integration with their package manager (e.g. apt, spack, yum) can link to those versions instead.
+
+Note that the actual dependencies of your build configuration may vary.
+
+In addition, ``spack/package.py`` contains a copy of the Spack package definition `upstream <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/arbor/package.py>`_. Here instructions for both in-repo and configure-time dependencies are defined.
+
+This document contains rules for when and how to update dependencies, and what to be mindful of when you do.
+
+List of dependencies
+--------------------
+
+A full list of dependencies is maintained at ``doc/dependencies.csv``:
+
+.. csv-table:: List of configure time dependencies
+ :file: ../dependencies.csv
+ :widths: 10, 20, 10, 70, 1
+ :header-rows: 1
+
+.. note::
+
+ CMake can generate an overview of the dependency tree. Run the following in the build dir:
+
+ .. code-block:: bash
+
+ cmake --graphviz=graphviz/arbor-dep-graph.dot . && dot graphviz/arbor-dep-graph.dot -T png -o graphviz/arbor-dep-graph.png
+
+ This plot can be tweaked with the ``CMakeGraphVizOptions.cmake`` file in the root of the project, which currently excludes tests, the ``lmorpho`` morphology generator and all C++ examples.
+
+User platforms
+--------------
+
+Although Arbor should in principle run everywhere modern compilers are found, a couple of user platforms
+are essential. These environments must be able to build Arbor without issue, if not we consider this a bug.
+Also, build instructions for each of them must be given in the documentation.
+
+* Ubuntu LTS-latest
+* Ubuntu LTS-latest-1
+* MacOS-latest
+* MacOS-latest-1
+* Cray programming environment on Piz Daint
+* Programming environment on Juwels Booster (**todo** CI at JSC)
+* Github Action venvs, see `list <https://github.com/actions/virtual-environments>`_.
+* Manylinux containers. For compatibility of manylinux tags, see `here <https://github.com/pypa/manylinux#readme>`_.
+
+Dependency update rules
+-----------------------
+
+#. ``doc/dependencies.csv``, git submodules and ``spack/package.py`` shall be in sync.
+#. Dependencies shall be set to a (commit hash corresponding to a) specific version tag. (All current dependencies use semver.)
+#. The version shall be compatible with the user platforms (see above).
+#. The version shall be compatible with the requirements in ``doc/dependencies.csv``.
+#. The version shall be the lowest possible. More recent versions of submodules are automatically tested through ``.github/workflows/check-submodules.yml``, to catch merge problems early.
+#. Moreover, dependencies shall not be updated past the most recent version of the dependency in Spack.
+
+ * This prevents Spack builds from pulling in ``master``, when a more recent version than available is required. `See here <https://spack.readthedocs.io/en/latest/packaging_guide.html#version-comparison>`_.
+ * This is a manual check, e.g. for pybind: `see pybind package.py <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/py-pybind11/package.py>`_
+#. Actually updating shall remain a manual process. Update may require nontrivial updates to Arbor, and updates to Spack upstream (e.g. make PR for pybind update).
+#. A dependency update shall have a separate PR, and such a PR updates a single dependency at a time, unless the dependency update requires other dependencies to be updated.
+#. This PR requires review by at least two reviewers.
+
+ * There appears to be no way to enforce that, unless we enforce it for all PRs.
+ * Optionally we could have a PR template auto-assigning to some or all of us, which means we'll at least be notified.
+#. We will try to keep compatible to a wide range in dependency versions.
+
+ * This includes making other components in `ext` git submodules, such that updates are more easily tracked.
diff --git a/doc/dependencies.csv b/doc/dependencies.csv
new file mode 100644
index 00000000..8cbc63c4
--- /dev/null
+++ b/doc/dependencies.csv
@@ -0,0 +1,30 @@
+Build option/target,Tool name,Minimum version,Notes,Update criteria
+--,CMake,3.18,,"* Is available as a module on HPC systems
+* When arbitrary changes in more recently released versions make supported"
+--,C++ compiler,,C++17 support. See below,
+--,GCC,8.4.0,"Use of ``std::fs`` is prohibited, despite being included in major compilers with C++17, on account of MacOS 10.14 not having the required syscall. We might lift this.","* it is supported by the minimum version of CUDA.
+* it is available either by default or can be installed using standard package manager on the supported Linux versions."
+--,Clang,8.0,Needs GCC 8 or later for standard library.,
+--,Apple Clang,9,Apple LLVM version 9.0.0 (clang-900.0.39.2),
+ARB_GPU,Hip Clang,ROCm 3.9,HIP support is currently experimental.,
+ARB_GPU,CUDA,10.0,,"* It is available on all of the target HPC systems (Piz Daint, Juwels Booster)"
+ARB_WITH_NEUROML,libxml2,,,
+ARB_WITH_MPI,MPI,,Many MPI implementations are supported.,
+"ARB_WITH_MPI, ARB_WITH_PYTHON ",mpi4py,3.1.0,Minimum version depends on compatiblity with your Python and MPI versions. 3.1.0 is the first release to support Python > 3.8.,
+html, .. literalinclude:: ../requirements.txt,,See ``doc/requirements.txt``,
+unit,googletest,,source copy ``test/gtest.*``,
+bench,Google-benchmark,,submodule ``ext/google-benchmark``,
+--,json,,source copy ``ext/json``,
+--,random123,,source copy ``ext/random123``,
+--,fmt,,submodule ``ext/fmt``,
+--,tinyopt,,source copy ``ext/tinyopt``,
+ARB_WITH_PYTHON,pybind11,,submodule ``python/pybind11``,
+ARB_WITH_PYTHON,Python,3.6,Python compatiblity is the range between the latest officially released point version and the minimum here specified.,"* it is not more advanced than the version specified by NEP 29.
+* it is available as a cray-python version on Piz Daint
+* it is available on labs.ebrains.eu
+
+ * ``python3`` = Python 3.8.10
+* Our (eventual) tutorial Jupyter Notebook env
+
+ * Google Colab has Python 3.7
+* it is available in the supported Linuxen"
diff --git a/doc/dev/index.rst b/doc/dev/index.rst
index dedd2bc7..674559d5 100644
--- a/doc/dev/index.rst
+++ b/doc/dev/index.rst
@@ -8,7 +8,7 @@ Here we document internal components of Arbor. These pages can be useful if you'
.. figure:: arbor-dep-graph.png
:align: center
- Arbor dependency graph, generated by ``cmake --graphviz=graphviz/arbor-dep-graph.dot .``
+ Arbor dependency graph. To see what dependencies Arbor has and how to generate this figure, see :ref:`contribdepverman`.
.. toctree::
:caption: Arbor Developers Guide:
diff --git a/doc/index.rst b/doc/index.rst
index 6c84aced..6c2f7a2b 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -156,6 +156,7 @@ A full list of our software attributions can be found `here <https://github.com/
contrib/example
contrib/test
contrib/release
+ contrib/dependency-management
.. meta::
:google-site-verification: KbkW8d9MLsBFZz8Ry0tfcQRkHsgxzkECCahcyRSjWDo
diff --git a/doc/install/build_install.rst b/doc/install/build_install.rst
index 2555b4ce..de6f6ede 100644
--- a/doc/install/build_install.rst
+++ b/doc/install/build_install.rst
@@ -11,26 +11,53 @@ Then we cover installing and running on `HPC clusters <cluster_>`_, followed by
.. note::
To get help in case of problems installing Arbor, please make an issue on the Arbor `Github issues <https://github.com/arbor-sim/arbor/issues>`_ page.
+Getting the code
+================
+
+The easiest way to acquire the latest version of Arbor is to check the code out from
+the `Github repository <https://github.com/arbor-sim/arbor>`_:
+
+.. code-block:: bash
+
+ git clone https://github.com/arbor-sim/arbor.git --recurse-submodules
+
+We recommend using a recursive checkout, because Arbor uses Git submodules for some
+of its library dependencies.
+The CMake configuration attempts to detect if a required submodule is available, and
+will print a helpful warning
+or error message if not, but it is up to the user to ensure that all required
+submodules are downloaded.
+
+The Git submodules can be updated, or initialized in a project that didn't use a
+recursive checkout:
+
+.. code-block:: bash
+
+ git submodule update --init --recursive
+
+You can also point your browser to Arbor's
+`Github page <https://github.com/arbor-sim/arbor>`_ and download a zip file.
+If you use the zip file, then don't forget to run Git submodule update manually.
+
.. _install_requirements:
Requirements
============
-Minimum requirements
---------------------
+Dependencies
+------------
+
+A full list of dependencies and minimum versions supported thereof is maintained in the file ``doc/dependencies.csv``, for completeness shown also in the following table.
-The non distributed (i.e. no MPI) version of Arbor can be compiled on Linux or OS X systems
-with very few tools.
+.. csv-table:: List of requirements
+ :file: ../dependencies.csv
+ :widths: 10, 20, 10, 70, 1
+ :header-rows: 1
-.. table:: Required Tools
+Minimum requirements
+--------------------
- =========== ============================================
- Tool Notes
- =========== ============================================
- Git To check out the code, minimum version 2.0.
- CMake To set up the build, minimum version 3.18.
- compiler A C++17 compiler. See `compilers <install-compilers_>`_.
- =========== ============================================
+Arbor can be configured with various features, which may introduce additional dependencies. The non distributed (i.e. no MPI) version of Arbor can be compiled on Linux or OS X systems with very few tools. In the table above the blank lines in the "Build option/target" column reflect the minimum you need, provided you have obtained a copy of Arbor which includes the git submodules.
.. _install-compilers:
@@ -38,18 +65,7 @@ Compilers
~~~~~~~~~
Arbor requires a C++ compiler that fully supports C++17.
-We recommend using GCC or Clang, for which Arbor has been tested and optimised.
-
-.. table:: Supported Compilers
-
- =========== ============ ============================================
- Compiler Min version Notes
- =========== ============ ============================================
- GCC 8.4.0
- Clang 8.0 Needs GCC 8 or later for standard library.
- Apple Clang 9 Apple LLVM version 9.0.0 (clang-900.0.39.2)
- Hip Clang ROCm 3.9 HIP support is currently experimental.
- =========== ============ ============================================
+We recommend using GCC or Clang, for which Arbor has been tested and optimised, see notes in the above table.
.. _note_CC:
@@ -161,34 +177,6 @@ and `nlohmann json <https://github.com/nlohmann/json>`_ from a copy in the check
It is also possible to select only one of the two libraries to be taken from the system or from Arbor.
-Getting the code
-================
-
-The easiest way to acquire the latest version of Arbor is to check the code out from
-the `Github repository <https://github.com/arbor-sim/arbor>`_:
-
-.. code-block:: bash
-
- git clone https://github.com/arbor-sim/arbor.git --recurse-submodules
-
-We recommend using a recursive checkout, because Arbor uses Git submodules for some
-of its library dependencies.
-The CMake configuration attempts to detect if a required submodule is available, and
-will print a helpful warning
-or error message if not, but it is up to the user to ensure that all required
-submodules are downloaded.
-
-The Git submodules can be updated, or initialized in a project that didn't use a
-recursive checkout:
-
-.. code-block:: bash
-
- git submodule update --init --recursive
-
-You can also point your browser to Arbor's
-`Github page <https://github.com/arbor-sim/arbor>`_ and download a zip file.
-If you use the zip file, then don't forget to run Git submodule update manually.
-
.. _building:
Building and installing Arbor
diff --git a/spack/package.py b/spack/package.py
index 23798c03..51877c76 100644
--- a/spack/package.py
+++ b/spack/package.py
@@ -27,7 +27,7 @@ class Arbor(CMakePackage, CudaPackage):
variant('python', default=True, description='Enable Python frontend support')
variant('vectorize', default=False, description='Enable vectorization of computational kernels')
- # https://docs.arbor-sim.org/en/latest/install/build_install.html?highlight=requirements#compilers
+ # https://docs.arbor-sim.org/en/latest/install/build_install.html#compilers
conflicts('%gcc@:8.3')
conflicts('%clang@:7')
# Cray compiler v9.2 and later is Clang-based.
diff --git a/sup/README.md b/sup/README.md
new file mode 100644
index 00000000..d61bd7bc
--- /dev/null
+++ b/sup/README.md
@@ -0,0 +1,3 @@
+The `sup` library are some little helpers used internally in Arbor tests and examples.
+
+They are not intended for distribution with or use in the Arbor library.
--
GitLab