From df52ad005e7efd10aa1146ab87dc01a2526a2db7 Mon Sep 17 00:00:00 2001
From: akuesters <42005107+akuesters@users.noreply.github.com>
Date: Fri, 15 Mar 2019 11:15:32 +0100
Subject: [PATCH] Include Python build and unit tests in Travis CI (#720)
Python 3.6 is tested (serial and mpi), in particular:
- 3.6.3 on linux with a virtualenv,
- 3.6.5 as downgrade from 3.7 on osx as in https://stackoverflow.com/questions/52584907/how-to-downgrade-python-from-3-7-to-3-6?rq=1 (other solutions did not work)
Fixes #715
---
.travis.yml | 83 +++++++++++++++----
.../unit_distributed/test_contexts_arbmpi.py | 1 -
.../unit_distributed/test_contexts_mpi4py.py | 1 -
scripts/travis/build.sh | 37 +++++++--
4 files changed, 95 insertions(+), 27 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index e1500dea..252dcbf5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,7 @@
# GCC 6.4.0
# Clang 4.0
# Apple Clang 900.0.39.2
+# Python 3.6
#######################################################
language: cpp
@@ -11,46 +12,55 @@ matrix:
include:
########################## OS X #########################
## test gcc6 - single node/rank with threading backend ##
- - name: "osx, gcc, serial"
+ - name: "osx, gcc, serial, py"
os: osx
osx_image: xcode9.2
+ python: 3.6
env:
- MATRIX_EVAL="brew install gcc@6 && brew link --force --overwrite gcc@6 && CC=gcc-6 && CXX=g++-6"
- - BUILD_NAME=cthread WITH_DISTRIBUTED=serial
+ - BUILD_NAME=cthread-osx-gcc-py
+ - WITH_DISTRIBUTED=serial WITH_PYTHON=true PY=3
compiler: gcc-6
## test gcc6 - mpi with threading backend ##
- - name: "osx, gcc, mpi"
+ - name: "osx, gcc, mpi, py"
os: osx
osx_image: xcode9.2
+ python: 3.6
env:
- MATRIX_EVAL="brew install gcc@6 && brew link --force --overwrite gcc@6 && CC=gcc-6 && CXX=g++-6"
- - BUILD_NAME=mpi WITH_DISTRIBUTED=mpi
+ - BUILD_NAME=mpi-osx-gcc-py
+ - WITH_DISTRIBUTED=mpi WITH_PYTHON=true PY=3
compiler: gcc-6
## test clang9 - single node/rank with threading backend ##
- - name: "osx, apple clang, serial"
+ - name: "osx, apple clang, serial, py"
os: osx
osx_image: xcode9.2
+ python: 3.6
env:
- MATRIX_EVAL="CC=clang && CXX=clang++"
- - BUILD_NAME=cthread WITH_DISTRIBUTED=serial
+ - BUILD_NAME=cthread-osx-clang-py
+ - WITH_DISTRIBUTED=serial WITH_PYTHON=true PY=3
compiler: clang
## test clang9 - mpi with threading backend ##
- - name: "osx, apple clang, mpi"
+ - name: "osx, apple clang, mpi, py"
os: osx
osx_image: xcode9.2
+ python: 3.6
env:
- MATRIX_EVAL="CC=clang && CXX=clang++"
- - BUILD_NAME=mpi WITH_DISTRIBUTED=mpi
+ - BUILD_NAME=mpi-osx-clang
+ - WITH_DISTRIBUTED=mpi WITH_PYTHON=true PY=3
compiler: clang
######################### LINUX #########################
## test gcc6 - single node/rank with threading backend ##
- - name: "linux, gcc, serial"
+ - name: "linux, gcc, serial, py"
os: linux
dist: trusty
+ python: 3.6
addons:
apt:
sources:
@@ -61,13 +71,15 @@ matrix:
- libopenmpi-dev
env:
- MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
- - BUILD_NAME=cthread WITH_DISTRIBUTED=serial
+ - BUILD_NAME=cthread-linux-gcc-py
+ - WITH_DISTRIBUTED=serial WITH_PYTHON=true PY=3
compiler: gcc-6
## test gcc6 - mpi with threading backend ##
- - name: "linux, gcc, mpi"
+ - name: "linux, gcc, mpi, py"
os: linux
dist: trusty
+ python: 3.6
addons:
apt:
sources:
@@ -78,13 +90,15 @@ matrix:
- libopenmpi-dev
env:
- MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
- - BUILD_NAME=mpi WITH_DISTRIBUTED=mpi
+ - BUILD_NAME=mpi-linux-gcc-py
+ - WITH_DISTRIBUTED=mpi WITH_PYTHON=true PY=3
compiler: gcc-6
## test clang4 - single node/rank with threading backend ##
- - name: "linux, clang, serial"
+ - name: "linux, clang, serial, py"
os: linux
dist: trusty
+ python: 3.6
addons:
apt:
sources:
@@ -97,13 +111,15 @@ matrix:
- libopenmpi-dev
env:
- MATRIX_EVAL="CC=clang-4.0 && CXX=clang++-4.0"
- - BUILD_NAME=cthread WITH_DISTRIBUTED=serial
+ - BUILD_NAME=cthread-linux-clang-py
+ - WITH_DISTRIBUTED=serial WITH_PYTHON=true PY=3
compiler: clang-4.0
## test clang4 - mpi with threading backend ##
- - name: "linux, clang, mpi"
+ - name: "linux, clang, mpi, py"
os: linux
dist: trusty
+ python: 3.6
addons:
apt:
sources:
@@ -116,12 +132,43 @@ matrix:
- libopenmpi-dev
env:
- MATRIX_EVAL="CC=clang-4.0 && CXX=clang++-4.0"
- - BUILD_NAME=mpi WITH_DISTRIBUTED=mpi
+ - BUILD_NAME=mpi-linux-clang-py
+ - WITH_DISTRIBUTED=mpi WITH_PYTHON=true PY=3
compiler: clang-4.0
-before_script:
+before_install:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export HOMEBREW_NO_AUTO_UPDATE=1; brew cask uninstall --force oclint; fi
- - if [[ ( "$TRAVIS_OS_NAME" == "osx" ) && ( "$WITH_DISTRIBUTED" == "mpi" ) ]]; then brew install open-mpi; fi
+ - |
+ if [[ "$WITH_PYTHON" == "true" ]]; then
+ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+ source ~/virtualenv/python3.6/bin/activate
+ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+ brew unlink python
+ brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/e128fa1bce3377de32cbf11bd8e46f7334dfd7a6/Formula/python.rb
+ brew switch python 3.6.5
+ fi
+ fi
+
+install:
+ - |
+ if [[ "$WITH_PYTHON" == "true" ]]; then
+ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+ python$PY get-pip.py
+ pip --version
+ fi
+ - |
+ if [[ "$WITH_DISTRIBUTED" == "mpi" ]]; then
+ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+ brew install open-mpi;
+ if [[ "$WITH_PYTHON" == "true" ]]; then
+ pip$PY install mpi4py
+ fi
+ elif [[ ( "$TRAVIS_OS_NAME" == "linux" ) && ( "$WITH_PYTHON" == "true" ) ]]; then
+ pip$PY install mpi4py
+ fi
+ fi
+
+before_script:
- eval "${MATRIX_EVAL}"
script: source ./scripts/travis/build.sh
diff --git a/python/test/unit_distributed/test_contexts_arbmpi.py b/python/test/unit_distributed/test_contexts_arbmpi.py
index e455b8ed..7698f8b1 100644
--- a/python/test/unit_distributed/test_contexts_arbmpi.py
+++ b/python/test/unit_distributed/test_contexts_arbmpi.py
@@ -3,7 +3,6 @@
# test_contexts_arbmpi.py
import unittest
-from collections import namedtuple
import arbor as arb
diff --git a/python/test/unit_distributed/test_contexts_mpi4py.py b/python/test/unit_distributed/test_contexts_mpi4py.py
index 2f64dd8e..be1dc248 100644
--- a/python/test/unit_distributed/test_contexts_mpi4py.py
+++ b/python/test/unit_distributed/test_contexts_mpi4py.py
@@ -3,7 +3,6 @@
# test_contexts_mpi4py.py
import unittest
-from collections import namedtuple
import arbor as arb
diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh
index a0e406b9..f0e3e777 100755
--- a/scripts/travis/build.sh
+++ b/scripts/travis/build.sh
@@ -21,7 +21,7 @@ echo "cmake : ${cmake_version}"
echo "build path : ${build_path}"
echo "base path : ${base_path}"
-if [[ "${WITH_DISTRIBUTED}" = "mpi" ]]; then
+if [[ "${WITH_DISTRIBUTED}" == "mpi" ]]; then
echo "mpi : on"
export OMPI_CC=${CC}
export OMPI_CXX=${CXX}
@@ -33,7 +33,7 @@ if [[ "${WITH_DISTRIBUTED}" = "mpi" ]]; then
# --mca btl tcp,self for Open MPI to use the "tcp" and "self" Byte Transfer Layers for transporting MPI messages
# "self" to deliver messages to the same rank as the sender
# "tcp" sends messages across TCP-based networks (Transmission Control Protocol with Internet Protocol)
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+ if [[ "$TRAVIS_OS_NAME" = "osx" ]]; then
launch="${launch} --oversubscribe --mca btl tcp,self"
fi
WITH_MPI="ON"
@@ -43,6 +43,18 @@ else
WITH_MPI="OFF"
fi
+if [[ "${WITH_PYTHON}" == "true" ]]; then
+ echo "python : on"
+ ARB_WITH_PYTHON="ON"
+ export PYTHONPATH=$PYTHONPATH:${base_path}/${build_path}/lib
+ python_path=$base_path/python
+ echo "python src : ${python_path}"
+ echo "PYTHONPATH : ${PYTHONPATH}"
+else
+ echo "python : off"
+ ARB_WITH_PYTHON="OFF"
+fi
+
#
# make build path
#
@@ -54,24 +66,35 @@ cd $build_path
#
progress "Configuring with cmake"
-cmake_flags="-DARB_WITH_ASSERTIONS=on -DARB_WITH_MPI=${WITH_MPI} ${CXX_FLAGS}"
+cmake_flags="-DARB_WITH_ASSERTIONS=ON -DARB_WITH_MPI=${WITH_MPI} -DARB_WITH_PYTHON=${ARB_WITH_PYTHON} ${CXX_FLAGS}"
echo "cmake flags: ${cmake_flags}"
cmake .. ${cmake_flags} || error "unable to configure cmake"
export NMC_NUM_THREADS=2
-progress "Unit tests"
+progress "C++ unit tests"
make unit -j4 || error "building unit tests"
./bin/unit --gtest_color=no || error "running unit tests"
-progress "Distributed unit tests (local)"
+progress "C++ distributed unit tests (local)"
make unit-local -j4 || error "building local distributed unit tests"
./bin/unit-local || error "running local distributed unit tests"
-if [[ "${WITH_DISTRIBUTED}" = "mpi" ]]; then
- progress "Distributed unit tests (MPI)"
+if [[ "${WITH_DISTRIBUTED}" == "mpi" ]]; then
+ progress "C++ distributed unit tests (MPI)"
make unit-mpi -j4 || error "building MPI distributed unit tests"
${launch} ./bin/unit-mpi || error "running MPI distributed unit tests"
fi
+if [[ "${WITH_PYTHON}" == "true" ]]; then
+ progress "Building python module"
+ make pyarb -j4 || error "building pyarb"
+ progress "Python unit tests"
+ python$PY $python_path/test/unit/runner.py -v2 || error "running python unit tests (serial)"
+ if [[ "${WITH_DISTRIBUTED}" = "mpi" ]]; then
+ progress "Python distributed unit tests (MPI)"
+ ${launch} python$PY $python_path/test/unit_distributed/runner.py -v2 || error "running python distributed unit tests (MPI)"
+ fi
+fi
+
cd $base_path
--
GitLab