Tests: moving to Docker compose

captain.yml

@@ -6,6 +6,4 @@ target_image:
   post:
     - echo "Finished building portal-backend image"
   test:
-    - ./tests/pre-test-run.sh
-    - ./tests/test-run.sh
-    - ./tests/post-test-run.sh
+    - ./tests/test.sh

src/main/java/eu/hbp/mip/controllers/VariablesApi.java

@@ -125,7 +125,7 @@ public class VariablesApi {
         LOGGER.info("Get groups and variables hierarchy");
 
         String sqlQuery = String.format(
-                "SELECT * FROM meta_variables where target_table='%s'", featuresMainTable.toUpperCase());
+                "SELECT * FROM meta_variables where upper(target_table)='%s'", featuresMainTable.toUpperCase());
         SqlRowSet data = metaJdbcTemplate.queryForRowSet(sqlQuery);
         data.next();
         String json = ((PGobject) data.getObject("hierarchy")).getValue();
@@ -138,7 +138,7 @@ public class VariablesApi {
 
     private List<String> loadVariables() {
         String sqlQuery = String.format(
-                "SELECT * FROM meta_variables where target_table='%s'", featuresMainTable.toUpperCase());
+                "SELECT * FROM meta_variables where upper(target_table)='%s'", featuresMainTable.toUpperCase());
         SqlRowSet data = metaJdbcTemplate.queryForRowSet(sqlQuery);
         data.next();
         String json = ((PGobject) data.getObject("hierarchy")).getValue();

tests/docker-compose.yml

+version: '2'
+services:
+
+  # Expose the database to the host
+  db:
+    image: postgres:9.6.5-alpine
+    ports:
+      - "5432:5432"
+    hostname: db
+    networks:
+      - portal-bridge
+    environment:
+      POSTGRES_PASSWORD: test
+
+  wait_dbs:
+    image: "waisbrot/wait"
+    restart: "no"
+    networks:
+      - portal-bridge
+    environment:
+      - TARGETS=db:5432
+
+  create_dbs:
+    image: "hbpmip/create-databases:1.0.0"
+    restart: "no"
+    networks:
+      - portal-bridge
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_ADMIN_USER: postgres
+      DB_ADMIN_PASSWORD: test
+      DB1: meta
+      USER1: meta
+      PASSWORD1: metapwd
+      DB2: features
+      USER2: features
+      PASSWORD2: featurespwd
+      DB3: portal
+      USER3: portal
+      PASSWORD3: portalpwd
+    depends_on:
+      - db
+
+  meta_db_setup:
+    image: "hbpmip/sample-meta-db-setup:0.2.0"
+    container_name: "meta-db-setup"
+    restart: "no"
+    networks:
+      - portal-bridge
+    environment:
+      FLYWAY_DBMS: postgresql
+      FLYWAY_HOST: db
+      FLYWAY_PORT: 5432
+      FLYWAY_DATABASE_NAME: meta
+      FLYWAY_USER: postgres
+      FLYWAY_PASSWORD: test
+    depends_on:
+      - db
+
+  sample_db_setup:
+    image: "hbpmip/sample-data-db-setup:0.3.2"
+    container_name: "data-db-setup"
+    restart: "no"
+    networks:
+      - portal-bridge
+    environment:
+      FLYWAY_DBMS: postgresql
+      FLYWAY_HOST: db
+      FLYWAY_PORT: 5432
+      FLYWAY_DATABASE_NAME: features
+      FLYWAY_USER: postgres
+      FLYWAY_PASSWORD: test
+    depends_on:
+      - db
+
+  # Use latest version as this is the image under test
+  portal_backend:
+    image: "hbpmip/portal-backend"
+    container_name: "backend-test"
+    restart: "no"
+    ports:
+      - "65440:8080"
+    networks:
+      - portal-bridge
+    environment:
+      PORTAL_DB_URL: jdbc:postgresql://db:5432/portal
+      PORTAL_DB_SERVER: db:5432
+      PORTAL_DB_USER: portal
+      PORTAL_DB_PASSWORD: portalpwd
+      META_DB_URL: jdbc:postgresql://db:5432/meta
+      META_DB_SERVER: db:5432
+      META_DB_USER: meta
+      META_DB_PASSWORD: metapwd
+      FEATURES_DB_URL: jdbc:postgresql://db:5432/features
+      FEATURES_DB_SERVER: db:5432
+      FEATURES_DB_USER: features
+      FEATURES_DB_PASSWORD: featurespwd
+      FEATURES_DB_MAIN_TABLE: sample_data
+      CONTEXT_PATH: /services
+      AUTHENTICATION: 0
+    depends_on:
+      - db
+
+  wait_portal_backend:
+    image: "waisbrot/wait"
+    restart: "no"
+    networks:
+      - portal-bridge
+    environment:
+      - TARGETS=portal_backend:8080
+    depends_on:
+      - portal_backend
+
+networks:
+  portal-bridge:
+    external:
+      name: portal-bridge

tests/features-db/sql/create.sql

-SET datestyle to 'European';
-
-CREATE TABLE IF NOT EXISTS FEATURES
-(
-  tv1 char(256),
-  tv2 numeric,
-  tv3 numeric,
-
-  CONSTRAINT pk_features PRIMARY KEY (tv2)
-)
-WITH (
-  OIDS=FALSE
-);
-
-COPY FEATURES FROM '/docker-entrypoint-initdb.d/values.csv' CSV HEADER;

tests/features-db/sql/ipv6_listen.sh

-#!/bin/sh
-# This script will be executed when Postgres starts
-echo "host all all fe80::/10 md5" >> /var/lib/postgresql/data/pg_hba.conf

tests/meta-db/sql/create.sql

-SET datestyle to 'European';
-
-CREATE TABLE IF NOT EXISTS meta_variables (
-  ID serial NOT NULL PRIMARY KEY,
-  source varchar(256) UNIQUE NOT NULL,
-  hierarchy json NOT NULL,
-  target_table varchar(256) UNIQUE NOT NULL
-);
-INSERT INTO meta_variables (source, hierarchy, target_table) VALUES (
-  'features',
-  '
-  {
-    "code": "root",
-    "groups": [{
-        "code": "tg1",
-        "label": "Test Group 1",
-        "groups": [{
-            "code": "tg3",
-            "label": "Test Group 3",
-            "variables": [{
-                "code": "tv1",
-                "label": "Test Variable 1",
-                "type": "text"
-            }]
-        }]
-    }, {
-        "code": "tg2",
-        "label": "Test Group 2",
-        "groups": [{
-            "code": "tg4",
-            "label": "Test Group 4",
-            "variables": [{
-                "code": "tv2",
-                "label": "Test Variable 2",
-                "type": "integer"
-            }, {
-                "code": "tv3",
-                "label": "Test Variable 3",
-                "type": "real"
-            }]
-        }]
-    }]
-  }
-  ',
-  'FEATURES'
-);

tests/meta-db/sql/ipv6_listen.sh

-#!/bin/sh
-# This script will be executed when Postgres starts
-echo "host all all fe80::/10 md5" >> /var/lib/postgresql/data/pg_hba.conf

tests/post-test-run.sh

-#!/usr/bin/env bash
-
-
-# Kill old containers
-
-if [ $(docker ps | grep backend-test | wc -l) -gt 0 ]; then
-  docker kill backend-test
-fi
-if [ $(docker ps | grep portal-db-test | wc -l) -gt 0 ]; then
-  docker kill portal-db-test
-fi
-if [ $(docker ps | grep features-db-test | wc -l) -gt 0 ]; then
-  docker kill features-db-test
-fi
-if [ $(docker ps | grep meta-db-test | wc -l) -gt 0 ]; then
-  docker kill meta-db-test
-fi
-
-
-# Remove old containers
-
-if [ $(docker ps -a | grep backend-test | wc -l) -gt 0 ]; then
-  docker rm -f backend-test
-fi
-if [ $(docker ps -a | grep portal-db-test | wc -l) -gt 0 ]; then
-  docker rm -f portal-db-test
-fi
-if [ $(docker ps -a | grep features-db-test | wc -l) -gt 0 ]; then
-  docker rm -f features-db-test
-fi
-if [ $(docker ps -a | grep meta-db-test | wc -l) -gt 0 ]; then
-  docker rm -f meta-db-test
-fi

tests/pre-test-run.sh

-#!/usr/bin/env bash
-
-
-# Kill old containers
-
-echo "Killing old containers..."
-if [ $(docker ps | grep backend-test | wc -l) -gt 0 ]; then
-  docker kill backend-test
-fi
-if [ $(docker ps | grep portal-db-test | wc -l) -gt 0 ]; then
-  docker kill portal-db-test
-fi
-if [ $(docker ps | grep features-db-test | wc -l) -gt 0 ]; then
-  docker kill features-db-test
-fi
-
-
-# Remove old containers
-
-echo "Removing old containers..."
-if [ $(docker ps -a | grep backend-test | wc -l) -gt 0 ]; then
-  docker rm -f backend-test
-fi
-if [ $(docker ps -a | grep portal-db-test | wc -l) -gt 0 ]; then
-  docker rm -f portal-db-test
-fi
-if [ $(docker ps -a | grep features-db-test | wc -l) -gt 0 ]; then
-  docker rm -f features-db-test
-fi
-
-
-# Run databases containers
-
-echo "Running databases containers..."
-docker run --name features-db-test -p 65432:5432 -v $(pwd)/tests/features-db/sql:/docker-entrypoint-initdb.d/ -e POSTGRES_USER=postgres -d postgres:9.6.5-alpine
-docker run --name meta-db-test -p 65433:5432 -v $(pwd)/tests/meta-db/sql:/docker-entrypoint-initdb.d/ -e POSTGRES_USER=postgres -d postgres:9.6.5-alpine
-docker run --name portal-db-test -p 65434:5432 -e POSTGRES_USER=postgres -d postgres:9.6.5-alpine
-
-
-# Get gateway IP
-
-echo "Searching gateway IP..."
-GATEWAY_IP=$(docker inspect features-db-test | grep \"Gateway\":\ \" | sed 's/.*Gateway\":\ \"\([^-]*\)\",/\1/' | head -n 1)
-echo "Gateway IP: $GATEWAY_IP"
-
-
-# Wait for databases to be ready
-
-echo "Waiting for features-db to start..."
-if [ "$CIRCLECI" = true ] ; then
-  sleep 10
-else
-  until [ $(docker exec features-db-test psql -U postgres -c "\q" 2>&1 | wc -l) -eq 0 ]; do
-      printf '.'
-      sleep 1
-  done
-fi
-echo ""
-
-echo "Waiting for meta-db to start..."
-if [ "$CIRCLECI" = true ] ; then
-  sleep 10
-else
-until [ $(docker exec meta-db-test psql -U postgres -c "\q" 2>&1 | wc -l) -eq 0 ]; do
-    printf '.'
-    sleep 1
-done
-fi
-echo ""
-
-echo "Waiting for portal-db to start..."
-if [ "$CIRCLECI" = true ] ; then
-  sleep 10
-else
-until [ $(docker exec portal-db-test psql -U postgres -c "\q" 2>&1 | wc -l) -eq 0 ]; do
-    printf '.'
-    sleep 1
-done
-fi
-echo ""
-
-
-# Run backend container
-
-echo "Running backend container..."
-docker run --name backend-test -p 65440:8080 \
--e "PORTAL_DB_URL=jdbc:postgresql://$GATEWAY_IP:65434/postgres" \
--e "PORTAL_DB_SERVER=$GATEWAY_IP:65434/postgres" \
--e "PORTAL_DB_USER=postgres" \
--e "META_DB_URL=jdbc:postgresql://$GATEWAY_IP:65433/postgres" \
--e "META_DB_SERVER=$GATEWAY_IP:65433/postgres" \
--e "META_DB_USER=postgres" \
--e "FEATURES_DB_URL=jdbc:postgresql://$GATEWAY_IP:65432/postgres" \
--e "FEATURES_DB_SERVER=$GATEWAY_IP:65432/postgres" \
--e "FEATURES_DB_USER=postgres" \
--e "CONTEXT_PATH=/services" \
--e "AUTHENTICATION=0" \
--d hbpmip/portal-backend:latest
-
-
-# Wait for backend to be ready
-
-echo "Waiting for backend to start..."
-until [ $(docker logs backend-test | grep "Started MIPApplication" | wc -l) -eq 1 ]; do
-    printf '.'
-    sleep 1
-done
-
-
-echo "DONE"

test.sh → tests/test.sh

 #!/usr/bin/env bash
+
 set -e
 
 get_script_dir () {
@@ -13,7 +14,7 @@ get_script_dir () {
      pwd
 }
 
-export WORKSPACE=$(get_script_dir)
+cd "$(get_script_dir)"
 
 if pgrep -lf sshuttle > /dev/null ; then
   echo "sshuttle detected. Please close this program as it messes with networking and prevents builds inside Docker to work"
@@ -21,11 +22,55 @@ if pgrep -lf sshuttle > /dev/null ; then
 fi
 
 if [[ $NO_SUDO || -n "$CIRCLECI" ]]; then
-  CAPTAIN="captain"
+  DOCKER="docker"
+  DOCKER_COMPOSE="docker-compose"
 elif groups $USER | grep &>/dev/null '\bdocker\b'; then
-  CAPTAIN="captain"
+  DOCKER="docker"
+  DOCKER_COMPOSE="docker-compose"
+else
+  DOCKER="sudo docker"
+  DOCKER_COMPOSE="sudo docker-compose"
+fi
+
+function _cleanup() {
+  local error_code="$?"
+  echo "Stopping the containers..."
+  $DOCKER_COMPOSE stop | true
+  $DOCKER_COMPOSE down | true
+  $DOCKER_COMPOSE rm -f > /dev/null 2> /dev/null | true
+  exit $error_code
+}
+trap _cleanup EXIT INT TERM
+
+if [ $($DOCKER network ls | grep -c 'portal-bridge') -lt 1 ]; then
+  echo "Create portal-bridge network..."
+  $DOCKER network create portal-bridge
 else
-  CAPTAIN="sudo captain"
+  echo "Found portal-bridge network !"
 fi
 
-$CAPTAIN test
+$DOCKER_COMPOSE up -d --remove-orphans db
+$DOCKER_COMPOSE run wait_dbs
+$DOCKER_COMPOSE run create_dbs
+
+echo "Migrate metadata database..."
+$DOCKER_COMPOSE run meta_db_setup
+
+echo "Migrate features database..."
+$DOCKER_COMPOSE run sample_db_setup
+
+echo
+echo "Start Portal Backend"
+$DOCKER_COMPOSE up -d portal_backend
+
+echo
+echo "Wait for Portal Backend to start..."
+$DOCKER_COMPOSE run wait_portal_backend
+
+echo
+echo "Test idempotence"
+#$DOCKER_COMPOSE run data_db_setup
+#$DOCKER_COMPOSE run data_db_check
+
+# Cleanup
+#_cleanup