diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 95e25a4cc45bfd628f2cfb53343f37bd368b74e3..0782acd7afd0e23cd323a2054a457623dca4877c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,41 +37,43 @@ jobs: if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - setup_cache_monetdb: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Load MONETDB cached image - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache/monetdb - key: ${{ runner.os }}-buildx-monetdb-${{hashFiles( 'monetdb/**' )}} - restore-keys: | - ${{ runner.os }}-buildx-monetdb- - - - name: Build MONETDB docker image - uses: docker/build-push-action@v2 - with: - context: . - file: monetdb/Dockerfile - push: false - load: true - tags: madgik/mipenginedb:latest - cache-from: type=local,src=/tmp/.buildx-cache/monetdb - cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move Docker images cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache +# Caching monetdb doesn't work because "apt-get update" in the Dockerfile messes up the caching, +# if it's removed then this could be added again. +# setup_cache_monetdb: +# runs-on: ubuntu-latest +# steps: +# - name: Check out repository +# uses: actions/checkout@v2 +# +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v1 +# +# - name: Load MONETDB cached image +# uses: actions/cache@v2 +# with: +# path: /tmp/.buildx-cache/monetdb +# key: ${{ runner.os }}-buildx-monetdb-${{hashFiles( 'monetdb/**' )}} +# restore-keys: | +# ${{ runner.os }}-buildx-monetdb- +# +# - name: Build MONETDB docker image +# uses: docker/build-push-action@v2 +# with: +# context: . +# file: monetdb/Dockerfile +# push: false +# load: true +# tags: madgik/mipenginedb:latest +# cache-from: type=local,src=/tmp/.buildx-cache/monetdb +# cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb +# +# # Temp fix +# # https://github.com/docker/build-push-action/issues/252 +# # https://github.com/moby/buildkit/issues/1896 +# - name: Move Docker images cache +# run: | +# rm -rf /tmp/.buildx-cache +# mv /tmp/.buildx-cache-new /tmp/.buildx-cache setup_cache_rabbitmq: @@ -153,7 +155,7 @@ jobs: ./coverage.xml:coverage.py integration_tests: - needs: [setup_cache_poetry, setup_cache_monetdb, setup_cache_rabbitmq] + needs: [setup_cache_poetry, setup_cache_rabbitmq] runs-on: ubuntu-latest steps: - name: Check out repository @@ -239,33 +241,19 @@ jobs: - name: Deploy all services run: poetry run inv deploy --no-install-dep --no-start-all -# - name: Load data into DBs -# run: poetry run inv load-data - - # Temp fix due to MONETDB bug with memory leak - - name: Load data into LOCALNODE1 - run: | - poetry run inv load-data --port 50001 - docker stop monetdb-localnode1 - docker start monetdb-localnode1 - - - name: Load data into LOCALNODE2 - run: | - poetry run inv load-data --port 50002 - docker stop monetdb-localnode2 - docker start monetdb-localnode2 + - name: Load data into DBs + run: poetry run inv load-data - name: Start services after data loading run: | poetry run inv start-node --all poetry run inv start-controller --detached - # End of temp fix - name: Run integration tests run: poetry run pytest tests/integration_tests e2e_tests: - needs: [setup_cache_poetry, setup_cache_monetdb, setup_cache_rabbitmq] + needs: [setup_cache_poetry, setup_cache_rabbitmq] runs-on: ubuntu-latest steps: - name: Check out repository @@ -388,7 +376,7 @@ jobs: with: time: '10s' - - name: Load data into DBs + - name: Load data into MONETDB run: poetry run inv load-data --port 50000 - name: Backup MONETDB data @@ -401,6 +389,9 @@ jobs: sudo cp -r /opt/monetdb/. /opt/monetdb1 sudo cp -r /opt/monetdb/. /opt/monetdb2 + - name: Stop MONETDB container + run: docker stop monetdb && docker rm monetdb + - name: Create k8s Kind Cluster uses: helm/kind-action@v1.2.0 with: @@ -436,7 +427,7 @@ jobs: - name: Wait for MIPENGINE services to start uses: jakejarvis/wait-action@master with: - time: '60s' + time: '120s' - name: Run e2e tests run: poetry run pytest tests/e2e_tests diff --git a/mipengine/node/monetdb_interface/monet_db_connection.py b/mipengine/node/monetdb_interface/monet_db_connection.py index a9ea06a5641aac2011a40d5ae84cbba767755c73..2e9b9b6312bac4241c1320a8286e65c5235315b8 100644 --- a/mipengine/node/monetdb_interface/monet_db_connection.py +++ b/mipengine/node/monetdb_interface/monet_db_connection.py @@ -3,6 +3,12 @@ from typing import List import pymonetdb + +from celery.utils.log import get_task_logger + +logging = get_task_logger(__name__) + + from mipengine.node import config as node_config BROKEN_PIPE_MAX_ATTEMPTS = 50 @@ -79,6 +85,9 @@ class MonetDB(metaclass=Singleton): 'many' option to provide the functionality of executemany, all results will be fetched. 'parameters' option to provide the functionality of bind-parameters. """ + logging.info( + f"Query: {query} \n, parameters: {str(parameters)}\n, many: {many}" + ) with self.cursor() as cur: cur.executemany(query, parameters) if many else cur.execute( @@ -97,6 +106,10 @@ class MonetDB(metaclass=Singleton): 'many' option to provide the functionality of executemany. 'parameters' option to provide the functionality of bind-parameters. """ + logging.info( + f"Query: {query} \n, parameters: {str(parameters)}\n, many: {many}" + ) + for _ in range(OCC_MAX_ATTEMPTS): with self.cursor() as cur: try: diff --git a/monetdb/Dockerfile b/monetdb/Dockerfile index c25649fbd7e0c91b52e26f059c1966408511de54..432234a54536cdd7cca04c1d1d7eb29c5fbd5d40 100755 --- a/monetdb/Dockerfile +++ b/monetdb/Dockerfile @@ -31,15 +31,15 @@ RUN pip3 install numpy ####################################################### # Download monetdb source files ####################################################### -RUN wget --output-document=/home/MonetDB-11.39.15.tar.bz2 --no-check-certificate https://www.monetdb.org/downloads/sources/Oct2020-SP4/MonetDB-11.39.15.tar.bz2 -RUN tar -xf /home/MonetDB-11.39.15.tar.bz2 -C /home/ +RUN wget --output-document=/home/MonetDB-11.41.11.tar.bz2 --no-check-certificate https://www.monetdb.org/downloads/sources/Jul2021-SP1/MonetDB-11.41.11.tar.bz2 +RUN tar -xf /home/MonetDB-11.41.11.tar.bz2 -C /home/ ####################################################### # Install monetdb ####################################################### RUN mkdir /home/monetdb-build WORKDIR /home/monetdb-build -RUN cmake -DCMAKE_BUILD_TYPE=Release -DASSERT=ON -DSTRICT=ON -DCMAKE_INSTALL_PREFIX=/usr/local/bin/monetdb /home/MonetDB-11.39.15 +RUN cmake -DCMAKE_BUILD_TYPE=Release -DASSERT=ON -DSTRICT=ON -DCMAKE_INSTALL_PREFIX=/usr/local/bin/monetdb /home/MonetDB-11.41.11 RUN cmake --build . RUN cmake --build . --target install ENV PATH="/usr/local/bin/monetdb/bin:$PATH" @@ -49,8 +49,8 @@ EXPOSE 50000 ####################################################### # Installation clean up ####################################################### -RUN rm /home/MonetDB-11.39.15.tar.bz2 -RUN rm -rf /home/MonetDB-11.39.15/ +RUN rm /home/MonetDB-11.41.11.tar.bz2 +RUN rm -rf /home/MonetDB-11.41.11/ RUN rm -rf /home/monetdb-build ####################################################### diff --git a/tests/e2e_tests/kind_configuration/kind_cluster.yaml b/tests/e2e_tests/kind_configuration/kind_cluster.yaml index 9e80c68daf9a0800cda277669b23a4beb6757dbf..ab278bd761aecaf6e7ce23f083ab49dd80432cb0 100644 --- a/tests/e2e_tests/kind_configuration/kind_cluster.yaml +++ b/tests/e2e_tests/kind_configuration/kind_cluster.yaml @@ -10,6 +10,8 @@ nodes: containerPath: /etc/hostname - hostPath: ./tests/demo_data containerPath: /opt/data + - hostPath: /opt/monetdb + containerPath: /opt/monetdb - role: worker extraMounts: - hostPath: ./tests/e2e_tests/kind_configuration/hostname_worker1