diff --git a/tasks.py b/tasks.py index 955eb9043123f82bc3da8f9d6a27b3c2d653ce4c..1efff4ab92e5e0283760755f7f12d0f908c50f07 100644 --- a/tasks.py +++ b/tasks.py @@ -63,6 +63,7 @@ from invoke import task from termcolor import colored import tests +from mipengine.udfgen import udfio PROJECT_ROOT = Path(__file__).parent DEPLOYMENT_CONFIG_FILE = PROJECT_ROOT / ".deployment.toml" @@ -237,6 +238,8 @@ def create_monetdb(c, node, image=None, log_level=None): get_docker_image(c, image) + udfio_full_path = path.abspath(udfio.__file__) + node_ids = node for node_id in node_ids: container_name = f"monetdb-{node_id}" @@ -250,7 +253,9 @@ def create_monetdb(c, node, image=None, log_level=None): f"Starting container {container_name} on ports {container_ports}...", Level.HEADER, ) - cmd = f"""docker run -d -P -p {container_ports} -e LOG_LEVEL={log_level} --name {container_name} {image}""" + # A volume is used to pass the udfio inside the monetdb container. + # This is done so that we don't need to rebuild every time the udfio.py file is changed. + cmd = f"""docker run -d -P -p {container_ports} -e LOG_LEVEL={log_level} -v {udfio_full_path}:/home/udflib/udfio.py --name {container_name} {image}""" run(c, cmd) @@ -665,6 +670,34 @@ def kill_all_flowers(c): message(f"No flower container to remove", level=Level.HEADER) +@task(iterable=["db"]) +def reload_udfio(c, db): + """ + Used for reloading the udfio module inside the monetdb containers. + + :param db: The names of the monetdb containers. + """ + dbs = db + for db in dbs: + sql_reload_query = """ +CREATE OR REPLACE FUNCTION +reload_udfio() +RETURNS +INT +LANGUAGE PYTHON +{ + import udfio + import importlib + importlib.reload(udfio) + return 0 +}; + +SELECT reload_udfio(); + """ + command = f'docker exec -t {db} mclient db --statement "{sql_reload_query}"' + run(c, command) + + def run(c, cmd, attach_=False, wait=True, warn=False, raise_error=False, show_ok=True): if attach_: c.run(cmd, pty=True) diff --git a/tests/standalone_tests/conftest.py b/tests/standalone_tests/conftest.py index 0641e72ecb8a45133940d01e1100d83f5f0e5e3c..788a38b9c9bfcd35ddae70799f932be1eb4ab837 100644 --- a/tests/standalone_tests/conftest.py +++ b/tests/standalone_tests/conftest.py @@ -10,6 +10,7 @@ import sqlalchemy as sql import toml from mipengine.controller.node_tasks_handler_celery import NodeTasksHandlerCelery +from mipengine.udfgen import udfio ALGORITHM_FOLDERS_ENV_VARIABLE_VALUE = "./mipengine/algorithms,./tests/algorithms" TESTING_RABBITMQ_CONT_IMAGE = "madgik/mipengine_rabbitmq:latest" @@ -80,10 +81,14 @@ def _create_monetdb_container(cont_name, cont_port): try: container = client.containers.get(cont_name) except docker.errors.NotFound: + udfio_full_path = path.abspath(udfio.__file__) + # A volume is used to pass the udfio inside the monetdb container. + # This is done so that we don't need to rebuild every time the udfio.py file is changed. container = client.containers.run( TESTING_MONETDB_CONT_IMAGE, detach=True, ports={"50000/tcp": cont_port}, + volumes=[f"{udfio_full_path}:/home/udflib/udfio.py"], name=cont_name, publish_all_ports=True, )