Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
E
Exareme2
Manage
Activity
Members
Labels
Plan
Issues
0
Issue boards
Milestones
Wiki
Code
Merge requests
3
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
HBP Medical Informatics Platform
Exareme2
Commits
ab0ca468
Commit
ab0ca468
authored
2 years ago
by
kfilippopolitis
Committed by
ThanKarab
2 years ago
Browse files
Options
Downloads
Patches
Plain Diff
Renamed make-> convert to idempotent
parent
d92302a4
No related branches found
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
mipengine/node/monetdb_interface/monet_db_facade.py
+5
-10
5 additions, 10 deletions
mipengine/node/monetdb_interface/monet_db_facade.py
tests/standalone_tests/test_monetdb_interface_if_not_exists.py
+4
-4
4 additions, 4 deletions
.../standalone_tests/test_monetdb_interface_if_not_exists.py
with
9 additions
and
14 deletions
mipengine/node/monetdb_interface/monet_db_facade.py
+
5
−
10
View file @
ab0ca468
...
...
@@ -39,7 +39,7 @@ def db_execute_and_fetchall(query: str, parameters=None) -> List:
def
db_execute_query
(
query
:
str
,
parameters
=
None
):
query_execution_timeout
=
node_config
.
celery
.
tasks_timeout
query
=
make
_idempotent
(
query
)
query
=
convert_to
_idempotent
(
query
)
db_execution_dto
=
_DBExecutionDTO
(
query
=
query
,
parameters
=
parameters
,
timeout
=
query_execution_timeout
)
...
...
@@ -53,7 +53,7 @@ def db_execute_udf(query: str, parameters=None):
raise
ValueError
(
f
"
UDF execution query:
{
query
}
should contain only one query.
"
)
udf_execution_timeout
=
node_config
.
celery
.
run_udf_task_timeout
query
=
make
_udf_execution_idempotent
(
query
)
query
=
convert
_udf_execution_
query_to_
idempotent
(
query
)
db_execution_dto
=
_DBExecutionDTO
(
query
=
query
,
parameters
=
parameters
,
timeout
=
udf_execution_timeout
)
...
...
@@ -99,7 +99,7 @@ def _validate_exception_is_recoverable(exc):
"""
Check whether the query needs to be re-executed and return True or False accordingly.
"""
if
isinstance
(
exc
,
BrokenPipeError
)
or
isinstance
(
exc
,
ConnectionResetError
):
if
isinstance
(
exc
,
(
BrokenPipeError
,
ConnectionResetError
)
)
:
return
True
elif
isinstance
(
exc
,
DatabaseError
):
return
"
ValueError
"
not
in
str
(
exc
)
and
not
isinstance
(
exc
,
ProgrammingError
)
...
...
@@ -128,11 +128,6 @@ def _execute_queries_with_error_handling(func):
try
:
return
func
(
**
kwargs
)
except
Exception
as
exc
:
if
isinstance
(
exc
,
ProgrammingError
)
and
"
is already part of MERGE TABLE
"
in
str
(
exc
):
return
if
not
_validate_exception_is_recoverable
(
exc
):
logger
.
error
(
f
"
Error occurred: Exception type:
'
{
type
(
exc
)
}
'
and exception message:
'
{
exc
}
'"
...
...
@@ -165,7 +160,7 @@ def _execute_and_fetchall(db_execution_dto) -> List:
return
result
def
make
_udf_execution_idempotent
(
query
:
str
)
->
str
:
def
convert
_udf_execution_
query_to_
idempotent
(
query
:
str
)
->
str
:
def
extract_table_name
(
query
:
str
)
->
str
:
"""
Extracts the name of the table from an INSERT INTO statement.
...
...
@@ -191,7 +186,7 @@ def make_udf_execution_idempotent(query: str) -> str:
)
def
make
_idempotent
(
query
:
str
)
->
str
:
def
convert_to
_idempotent
(
query
:
str
)
->
str
:
"""
This function creates an idempotent query to protect from a potential edge case
where a table creation query is interrupted due to a UDF running and allocating memory.
...
...
This diff is collapsed.
Click to expand it.
tests/standalone_tests/test_monetdb_interface_if_not_exists.py
+
4
−
4
View file @
ab0ca468
import
pytest
from
mipengine.node.monetdb_interface.monet_db_facade
import
make
_idempotent
from
mipengine.node.monetdb_interface.monet_db_facade
import
convert_to
_idempotent
from
mipengine.node.monetdb_interface.monet_db_facade
import
(
make
_udf_execution_idempotent
,
convert
_udf_execution_
query_to_
idempotent
,
)
...
...
@@ -25,12 +25,12 @@ from mipengine.node.monetdb_interface.monet_db_facade import (
],
)
def
test_make_idempotent
(
query
,
expected_idempotent_query
):
assert
make
_idempotent
(
query
)
==
expected_idempotent_query
assert
convert_to
_idempotent
(
query
)
==
expected_idempotent_query
def
test_make_udf_execution_idempotent
():
assert
(
make
_udf_execution_idempotent
(
"
INSERT INTO my_tbl1 VALUES (1);
"
)
convert
_udf_execution_
query_to_
idempotent
(
"
INSERT INTO my_tbl1 VALUES (1);
"
)
==
"
INSERT INTO my_tbl1 VALUES (1)
\n
"
"
WHERE NOT EXISTS (SELECT * FROM my_tbl1);
"
)
This diff is collapsed.
Click to expand it.
Preview
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment