From 30c06d2e0d526257f70ee9e8bcde3c1da2e3db6f Mon Sep 17 00:00:00 2001
From: kfilippopolitis <kostasfilippop@gmail.com>
Date: Fri, 4 Jun 2021 03:28:08 -0700
Subject: [PATCH] Mip engine integration first touce
---
docker/README.md | 2 +
docker/config/application.tmpl | 3 +
.../eu/hbp/mip/controllers/AlgorithmsAPI.java | 50 ++++++-
.../models/DTOs/MIPEngineAlgorithmDTO.java | 119 +++++++++++++++++
.../hbp/mip/services/ExperimentService.java | 123 ++++++++++++------
src/main/resources/application.yml | 2 +
6 files changed, 254 insertions(+), 45 deletions(-)
create mode 100644 src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java
diff --git a/docker/README.md b/docker/README.md
index 9636ff324..8693aad09 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -24,6 +24,8 @@ To use this image, you need a running instance of PostgreSQL and to configure th
### EXTERNAL SERVICES ###
+* MIPENGINE_URL: URL to MIPENGINE server. Default is "http://localhost:5000" .
+
* EXAREME_URL: URL to Exareme server. Default is "http://localhost:9090" .
* GALAXY_URL: URL to Workflow server. Default is "http://localhost:8090/" .
diff --git a/docker/config/application.tmpl b/docker/config/application.tmpl
index ae10744aa..0913c99e7 100644
--- a/docker/config/application.tmpl
+++ b/docker/config/application.tmpl
@@ -34,6 +34,9 @@ spring:
### EXTERNAL SERVICES ###
services:
+ mipengine:
+ algorithmsUrl: {{ default .Env.MIPENGINE_URL "http://localhost:5000" }}/algorithms
+
exareme:
queryExaremeUrl: {{ default .Env.EXAREME_URL "http://localhost:9090" }}/mining/query
algorithmsUrl: {{ default .Env.EXAREME_URL "http://localhost:9090" }}/mining/algorithms.json
diff --git a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java
index 48500bb10..1ca68f13b 100644
--- a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java
+++ b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java
@@ -9,6 +9,7 @@ import com.google.gson.reflect.TypeToken;
import eu.hbp.mip.controllers.galaxy.retrofit.RetroFitGalaxyClients;
import eu.hbp.mip.controllers.galaxy.retrofit.RetrofitClientInstance;
import eu.hbp.mip.models.DTOs.AlgorithmDTO;
+import eu.hbp.mip.models.DTOs.MIPEngineAlgorithmDTO;
import eu.hbp.mip.models.galaxy.WorkflowDTO;
import eu.hbp.mip.services.ActiveUserService;
import eu.hbp.mip.utils.CustomResourceLoader;
@@ -42,6 +43,9 @@ public class AlgorithmsAPI {
private final ActiveUserService activeUserService;
+ @Value("#{'${services.mipengine.algorithmsUrl}'}")
+ private String mipengineAlgorithmsUrl;
+
@Value("#{'${services.exareme.algorithmsUrl}'}")
private String exaremeAlgorithmsUrl;
@@ -64,8 +68,9 @@ public class AlgorithmsAPI {
public ResponseEntity<List<AlgorithmDTO>> getAlgorithms() {
Logger logger = new Logger(activeUserService.getActiveUser().getUsername(), "(GET) /algorithms");
- logger.LogUserAction("Executing...");
-
+// logger.LogUserAction("Executing...");
+ LinkedList<AlgorithmDTO> mipengineAlgorithms = getMIPEngineAlgorithms(logger);
+ logger.LogUserAction("Loaded " + mipengineAlgorithms.size() + " mipengine algorithms");
LinkedList<AlgorithmDTO> exaremeAlgorithms = getExaremeAlgorithms(logger);
logger.LogUserAction("Loaded " + exaremeAlgorithms.size() + " exareme algorithms");
LinkedList<AlgorithmDTO> galaxyAlgorithms = getGalaxyWorkflows(logger);
@@ -73,12 +78,17 @@ public class AlgorithmsAPI {
LinkedList<AlgorithmDTO> algorithms = new LinkedList<>();
if (exaremeAlgorithms != null) {
- algorithms.addAll(exaremeAlgorithms);
+// algorithms.addAll(exaremeAlgorithms);
} else {
logger.LogUserAction("Getting exareme algorithms failed and returned null");
}
+ if (mipengineAlgorithms != null) {
+ algorithms.addAll(mipengineAlgorithms);
+ } else {
+ logger.LogUserAction("Getting mipengine algorithms failed and returned null");
+ }
if (galaxyAlgorithms != null) {
- algorithms.addAll(galaxyAlgorithms);
+// algorithms.addAll(galaxyAlgorithms);
} else {
logger.LogUserAction("Getting galaxy workflows failed and returned null");
}
@@ -112,7 +122,6 @@ public class AlgorithmsAPI {
try {
StringBuilder response = new StringBuilder();
HTTPUtil.sendGet(exaremeAlgorithmsUrl, response);
-
algorithms = gson.fromJson(
response.toString(),
new TypeToken<LinkedList<AlgorithmDTO>>() {
@@ -127,6 +136,37 @@ public class AlgorithmsAPI {
return algorithms;
}
+ /**
+ * This method gets all the available mipengine algorithms and
+ *
+ * @return a list of AlgorithmDTOs or null if something fails
+ */
+ public LinkedList<AlgorithmDTO> getMIPEngineAlgorithms(Logger logger) {
+ LinkedList<MIPEngineAlgorithmDTO> mipEngineAlgorithms;
+ // Get MIPEngine algorithms
+ try {
+ StringBuilder response = new StringBuilder();
+ HTTPUtil.sendGet(mipengineAlgorithmsUrl, response);
+ logger.LogUserAction(response.toString());
+
+ mipEngineAlgorithms = gson.fromJson(
+ response.toString(),
+ new TypeToken<LinkedList<MIPEngineAlgorithmDTO>>() {
+ }.getType()
+ );
+ } catch (IOException e) {
+ logger.LogUserAction("An exception occurred: " + e.getMessage());
+ return null;
+ }
+
+
+ LinkedList<AlgorithmDTO> algorithms = new LinkedList<>();
+ mipEngineAlgorithms.forEach(mipEngineAlgorithm -> algorithms.add(mipEngineAlgorithm.convertToAlgorithmDTO()));
+
+ logger.LogUserAction("Completed, returned " + algorithms.size() + " algorithms.");
+ return algorithms;
+ }
+
/**
* This method gets all the available galaxy workflows, converts them into algorithms and
*
diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java
new file mode 100644
index 000000000..a250e4249
--- /dev/null
+++ b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java
@@ -0,0 +1,119 @@
+package eu.hbp.mip.models.DTOs;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.ArrayList;
+import java.util.Hashtable;
+import java.util.List;
+
+@Getter
+@Setter
+public class MIPEngineAlgorithmDTO {
+
+ @SerializedName("name")
+ private String name;
+
+ @SerializedName("desc")
+ private String desc;
+
+ @SerializedName("label")
+ private String label;
+
+ @SerializedName("type")
+ private String type;
+
+ @SerializedName("parameters")
+ private Object parameters;
+
+ @SerializedName("crossvalidation")
+ private String crossvalidation;
+
+ @SerializedName("inputdata")
+ private Hashtable<String, InputDataParamDTO> inputdata;
+
+ @Getter
+ @Setter
+ public static class InputDataParamDTO {
+
+ @SerializedName("stattypes")
+ private List<String> stattypes;
+
+ @SerializedName("label")
+ private String label;
+
+ @SerializedName("notblank")
+ private String notblank;
+
+ @SerializedName("enumslen")
+ private Integer enumslen;
+
+ @SerializedName("multiple")
+ private String multiple;
+
+ @SerializedName("types")
+ private List<String> types;
+
+ @SerializedName("desc")
+ private String desc;
+ public AlgorithmDTO.AlgorithmParamDTO convertToAlgorithmParamDTO(String name) throws Exception {
+ AlgorithmDTO.AlgorithmParamDTO algorithmParamDTO = new AlgorithmDTO.AlgorithmParamDTO();
+ algorithmParamDTO.setName(name);
+ algorithmParamDTO.setDesc(this.desc);
+ algorithmParamDTO.setLabel(this.label);
+ if(name.equals("datasets") || name.equals("filter") || name.equals("pathology")){
+ algorithmParamDTO.setType(name.equals("datasets")? "dataset":name);
+ algorithmParamDTO.setValueType(this.types.get(0));
+ }
+ else{
+ if(stattypes == null){
+ System.out.println("Dsafasfads------------------------->>>>>>>>>>>>>>>");
+ System.out.println(name);
+ }
+ algorithmParamDTO.setType("column");
+ algorithmParamDTO.setColumnValuesSQLType(String.join(", ", this.types));
+ algorithmParamDTO.setColumnValuesIsCategorical(getColumnValuesIsCategorical(this.stattypes));
+ }
+ algorithmParamDTO.setValue("");
+ algorithmParamDTO.setValueNotBlank(this.notblank);
+ algorithmParamDTO.setValueMultiple(this.multiple);
+ return algorithmParamDTO;
+ }
+ private String getColumnValuesIsCategorical(List<String> stattypes) throws Exception {
+
+ if (stattypes.contains("nominal") && stattypes.contains("numerical")){
+ return "";
+ }
+ else if (stattypes.contains("nominal")){
+ return "true";
+ }
+ else if (stattypes.contains("numerical")){
+ return "false";
+ }
+ else{
+ throw new Exception("Invalid stattypes");
+ }
+ }
+ }
+
+ public AlgorithmDTO convertToAlgorithmDTO()
+ {
+ AlgorithmDTO algorithmDTO = new AlgorithmDTO();
+ algorithmDTO.setName(this.name);
+ algorithmDTO.setLabel(this.label);
+ algorithmDTO.setDesc(this.desc);
+ algorithmDTO.setType("mipengine");
+ List<AlgorithmDTO.AlgorithmParamDTO> parameters = new ArrayList<>();
+ this.inputdata.forEach((name, inputDataParamDTO) -> {
+ try {
+ AlgorithmDTO.AlgorithmParamDTO parameter = inputDataParamDTO.convertToAlgorithmParamDTO(name);
+ parameters.add(parameter);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ });
+ algorithmDTO.setParameters(parameters);
+ return algorithmDTO;
+ }
+}
diff --git a/src/main/java/eu/hbp/mip/services/ExperimentService.java b/src/main/java/eu/hbp/mip/services/ExperimentService.java
index 45345c8a5..a8b176d76 100644
--- a/src/main/java/eu/hbp/mip/services/ExperimentService.java
+++ b/src/main/java/eu/hbp/mip/services/ExperimentService.java
@@ -50,6 +50,9 @@ public class ExperimentService {
@Value("#{'${services.exareme.queryExaremeUrl}'}")
private String queryExaremeUrl;
+ @Value("#{'${services.mipengine.algorithmsUrl}'}")
+ private String mipengineAlgorithmsUrl;
+
@Value("#{'${services.galaxy.galaxyUrl}'}")
private String galaxyUrl;
@@ -194,7 +197,7 @@ public class ExperimentService {
return runGalaxyWorkflow(experimentDTO, logger);
} else {
logger.LogUserAction("Algorithm runs on Exareme.");
- return createExaremeExperiment(experimentDTO, logger);
+ return createExperiment(experimentDTO, logger);
}
}
@@ -219,13 +222,6 @@ public class ExperimentService {
throw new BadRequestException("You can not run workflow algorithms transiently.");
}
- // Get the parameters
- List<AlgorithmDTO.AlgorithmParamDTO> algorithmParameters
- = experimentDTO.getAlgorithm().getParameters();
-
- // Get the type and name of algorithm
- String algorithmName = experimentDTO.getAlgorithm().getName();
-
algorithmParametersLogging(experimentDTO, logger);
if (authenticationIsEnabled) {
@@ -233,19 +229,15 @@ public class ExperimentService {
ClaimUtils.validateAccessRightsOnDatasets(authentication, experimentDatasets, logger);
}
- String body = gson.toJson(algorithmParameters);
- String url = queryExaremeUrl + "/" + algorithmName;
- logger.LogUserAction("url: " + url + ", body: " + body);
-
- logger.LogUserAction("Completed, returning: " + experimentDTO.toString());
+ logger.LogUserAction("Completed, returning: " + experimentDTO);
// Results are stored in the experiment object
- ExaremeResult exaremeResult = runExaremeExperiment(url, body, experimentDTO);
+ ExperimentResult experimentResult = runExperiment(experimentDTO, logger);
- logger.LogUserAction("Experiment with uuid: " + experimentDTO.getUuid() + "gave response code: " + exaremeResult.getCode() + " and result: " + exaremeResult.getResults());
+ logger.LogUserAction("Experiment with uuid: " + experimentDTO.getUuid() + "gave response code: " + experimentResult.getCode() + " and result: " + experimentResult.getResults());
- experimentDTO.setResult((exaremeResult.getCode() >= 400) ? null : exaremeResult.getResults());
- experimentDTO.setStatus((exaremeResult.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success);
+ experimentDTO.setResult((experimentResult.getCode() >= 400) ? null : experimentResult.getResults());
+ experimentDTO.setStatus((experimentResult.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success);
return experimentDTO;
}
@@ -514,15 +506,52 @@ public class ExperimentService {
return JsonConverters.convertObjectToJsonString(finalJsonObject);
}
+ /**
+ * The runExperiment will run the experiment to exareme or MIPEngine.
+ *
+ * @param experimentDTO is the request with the experiment information
+ * @return the result of experiment as well as the http status that was retrieved
+ */
+ public ExperimentResult runExperiment(ExperimentDTO experimentDTO, Logger logger) {
+
+ // Algorithm type
+ String algorithmType = experimentDTO.getAlgorithm().getType();
+
+ // Run the 1st algorithm from the list
+ String algorithmName = experimentDTO.getAlgorithm().getName();
+
+ // Get the parameters
+ List<AlgorithmDTO.AlgorithmParamDTO> algorithmParameters
+ = experimentDTO.getAlgorithm().getParameters();
+
+ String body = gson.toJson(algorithmParameters);
+
+ logger.LogUserAction("Completed, returning: " + experimentDTO);
+
+
+ // Run with the appropriate engine
+ if (algorithmType.equals("mipengine")) {
+ String url = mipengineAlgorithmsUrl + "/" + algorithmName;
+ logger.LogUserAction("url: " + url + ", body: " + body);
+ logger.LogUserAction("Algorithm runs on MIPEngine.");
+ return runMIPEngineExperiment(url, body);
+ } else {
+ String url = queryExaremeUrl + "/" + algorithmName;
+ logger.LogUserAction("url: " + url + ", body: " + body);
+ logger.LogUserAction("Algorithm runs on Exareme.");
+ return runExaremeExperiment(url, body);
+ }
+ }
+
+
/**
* The runExaremeExperiment will run to exareme the experiment
*
* @param url is the url that contain the results of the experiment
* @param body is the parameters of the algorithm
- * @param experimentDTO is the experiment information to be executed in the exareme
* @return the result of exareme as well as the http status that was retrieved
*/
- public ExaremeResult runExaremeExperiment(String url, String body, ExperimentDTO experimentDTO) {
+ public ExperimentResult runExaremeExperiment(String url, String body) {
StringBuilder results = new StringBuilder();
int code;
@@ -536,7 +565,32 @@ public class ExperimentService {
ExperimentDTO experimentDTOWithOnlyResult = JsonConverters.convertJsonStringToObject(String.valueOf(results), ExperimentDTO.class);
List<Object> resultDTOS = experimentDTOWithOnlyResult.getResult();
- return new ExaremeResult(code, resultDTOS);
+ return new ExperimentResult(code, resultDTOS);
+ }
+
+ /**
+ * The runExaremeExperiment will run to exareme the experiment
+ *
+ * @param url is the url that contain the results of the experiment
+ * @param body is the parameters of the algorithm
+ * @return the result of exareme as well as the http status that was retrieved
+ */
+ public ExperimentResult runMIPEngineExperiment(String url, String body) {
+
+ StringBuilder results = new StringBuilder();
+ int code;
+ try {
+ code = HTTPUtil.sendPost(url, body, results);
+ } catch (Exception e) {
+ throw new InternalServerError("Error occurred : " + e.getMessage());
+ }
+ System.out.println("----------------------------------->>>>>>>>>>>>>>>>>>>>>>");
+ System.out.println(results);
+ // Results are stored in the experiment object
+ ExperimentDTO experimentDTOWithOnlyResult = JsonConverters.convertJsonStringToObject(String.valueOf(results), ExperimentDTO.class);
+ List<Object> resultDTOS = experimentDTOWithOnlyResult.getResult();
+
+ return new ExperimentResult(code, resultDTOS);
}
/* -------------------------------------- EXAREME CALLS ---------------------------------------------------------*/
@@ -548,27 +602,15 @@ public class ExperimentService {
* @param logger contains username and the endpoint.
* @return the experiment information that was retrieved from exareme
*/
- public ExperimentDTO createExaremeExperiment(ExperimentDTO experimentDTO, Logger logger) {
+ public ExperimentDTO createExperiment(ExperimentDTO experimentDTO, Logger logger) {
logger.LogUserAction("Running the algorithm...");
ExperimentDAO experimentDAO = createExperimentInTheDatabase(experimentDTO, logger);
logger.LogUserAction("Created experiment with uuid :" + experimentDAO.getUuid());
- // Run the 1st algorithm from the list
- String algorithmName = experimentDTO.getAlgorithm().getName();
-
- // Get the parameters
- List<AlgorithmDTO.AlgorithmParamDTO> algorithmParameters
- = experimentDTO.getAlgorithm().getParameters();
-
- String body = gson.toJson(algorithmParameters);
- String url = queryExaremeUrl + "/" + algorithmName;
- logger.LogUserAction("url: " + url + ", body: " + body);
-
- logger.LogUserAction("Completed, returning: " + experimentDTO.toString());
- logger.LogUserAction("Starting exareme execution thread");
+ logger.LogUserAction("Starting execution in thread");
ExperimentDTO finalExperimentDTO = experimentDTO;
new Thread(() -> {
@@ -576,13 +618,14 @@ public class ExperimentService {
Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Thread named :" + Thread.currentThread().getName() + " with id :" + Thread.currentThread().getId() + " started!");
try {
+
// Results are stored in the experiment object
- ExaremeResult exaremeResult = runExaremeExperiment(url, body, finalExperimentDTO);
+ ExperimentResult experimentResult = runExperiment(finalExperimentDTO, logger);
- Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Experiment with uuid: " + experimentDAO.getUuid() + "gave response code: " + exaremeResult.getCode() + " and result: " + exaremeResult.getResults());
+ Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Experiment with uuid: " + experimentDAO.getUuid() + "gave response code: " + experimentResult.getCode() + " and result: " + experimentResult.getResults());
- experimentDAO.setResult((exaremeResult.getCode() >= 400) ? null : JsonConverters.convertObjectToJsonString(exaremeResult.getResults()));
- experimentDAO.setStatus((exaremeResult.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success);
+ experimentDAO.setResult((experimentResult.getCode() >= 400) ? null : JsonConverters.convertObjectToJsonString(experimentResult.getResults()));
+ experimentDAO.setStatus((experimentResult.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success);
} catch (Exception e) {
Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "There was an exception: " + e.getMessage());
@@ -988,11 +1031,11 @@ public class ExperimentService {
return returnError;
}
- static class ExaremeResult {
+ static class ExperimentResult {
private final int code;
private final List<Object> results;
- public ExaremeResult(int code, List<Object> results) {
+ public ExperimentResult(int code, List<Object> results) {
this.code = code;
this.results = results;
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 345fc4bfa..d8d542b6b 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -34,6 +34,8 @@ spring:
### EXTERNAL SERVICES ###
services:
+ mipengine:
+ algorithmsUrl: "http://127.0.0.1:5000/algorithms"
exareme:
queryExaremeUrl: "http://127.0.0.1:9090/mining/query"
algorithmsUrl: "http://127.0.0.1:9090/mining/algorithms.json"
--
GitLab