diff --git a/docker/README.md b/docker/README.md index 9636ff324e43b29c67fa367165370bbbf5c3a5f7..8693aad09f765b12de66b77c37f700563fa4eec7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -24,6 +24,8 @@ To use this image, you need a running instance of PostgreSQL and to configure th ### EXTERNAL SERVICES ### +* MIPENGINE_URL: URL to MIPENGINE server. Default is "http://localhost:5000" . + * EXAREME_URL: URL to Exareme server. Default is "http://localhost:9090" . * GALAXY_URL: URL to Workflow server. Default is "http://localhost:8090/" . diff --git a/docker/config/application.tmpl b/docker/config/application.tmpl index ae10744aa2424e7421da83da173d6e662608e843..a8bfdf9ff3d49b00668dc9b3e8b581dd054180a6 100644 --- a/docker/config/application.tmpl +++ b/docker/config/application.tmpl @@ -34,6 +34,9 @@ spring: ### EXTERNAL SERVICES ### services: + mipengine: + algorithmsUrl: {{ .Env.MIPENGINE_URL}}/algorithms + exareme: queryExaremeUrl: {{ default .Env.EXAREME_URL "http://localhost:9090" }}/mining/query algorithmsUrl: {{ default .Env.EXAREME_URL "http://localhost:9090" }}/mining/algorithms.json diff --git a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java index 48500bb10470059f1fe7d37079dfb391ef395b31..a15fd37cdfa9ac00e69687e1a419f5359522edd6 100644 --- a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java +++ b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java @@ -8,7 +8,8 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import eu.hbp.mip.controllers.galaxy.retrofit.RetroFitGalaxyClients; import eu.hbp.mip.controllers.galaxy.retrofit.RetrofitClientInstance; -import eu.hbp.mip.models.DTOs.AlgorithmDTO; +import eu.hbp.mip.models.DTOs.ExaremeAlgorithmDTO; +import eu.hbp.mip.models.DTOs.MIPEngineAlgorithmDTO; import eu.hbp.mip.models.galaxy.WorkflowDTO; import eu.hbp.mip.services.ActiveUserService; import eu.hbp.mip.utils.CustomResourceLoader; @@ -26,8 +27,8 @@ import retrofit2.Call; import retrofit2.Response; import java.io.IOException; +import java.net.ConnectException; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import static eu.hbp.mip.utils.InputStreamConverter.convertInputStreamToString; @@ -42,6 +43,9 @@ public class AlgorithmsAPI { private final ActiveUserService activeUserService; + @Value("#{'${services.mipengine.algorithmsUrl}'}") + private String mipengineAlgorithmsUrl; + @Value("#{'${services.exareme.algorithmsUrl}'}") private String exaremeAlgorithmsUrl; @@ -61,26 +65,32 @@ public class AlgorithmsAPI { @ApiOperation(value = "List all algorithms", response = String.class) @RequestMapping(method = RequestMethod.GET) - public ResponseEntity<List<AlgorithmDTO>> getAlgorithms() { + public ResponseEntity<List<ExaremeAlgorithmDTO>> getAlgorithms() { Logger logger = new Logger(activeUserService.getActiveUser().getUsername(), "(GET) /algorithms"); logger.LogUserAction("Executing..."); + ArrayList<ExaremeAlgorithmDTO> mipengineAlgorithms = getMIPEngineAlgorithms(logger); + ArrayList<ExaremeAlgorithmDTO> exaremeAlgorithms = getExaremeAlgorithms(logger); + ArrayList<ExaremeAlgorithmDTO> galaxyAlgorithms = getGalaxyWorkflows(logger); - LinkedList<AlgorithmDTO> exaremeAlgorithms = getExaremeAlgorithms(logger); - logger.LogUserAction("Loaded " + exaremeAlgorithms.size() + " exareme algorithms"); - LinkedList<AlgorithmDTO> galaxyAlgorithms = getGalaxyWorkflows(logger); - logger.LogUserAction("Loaded " + galaxyAlgorithms.size() + " galaxy algorithms"); - - LinkedList<AlgorithmDTO> algorithms = new LinkedList<>(); + ArrayList<ExaremeAlgorithmDTO> algorithms = new ArrayList<>(); if (exaremeAlgorithms != null) { algorithms.addAll(exaremeAlgorithms); + logger.LogUserAction("Loaded " + exaremeAlgorithms.size() + " exareme algorithms"); + } else { + logger.LogUserAction("Fetching exareme algorithms failed"); + } + if (mipengineAlgorithms != null) { + algorithms.addAll(mipengineAlgorithms); + logger.LogUserAction("Loaded " + mipengineAlgorithms.size() + " mipengine algorithms"); } else { - logger.LogUserAction("Getting exareme algorithms failed and returned null"); + logger.LogUserAction("Fetching mipengine algorithms failed"); } if (galaxyAlgorithms != null) { algorithms.addAll(galaxyAlgorithms); + logger.LogUserAction("Loaded " + galaxyAlgorithms.size() + " galaxy algorithms"); } else { - logger.LogUserAction("Getting galaxy workflows failed and returned null"); + logger.LogUserAction("Fetching galaxy workflows failed"); } List<String> disabledAlgorithms = new ArrayList<>(); @@ -91,8 +101,8 @@ public class AlgorithmsAPI { } // Remove any disabled algorithm - LinkedList<AlgorithmDTO> allowedAlgorithms = new LinkedList<>(); - for (AlgorithmDTO algorithm : algorithms) { + ArrayList<ExaremeAlgorithmDTO> allowedAlgorithms = new ArrayList<>(); + for (ExaremeAlgorithmDTO algorithm : algorithms) { if (!disabledAlgorithms.contains(algorithm.getName())) { allowedAlgorithms.add(algorithm); } @@ -106,18 +116,20 @@ public class AlgorithmsAPI { * * @return a list of AlgorithmDTOs or null if something fails */ - public LinkedList<AlgorithmDTO> getExaremeAlgorithms(Logger logger) { - LinkedList<AlgorithmDTO> algorithms; + public ArrayList<ExaremeAlgorithmDTO> getExaremeAlgorithms(Logger logger) { + ArrayList<ExaremeAlgorithmDTO> algorithms; // Get exareme algorithms try { StringBuilder response = new StringBuilder(); HTTPUtil.sendGet(exaremeAlgorithmsUrl, response); - algorithms = gson.fromJson( response.toString(), - new TypeToken<LinkedList<AlgorithmDTO>>() { + new TypeToken<ArrayList<ExaremeAlgorithmDTO>>() { }.getType() ); + } catch (ConnectException e) { + logger.LogUserAction("An exception occurred: " + e.getMessage()); + return null; } catch (IOException e) { logger.LogUserAction("An exception occurred: " + e.getMessage()); return null; @@ -127,12 +139,45 @@ public class AlgorithmsAPI { return algorithms; } + /** + * This method gets all the available mipengine algorithms and + * + * @return a list of AlgorithmDTOs or null if something fails + */ + public ArrayList<ExaremeAlgorithmDTO> getMIPEngineAlgorithms(Logger logger) { + ArrayList<MIPEngineAlgorithmDTO> mipEngineAlgorithms; + // Get MIPEngine algorithms + try { + StringBuilder response = new StringBuilder(); + HTTPUtil.sendGet(mipengineAlgorithmsUrl, response); + logger.LogUserAction(response.toString()); + + mipEngineAlgorithms = gson.fromJson( + response.toString(), + new TypeToken<ArrayList<MIPEngineAlgorithmDTO>>() { + }.getType() + ); + } catch (ConnectException e) { + logger.LogUserAction("An exception occurred: " + e.getMessage()); + return null; + } catch (IOException e) { + logger.LogUserAction("An exception occurred: " + e.getMessage()); + return null; + } + + ArrayList<ExaremeAlgorithmDTO> algorithms = new ArrayList<>(); + mipEngineAlgorithms.forEach(mipEngineAlgorithm -> algorithms.add(new ExaremeAlgorithmDTO(mipEngineAlgorithm))); + + logger.LogUserAction("Completed, returned " + algorithms.size() + " algorithms."); + return algorithms; + } + /** * This method gets all the available galaxy workflows, converts them into algorithms and * * @return a list of AlgorithmDTOs or null if something fails */ - public LinkedList<AlgorithmDTO> getGalaxyWorkflows(Logger logger) { + public ArrayList<ExaremeAlgorithmDTO> getGalaxyWorkflows(Logger logger) { List<Workflow> workflowList; try { // Get all the workflows with the galaxy client @@ -146,7 +191,7 @@ public class AlgorithmsAPI { } // Get the workflow details with the custom client to receive them as a WorkflowDTO - List<WorkflowDTO> workflows = new LinkedList<>(); + List<WorkflowDTO> workflows = new ArrayList<>(); // Create the request client RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); for (Workflow workflow : workflowList) { @@ -171,7 +216,7 @@ public class AlgorithmsAPI { logger.LogUserAction("Workflows fetched: " + workflows.size()); // Convert the workflows to algorithms - LinkedList<AlgorithmDTO> algorithms = new LinkedList<>(); + ArrayList<ExaremeAlgorithmDTO> algorithms = new ArrayList<>(); for (WorkflowDTO workflow : workflows) { logger.LogUserAction("Converting workflow: " + workflow); @@ -202,4 +247,4 @@ public class AlgorithmsAPI { }.getType() ); } -} +} \ No newline at end of file diff --git a/src/main/java/eu/hbp/mip/models/DAOs/ExperimentDAO.java b/src/main/java/eu/hbp/mip/models/DAOs/ExperimentDAO.java index 585bd8bd6694275d341fe1aea19f82f3105d97ea..8eb343c9cc658b97e8f0c1a1b6a7be19f09c9c94 100644 --- a/src/main/java/eu/hbp/mip/models/DAOs/ExperimentDAO.java +++ b/src/main/java/eu/hbp/mip/models/DAOs/ExperimentDAO.java @@ -3,12 +3,9 @@ package eu.hbp.mip.models.DAOs; import com.fasterxml.jackson.annotation.JsonInclude; import com.google.gson.Gson; import com.google.gson.annotations.Expose; -import eu.hbp.mip.models.DTOs.AlgorithmDTO; -import eu.hbp.mip.models.DTOs.ExperimentDTO; -import eu.hbp.mip.utils.JsonConverters; import io.swagger.annotations.ApiModel; -import lombok.Getter; -import lombok.Setter; +import lombok.AllArgsConstructor; +import lombok.Data; import javax.persistence.*; import java.util.*; @@ -17,8 +14,8 @@ import java.util.*; * Created by habfast on 21/04/16. */ @Entity -@Getter -@Setter +@Data +@AllArgsConstructor @Table(name = "`experiment`") @ApiModel @JsonInclude(JsonInclude.Include.NON_NULL) @@ -95,22 +92,4 @@ public class ExperimentDAO { */ } - public ExperimentDTO convertToDTO(boolean includeResult) - { - ExperimentDTO experimentDTO = new ExperimentDTO(); - experimentDTO.setAlgorithm(JsonConverters.convertJsonStringToObject(this.algorithm, AlgorithmDTO.class)); - experimentDTO.setCreated(this.created); - experimentDTO.setUpdated(this.updated); - experimentDTO.setFinished(this.finished); - experimentDTO.setCreatedBy(this.createdBy.getUsername()); - experimentDTO.setName(this.name); - if(includeResult){ - experimentDTO.setResult(JsonConverters.convertJsonStringToObject(String.valueOf(this.result), new ArrayList<>().getClass())); - } - experimentDTO.setStatus(this.status); - experimentDTO.setShared(this.shared); - experimentDTO.setUuid(this.uuid); - experimentDTO.setViewed(this.viewed); - return experimentDTO; - } } diff --git a/src/main/java/eu/hbp/mip/models/DAOs/UserDAO.java b/src/main/java/eu/hbp/mip/models/DAOs/UserDAO.java index 9279b3793c44c449302c083b823da674d7e4d46f..3bb43f3eec865cc92cd1be4e97fdf660d90e3089 100644 --- a/src/main/java/eu/hbp/mip/models/DAOs/UserDAO.java +++ b/src/main/java/eu/hbp/mip/models/DAOs/UserDAO.java @@ -7,16 +7,16 @@ package eu.hbp.mip.models.DAOs; import com.fasterxml.jackson.annotation.JsonInclude; import com.google.gson.annotations.Expose; import io.swagger.annotations.ApiModel; -import lombok.Getter; -import lombok.Setter; +import lombok.AllArgsConstructor; +import lombok.Data; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; @Entity -@Getter -@Setter +@Data +@AllArgsConstructor @Table(name = "`user`") @ApiModel @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/src/main/java/eu/hbp/mip/models/DTOs/AlgorithmDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/AlgorithmDTO.java deleted file mode 100644 index 52db979d5ddd7b043572be8fc9255c45fa0e9c98..0000000000000000000000000000000000000000 --- a/src/main/java/eu/hbp/mip/models/DTOs/AlgorithmDTO.java +++ /dev/null @@ -1,73 +0,0 @@ -package eu.hbp.mip.models.DTOs; - -import com.google.gson.annotations.SerializedName; -import lombok.Getter; -import lombok.Setter; - -import java.util.List; - -@Getter -@Setter -public class AlgorithmDTO { - - @SerializedName("name") - private String name; - - @SerializedName("desc") - private String desc; - - @SerializedName("label") - private String label; - - @SerializedName("type") - private String type; - - @SerializedName("parameters") - private List<AlgorithmParamDTO> parameters; - - @Getter - @Setter - public static class AlgorithmParamDTO { - @SerializedName("name") - private String name; - - @SerializedName("desc") - private String desc; - - @SerializedName("label") - private String label; - - @SerializedName("type") - private String type; - - @SerializedName("columnValuesSQLType") - private String columnValuesSQLType; - - @SerializedName("columnValuesIsCategorical") - private String columnValuesIsCategorical; - - @SerializedName("value") - private String value; - - @SerializedName("defaultValue") - private String defaultValue; - - @SerializedName("valueType") - private String valueType; - - @SerializedName("valueNotBlank") - private String valueNotBlank; - - @SerializedName("valueMultiple") - private String valueMultiple; - - @SerializedName("valueMin") - private String valueMin; - - @SerializedName("valueMax") - private String valueMax; - - @SerializedName("valueEnumerations") - private List<String> valueEnumerations; - } -} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..584ffd424f4c691a168cd26b3b8b1fa2566658bf --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmDTO.java @@ -0,0 +1,67 @@ +package eu.hbp.mip.models.DTOs; + +import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.*; + +@Data +@AllArgsConstructor +public class ExaremeAlgorithmDTO { + + @SerializedName("name") + private String name; + + @SerializedName("desc") + private String desc; + + @SerializedName("label") + private String label; + + @SerializedName("type") + private String type; + + @SerializedName("parameters") + private List<ExaremeAlgorithmRequestParamDTO> parameters; + + public ExaremeAlgorithmDTO() + { + + } + + public ExaremeAlgorithmDTO(MIPEngineAlgorithmDTO mipEngineAlgorithm ) + { + this.name = mipEngineAlgorithm.getName().toUpperCase(); + this.label = mipEngineAlgorithm.getLabel(); + this.desc = mipEngineAlgorithm.getDesc(); + this.type = "mipengine"; + List<ExaremeAlgorithmRequestParamDTO> parameters = new ArrayList<>(); + parameters.add(new ExaremeAlgorithmRequestParamDTO("x", mipEngineAlgorithm.getInputdata().getX())); + parameters.add(new ExaremeAlgorithmRequestParamDTO("y", mipEngineAlgorithm.getInputdata().getY())); + parameters.add(new ExaremeAlgorithmRequestParamDTO("pathology", mipEngineAlgorithm.getInputdata().getPathology())); + parameters.add(new ExaremeAlgorithmRequestParamDTO("dataset", mipEngineAlgorithm.getInputdata().getDatasets())); + parameters.add(new ExaremeAlgorithmRequestParamDTO("filter", mipEngineAlgorithm.getInputdata().getFilter())); + mipEngineAlgorithm.getParameters().forEach((name, parameterDTO) -> { + ExaremeAlgorithmRequestParamDTO parameter = new ExaremeAlgorithmRequestParamDTO(name, parameterDTO); + parameters.add(parameter); + }); + this.setParameters(parameters); + } + @Data + @AllArgsConstructor + static class Rule + { + @SerializedName("id") + private String id; + + @SerializedName("type") + private String type; + + @SerializedName("operator") + private String operator; + + @SerializedName("value") + private Object value; + } +} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmRequestParamDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmRequestParamDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..4109c5e5916b65c0b1af9f0eaed04aa96249c640 --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmRequestParamDTO.java @@ -0,0 +1,106 @@ +package eu.hbp.mip.models.DTOs; + +import com.google.gson.annotations.SerializedName; +import eu.hbp.mip.utils.Exceptions.InternalServerError; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.Arrays; +import java.util.List; + +//The request of an exareme algorithm is a list of ExaremeAlgorithmRequestParamDTOs. +@Data +@AllArgsConstructor +public class ExaremeAlgorithmRequestParamDTO { + @SerializedName("name") + private String name; + + @SerializedName("desc") + private String desc; + + @SerializedName("label") + private String label; + + @SerializedName("type") + private String type; + + @SerializedName("columnValuesSQLType") + private String columnValuesSQLType; + + @SerializedName("columnValuesIsCategorical") + private String columnValuesIsCategorical; + + @SerializedName("value") + private String value; + + @SerializedName("defaultValue") + private String defaultValue; + + @SerializedName("valueType") + private String valueType; + + @SerializedName("valueNotBlank") + private String valueNotBlank; + + @SerializedName("valueMultiple") + private String valueMultiple; + + @SerializedName("valueMin") + private String valueMin; + + @SerializedName("valueMax") + private String valueMax; + + @SerializedName("valueEnumerations") + private List<String> valueEnumerations; + public ExaremeAlgorithmRequestParamDTO (){} + + public ExaremeAlgorithmRequestParamDTO (String name, MIPEngineAlgorithmDTO.MIPEngineAlgorithmParameterDTO parameter){ + this.name = name; + this.desc = parameter.getDesc(); + this.valueType = parameter.getType(); + this.type = "other"; + this.defaultValue = parameter.getDefault_value(); + this.valueNotBlank = parameter.getNotblank(); + this.label = parameter.getLabel(); + this.valueEnumerations = parameter.getEnums(); + this.valueMultiple = parameter.getMultiple(); + this.valueMax = parameter.getMin(); + this.valueMin = parameter.getMax(); + } + + public ExaremeAlgorithmRequestParamDTO (String name, MIPEngineAlgorithmDTO.MIPEngineAlgorithmInputDataDetailDTO inputDataDetail){ + this.name = name; + this.desc = inputDataDetail.getDesc(); + this.value = ""; + this.valueNotBlank = inputDataDetail.getNotblank(); + this.valueMultiple = inputDataDetail.getMultiple(); + String[] hidden = {"x","y","dataset", "filter","pathology","centers","formula"}; + this.label = (Arrays.asList(hidden).contains(this.name) ? this.name : inputDataDetail.getLabel()); + if(name.equals("dataset") || name.equals("filter") || name.equals("pathology")){ + this.valueType = inputDataDetail.getTypes().get(0); + this.type = this.name; + } + else{ + this.type = "column"; + this.columnValuesSQLType = String.join(", ", inputDataDetail.getTypes()); + this.columnValuesIsCategorical = getColumnValuesIsCategorical(inputDataDetail.getStattypes()); + } + } + + private String getColumnValuesIsCategorical(List<String> stattypes){ + + if (stattypes.contains("nominal") && stattypes.contains("numerical")){ + return ""; + } + else if (stattypes.contains("nominal")){ + return "true"; + } + else if (stattypes.contains("numerical")){ + return "false"; + } + else{ + throw new InternalServerError("Invalid stattypes"); + } + } +} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmResultDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmResultDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..88b5fb627da14791c2e772fca65d06cef3a4f5e2 --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/ExaremeAlgorithmResultDTO.java @@ -0,0 +1,15 @@ +package eu.hbp.mip.models.DTOs; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class ExaremeAlgorithmResultDTO { + private int code; + private List<Object> result; +} + + diff --git a/src/main/java/eu/hbp/mip/models/DTOs/ExperimentDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/ExperimentDTO.java index e64c6f75e7d7a4d7aafa8ba93d38f9d3edf6bf19..0dd28c1d6c4fde45164d9077aa5c9be3c90f57fb 100644 --- a/src/main/java/eu/hbp/mip/models/DTOs/ExperimentDTO.java +++ b/src/main/java/eu/hbp/mip/models/DTOs/ExperimentDTO.java @@ -2,15 +2,17 @@ package eu.hbp.mip.models.DTOs; import com.fasterxml.jackson.annotation.JsonInclude; import eu.hbp.mip.models.DAOs.ExperimentDAO; -import lombok.Getter; -import lombok.Setter; +import eu.hbp.mip.utils.JsonConverters; +import lombok.AllArgsConstructor; +import lombok.Data; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.UUID; -@Getter -@Setter +@Data +@AllArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) public class ExperimentDTO { @@ -22,10 +24,31 @@ public class ExperimentDTO { private Date finished; private Boolean shared; private Boolean viewed; + // Result is a list of objects because there is a limitation that java has in types. + // Exareme has result in the type of a List<HashMap<String, Object>> + // Galaxy has result in the type of a List<HashMap<String, List<Object>>> + //And there is no generic type that describes either an object or a list of objects private List<Object> result; private ExperimentDAO.Status status; - private AlgorithmDTO algorithm; + private ExaremeAlgorithmDTO algorithm; - public ExperimentDTO() { + public ExperimentDTO(){ + + } + public ExperimentDTO(boolean includeResult, ExperimentDAO experimentDAO) + { + this.algorithm = JsonConverters.convertJsonStringToObject(experimentDAO.getAlgorithm(), ExaremeAlgorithmDTO.class); + this.created = experimentDAO.getCreated(); + this.updated = experimentDAO.getUpdated(); + this.finished = experimentDAO.getFinished(); + this.createdBy = experimentDAO.getCreatedBy().getUsername(); + this.name = experimentDAO.getName(); + if(includeResult){ + this.result = JsonConverters.convertJsonStringToObject(String.valueOf(experimentDAO.getResult()), new ArrayList<>().getClass()); + } + this.status = experimentDAO.getStatus(); + this.uuid = experimentDAO.getUuid(); + this.shared = experimentDAO.isShared(); + this.viewed = experimentDAO.isViewed(); } } diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..5edfbfe77827179234caf339f3f627235f72e967 --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmDTO.java @@ -0,0 +1,111 @@ +package eu.hbp.mip.models.DTOs; + +import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.Hashtable; +import java.util.List; + +@Data +@AllArgsConstructor +public class MIPEngineAlgorithmDTO { + + @SerializedName("name") + private String name; + + @SerializedName("desc") + private String desc; + + @SerializedName("label") + private String label; + + @SerializedName("type") + private String type; + + @SerializedName("parameters") + private Hashtable<String, MIPEngineAlgorithmParameterDTO> parameters; + + @SerializedName("crossvalidation") + private String crossvalidation; + + @SerializedName("inputdata") + private MIPEngineAlgorithmInputdataDTO inputdata; + + @Data + @AllArgsConstructor + public static class MIPEngineAlgorithmParameterDTO { + + @SerializedName("label") + private String label; + + @SerializedName("notblank") + private String notblank; + + @SerializedName("multiple") + private String multiple; + + @SerializedName("types") + private String type; + + @SerializedName("desc") + private String desc; + + @SerializedName("min") + private String min; + + @SerializedName("max") + private String max; + + @SerializedName("default_value") + private String default_value; + + @SerializedName("enums") + private List<String> enums; + } + + @Data + @AllArgsConstructor + public static class MIPEngineAlgorithmInputdataDTO { + @SerializedName("x") + private MIPEngineAlgorithmInputDataDetailDTO x; + + @SerializedName("y") + private MIPEngineAlgorithmInputDataDetailDTO y; + + @SerializedName("pathology") + private MIPEngineAlgorithmInputDataDetailDTO pathology; + + @SerializedName("datasets") + private MIPEngineAlgorithmInputDataDetailDTO datasets; + + @SerializedName("filter") + private MIPEngineAlgorithmInputDataDetailDTO filter; + } + + @Data + @AllArgsConstructor + public static class MIPEngineAlgorithmInputDataDetailDTO { + + @SerializedName("stattypes") + private List<String> stattypes; + + @SerializedName("label") + private String label; + + @SerializedName("notblank") + private String notblank; + + @SerializedName("enumslen") + private Integer enumslen; + + @SerializedName("multiple") + private String multiple; + + @SerializedName("types") + private List<String> types; + + @SerializedName("desc") + private String desc; + } +} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmRequestDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmRequestDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..a5604f2e7b9b619d023523535c636531c139f937 --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmRequestDTO.java @@ -0,0 +1,92 @@ +package eu.hbp.mip.models.DTOs; + +import com.google.gson.annotations.SerializedName; +import eu.hbp.mip.utils.JsonConverters; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +@Data +@AllArgsConstructor +public class MIPEngineAlgorithmRequestDTO { + @SerializedName("inputdata") + private InputData inputdata; + @SerializedName("parameters") + private HashMap<String, Object> parameters; + + public MIPEngineAlgorithmRequestDTO(List<ExaremeAlgorithmRequestParamDTO> exaremeAlgorithmRequestParamDTOs) + { + MIPEngineAlgorithmRequestDTO.InputData inputData = new MIPEngineAlgorithmRequestDTO.InputData(); + HashMap<String, Object> mipEngineParameters = new HashMap<>(); + + List<Object> rules = new ArrayList<>(); + exaremeAlgorithmRequestParamDTOs.forEach(parameter -> { + + switch (parameter.getName()) { + case "x": + List<String> x = Arrays.asList(parameter.getValue().split(",")); + x.forEach(column -> rules.add(new ExaremeAlgorithmDTO.Rule(column, parameter.getColumnValuesSQLType(), "is_not_null", null))); + inputData.setX(x); + break; + case "y": + List<String> y = Arrays.asList(parameter.getValue().split(",")); + y.forEach(column -> rules.add(new ExaremeAlgorithmDTO.Rule(column, parameter.getColumnValuesSQLType(), "is_not_null", null))); + inputData.setY(y); + break; + case "dataset": + List<String> datasets = Arrays.asList(parameter.getValue().split(",")); + rules.add(new ExaremeAlgorithmDTO.Rule("dataset", "string", "in", datasets)); + inputData.setDatasets(datasets); + break; + case "pathology": + inputData.setPathology(parameter.getValue()); + break; + case "filter": + if (!parameter.getValue().equals("")) + rules.add(JsonConverters.convertJsonStringToObject(parameter.getValue(), MIPEngineAlgorithmRequestDTO.Filter.class)); + break; + default: + mipEngineParameters.put(parameter.getName(), Arrays.asList(parameter.getValue().split(","))); + break; + } + }); + MIPEngineAlgorithmRequestDTO.Filter filter = new MIPEngineAlgorithmRequestDTO.Filter("AND", rules); + inputData.setFilters(filter); + this.inputdata = inputData; + this.parameters = mipEngineParameters; + } + + + @Data + @AllArgsConstructor + public static class InputData { + @SerializedName("pathology") + private String pathology; + @SerializedName("datasets") + private List<String> datasets; + @SerializedName("filters") + private Filter filters; + @SerializedName("x") + private List<String> x; + @SerializedName("y") + private List<String> y; + public InputData(){ + + } + } + + @Data + @AllArgsConstructor + public static class Filter + { + @SerializedName("condition") + private String condition; + + @SerializedName("rules") + private List<Object> rules; + } +} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmResultDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmResultDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..4491ddb443ddaba506e3165ab95c0e2ba3e20d87 --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAlgorithmResultDTO.java @@ -0,0 +1,14 @@ +package eu.hbp.mip.models.DTOs; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class MIPEngineAlgorithmResultDTO { + private final String title; + private final List<TabularVisualizationDTO.Field> columns; + private final List<List<Object>> data; +} \ No newline at end of file diff --git a/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java index aa78ef8b0e315a2f71261dc1fed480cb8617ccb2..ecc5b723bef81a572c61ac14dd46f48a5593eb50 100644 --- a/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java +++ b/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java @@ -1,13 +1,13 @@ package eu.hbp.mip.models.DTOs; import com.google.gson.annotations.SerializedName; -import lombok.Getter; -import lombok.Setter; +import lombok.AllArgsConstructor; +import lombok.Data; import java.util.List; -@Getter -@Setter +@Data +@AllArgsConstructor public class PathologyDTO { @SerializedName("code") @@ -22,8 +22,11 @@ public class PathologyDTO { @SerializedName("datasets") private List<PathologyDatasetDTO> datasets; - @Getter - @Setter + public PathologyDTO(){ + + } + @Data + @AllArgsConstructor public static class PathologyDatasetDTO { @SerializedName("code") private String code; diff --git a/src/main/java/eu/hbp/mip/models/DTOs/TabularVisualizationDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/TabularVisualizationDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..3e17932f7e716c001642c86ea2c6a54bf841ca43 --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/TabularVisualizationDTO.java @@ -0,0 +1,33 @@ +package eu.hbp.mip.models.DTOs; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.HashMap; +import java.util.List; + +@Data +@AllArgsConstructor +public class TabularVisualizationDTO { + private final String name; + private final String profile; + private final HashMap<String, List<Field>> schema; + private final List<List<Object>> data; + + + public TabularVisualizationDTO(MIPEngineAlgorithmResultDTO mipEngineAlgorithmResultDTO) { + HashMap<String, List<TabularVisualizationDTO.Field>> schema = new HashMap<>(); + schema.put("fields", mipEngineAlgorithmResultDTO.getColumns()); + this.name = mipEngineAlgorithmResultDTO.getTitle(); + this.profile = "tabular-data-resource"; + this.schema = schema; + this.data = mipEngineAlgorithmResultDTO.getData(); + } + @Data + @AllArgsConstructor + public static class Field { + private final String name; + private final String type; + } +} + diff --git a/src/main/java/eu/hbp/mip/models/galaxy/WorkflowDTO.java b/src/main/java/eu/hbp/mip/models/galaxy/WorkflowDTO.java index 496ee2dfebf50324b1443774cc48edd4e32c2de0..eae07531e6f126aa69465a9fcdf6c6cf44c1d267 100644 --- a/src/main/java/eu/hbp/mip/models/galaxy/WorkflowDTO.java +++ b/src/main/java/eu/hbp/mip/models/galaxy/WorkflowDTO.java @@ -2,7 +2,8 @@ package eu.hbp.mip.models.galaxy; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; -import eu.hbp.mip.models.DTOs.AlgorithmDTO; +import eu.hbp.mip.models.DTOs.ExaremeAlgorithmDTO; +import eu.hbp.mip.models.DTOs.ExaremeAlgorithmRequestParamDTO; import java.util.*; @@ -103,29 +104,29 @@ public class WorkflowDTO { } } - public AlgorithmDTO convertToAlgorithmDTO() { + public ExaremeAlgorithmDTO convertToAlgorithmDTO() { - AlgorithmDTO algorithmDTO = new AlgorithmDTO(); + ExaremeAlgorithmDTO exaremeAlgorithmDTO = new ExaremeAlgorithmDTO(); // Transfer workflow information - algorithmDTO.setName(id); - algorithmDTO.setDesc(""); - algorithmDTO.setLabel(name); - algorithmDTO.setType("workflow"); + exaremeAlgorithmDTO.setName(id); + exaremeAlgorithmDTO.setDesc(""); + exaremeAlgorithmDTO.setLabel(name); + exaremeAlgorithmDTO.setType("workflow"); // Transfer workflow parameters information - List<AlgorithmDTO.AlgorithmParamDTO> algorithmParams = new LinkedList<>(); + List<ExaremeAlgorithmRequestParamDTO> algorithmParams = new LinkedList<>(); Gson gson = new Gson(); for (Map.Entry<String, WorkflowInputDTO> workflowInput : getInputs().entrySet()) { // Convert the annotation to algorithm Parameter - AlgorithmDTO.AlgorithmParamDTO algorithmParam; + ExaremeAlgorithmRequestParamDTO algorithmParam; if (steps.get(workflowInput.getKey()).getAnnotation() != null) { algorithmParam = gson.fromJson(steps.get(workflowInput.getKey()).getAnnotation(), - AlgorithmDTO.AlgorithmParamDTO.class); + ExaremeAlgorithmRequestParamDTO.class); } else { // If annotation is not provided, auto-fill some information - algorithmParam = new AlgorithmDTO.AlgorithmParamDTO(); + algorithmParam = new ExaremeAlgorithmRequestParamDTO(); // When the constraints are not known, set the most relaxed constraints algorithmParam.setDesc(""); algorithmParam.setValue(""); @@ -154,9 +155,9 @@ public class WorkflowDTO { algorithmParams.add(algorithmParam); } - algorithmDTO.setParameters(algorithmParams); + exaremeAlgorithmDTO.setParameters(algorithmParams); - return algorithmDTO; + return exaremeAlgorithmDTO; } } diff --git a/src/main/java/eu/hbp/mip/repositories/ExperimentRepository.java b/src/main/java/eu/hbp/mip/repositories/ExperimentRepository.java index 9ac54cada906b78ad5ab5ac41d77d05562bedc4e..b776b6c46d147e0c146305b4567a0c518b3ea11c 100644 --- a/src/main/java/eu/hbp/mip/repositories/ExperimentRepository.java +++ b/src/main/java/eu/hbp/mip/repositories/ExperimentRepository.java @@ -1,9 +1,17 @@ package eu.hbp.mip.repositories; import eu.hbp.mip.models.DAOs.ExperimentDAO; +import eu.hbp.mip.models.DAOs.UserDAO; +import eu.hbp.mip.models.DTOs.ExperimentDTO; +import eu.hbp.mip.utils.Exceptions.BadRequestException; +import eu.hbp.mip.utils.Exceptions.ExperimentNotFoundException; +import eu.hbp.mip.utils.Exceptions.InternalServerError; +import eu.hbp.mip.utils.JsonConverters; +import eu.hbp.mip.utils.Logger; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.repository.CrudRepository; +import java.util.Date; import java.util.Optional; import java.util.UUID; @@ -14,4 +22,91 @@ import java.util.UUID; public interface ExperimentRepository extends CrudRepository<ExperimentDAO, UUID>, JpaSpecificationExecutor<ExperimentDAO> { ExperimentDAO findByUuid(UUID experimentUuid); + + /** + * The loadExperiment access the database and load the information of a specific experiment + * + * @param uuid is the id of the experiment to be retrieved + * @return the experiment information that was retrieved from database + */ + default ExperimentDAO loadExperiment(String uuid, Logger logger) { + UUID experimentUuid; + ExperimentDAO experimentDAO; + + try { + experimentUuid = UUID.fromString(uuid); + } catch (Exception e) { + logger.LogUserAction( e.getMessage()); + throw new BadRequestException(e.getMessage()); + } + + experimentDAO = findByUuid(experimentUuid); + if (experimentDAO == null) { + logger.LogUserAction( "Experiment with uuid : " + uuid + "was not found."); + throw new ExperimentNotFoundException("Experiment with uuid : " + uuid + " was not found."); + } + + return experimentDAO; + } + + /** + * The createExperimentInTheDatabase will insert a new experiment in the database according to the given experiment information + * + * @param experimentDTO is the experiment information to inserted in the database + * @return the experiment information that was inserted into the database + * @Note In the database there will be stored Algorithm Details that is the whole information about the algorithm + * and an Algorithm column that is required for the filtering with algorithm name in the GET /experiments. + */ + default ExperimentDAO createExperimentInTheDatabase(ExperimentDTO experimentDTO, UserDAO user, Logger logger) { + + ExperimentDAO experimentDAO = new ExperimentDAO(); + experimentDAO.setUuid(UUID.randomUUID()); + experimentDAO.setCreatedBy(user); + experimentDAO.setAlgorithm(JsonConverters.convertObjectToJsonString(experimentDTO.getAlgorithm())); + experimentDAO.setAlgorithmId(experimentDTO.getAlgorithm().getName()); + experimentDAO.setName(experimentDTO.getName()); + experimentDAO.setStatus(ExperimentDAO.Status.pending); + + try { + save(experimentDAO); + } catch (Exception e) { + logger.LogUserAction("Attempted to save changes to database but an error ocurred : " + e.getMessage() + "."); + throw new InternalServerError(e.getMessage()); + } + + logger.LogUserAction(" id : " + experimentDAO.getUuid()); + logger.LogUserAction(" algorithm : " + experimentDAO.getAlgorithm()); + logger.LogUserAction(" name : " + experimentDAO.getName()); + return experimentDAO; + } + + default void saveExperiment(ExperimentDAO experimentDAO, Logger logger) { + + logger.LogUserAction(" id : " + experimentDAO.getUuid()); + logger.LogUserAction(" algorithm : " + experimentDAO.getAlgorithm()); + logger.LogUserAction(" name : " + experimentDAO.getName()); + logger.LogUserAction(" historyId : " + experimentDAO.getWorkflowHistoryId()); + logger.LogUserAction(" status : " + experimentDAO.getStatus()); + + try { + save(experimentDAO); + } catch (Exception e) { + logger.LogUserAction("Attempted to save changes to database but an error ocurred : " + e.getMessage() + "."); + throw new InternalServerError(e.getMessage()); + } + + logger.LogUserAction("Saved experiment"); + } + + default void finishExperiment(ExperimentDAO experimentDAO, Logger logger) { + experimentDAO.setFinished(new Date()); + + try { + save(experimentDAO); + } catch (Exception e) { + logger.LogUserAction( "Attempted to save changes to database but an error ocurred : " + e.getMessage() + "."); + throw new InternalServerError(e.getMessage()); + } + } + } diff --git a/src/main/java/eu/hbp/mip/services/ExperimentService.java b/src/main/java/eu/hbp/mip/services/ExperimentService.java index 45345c8a5b33d5826c149cd384a5b17c0d3b7a78..1bd848ed64f22c27c2170a450ca103a723199f0c 100644 --- a/src/main/java/eu/hbp/mip/services/ExperimentService.java +++ b/src/main/java/eu/hbp/mip/services/ExperimentService.java @@ -1,24 +1,10 @@ package eu.hbp.mip.services; -import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; -import com.github.jmchilton.blend4j.galaxy.GalaxyInstanceFactory; -import com.github.jmchilton.blend4j.galaxy.WorkflowsClient; -import com.github.jmchilton.blend4j.galaxy.beans.Workflow; -import com.github.jmchilton.blend4j.galaxy.beans.WorkflowDetails; -import com.github.jmchilton.blend4j.galaxy.beans.WorkflowInputDefinition; import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import com.google.gson.internal.LinkedTreeMap; -import eu.hbp.mip.controllers.galaxy.retrofit.RetroFitGalaxyClients; -import eu.hbp.mip.controllers.galaxy.retrofit.RetrofitClientInstance; import eu.hbp.mip.models.DAOs.ExperimentDAO; import eu.hbp.mip.models.DAOs.UserDAO; -import eu.hbp.mip.models.DTOs.AlgorithmDTO; -import eu.hbp.mip.models.DTOs.ExperimentDTO; -import eu.hbp.mip.models.galaxy.GalaxyWorkflowResult; -import eu.hbp.mip.models.galaxy.PostWorkflowToGalaxyDtoResponse; +import eu.hbp.mip.models.DTOs.*; import eu.hbp.mip.repositories.ExperimentRepository; import eu.hbp.mip.services.Specifications.ExperimentSpecifications; import eu.hbp.mip.utils.ClaimUtils; @@ -26,8 +12,6 @@ import eu.hbp.mip.utils.Exceptions.*; import eu.hbp.mip.utils.HTTPUtil; import eu.hbp.mip.utils.JsonConverters; import eu.hbp.mip.utils.Logger; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -35,13 +19,8 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; import org.springframework.security.core.Authentication; import org.springframework.stereotype.Service; -import retrofit2.Call; -import retrofit2.Response; - -import java.io.IOException; import java.util.*; -import static java.lang.Thread.sleep; @Service public class ExperimentService { @@ -50,11 +29,8 @@ public class ExperimentService { @Value("#{'${services.exareme.queryExaremeUrl}'}") private String queryExaremeUrl; - @Value("#{'${services.galaxy.galaxyUrl}'}") - private String galaxyUrl; - - @Value("#{'${services.galaxy.galaxyApiKey}'}") - private String galaxyApiKey; + @Value("#{'${services.mipengine.algorithmsUrl}'}") + private String mipengineAlgorithmsUrl; @Value("#{'${authentication.enabled}'}") private boolean authenticationIsEnabled; @@ -62,10 +38,12 @@ public class ExperimentService { private static final Gson gson = new Gson(); private final ActiveUserService activeUserService; + private final GalaxyService galaxyService; private final ExperimentRepository experimentRepository; - public ExperimentService(ActiveUserService activeUserService, ExperimentRepository experimentRepository) { + public ExperimentService(ActiveUserService activeUserService, GalaxyService galaxyService, ExperimentRepository experimentRepository) { this.activeUserService = activeUserService; + this.galaxyService = galaxyService; this.experimentRepository = experimentRepository; } @@ -119,7 +97,7 @@ public class ExperimentService { throw new NoContent("No experiment found with the filters provided."); List<ExperimentDTO> experimentDTOs = new ArrayList<>(); - experimentDAOs.forEach(experimentDAO -> experimentDTOs.add(experimentDAO.convertToDTO(false))); + experimentDAOs.forEach(experimentDAO -> experimentDTOs.add(new ExperimentDTO(false, experimentDAO))); Map<String, Object> response = new HashMap<>(); response.put("experiments", experimentDTOs); @@ -144,7 +122,7 @@ public class ExperimentService { logger.LogUserAction("Loading Experiment with uuid : " + uuid); - experimentDAO = loadExperiment(uuid, logger); + experimentDAO = experimentRepository.loadExperiment(uuid, logger); if ( !experimentDAO.isShared() && !experimentDAO.getCreatedBy().getUsername().equals(user.getUsername()) @@ -154,7 +132,7 @@ public class ExperimentService { logger.LogUserAction("Accessing Experiment is unauthorized."); throw new UnauthorizedException("You don't have access to the experiment."); } - ExperimentDTO experimentDTO = experimentDAO.convertToDTO(true); + ExperimentDTO experimentDTO = new ExperimentDTO(true, experimentDAO); logger.LogUserAction("Experiment was Loaded with uuid : " + uuid + "."); return experimentDTO; @@ -191,10 +169,10 @@ public class ExperimentService { // Run with the appropriate engine if (algorithmType.equals("workflow")) { logger.LogUserAction("Algorithm runs on Galaxy."); - return runGalaxyWorkflow(experimentDTO, logger); + return galaxyService.runGalaxyWorkflow(experimentDTO, logger); } else { logger.LogUserAction("Algorithm runs on Exareme."); - return createExaremeExperiment(experimentDTO, logger); + return createExperiment(experimentDTO, logger); } } @@ -219,13 +197,6 @@ public class ExperimentService { throw new BadRequestException("You can not run workflow algorithms transiently."); } - // Get the parameters - List<AlgorithmDTO.AlgorithmParamDTO> algorithmParameters - = experimentDTO.getAlgorithm().getParameters(); - - // Get the type and name of algorithm - String algorithmName = experimentDTO.getAlgorithm().getName(); - algorithmParametersLogging(experimentDTO, logger); if (authenticationIsEnabled) { @@ -233,19 +204,15 @@ public class ExperimentService { ClaimUtils.validateAccessRightsOnDatasets(authentication, experimentDatasets, logger); } - String body = gson.toJson(algorithmParameters); - String url = queryExaremeUrl + "/" + algorithmName; - logger.LogUserAction("url: " + url + ", body: " + body); - - logger.LogUserAction("Completed, returning: " + experimentDTO.toString()); + logger.LogUserAction("Completed, returning: " + experimentDTO); // Results are stored in the experiment object - ExaremeResult exaremeResult = runExaremeExperiment(url, body, experimentDTO); + ExaremeAlgorithmResultDTO exaremeAlgorithmResultDTO = runExperiment(experimentDTO, logger); - logger.LogUserAction("Experiment with uuid: " + experimentDTO.getUuid() + "gave response code: " + exaremeResult.getCode() + " and result: " + exaremeResult.getResults()); + logger.LogUserAction("Experiment with uuid: " + experimentDTO.getUuid() + "gave response code: " + exaremeAlgorithmResultDTO.getCode() + " and result: " + exaremeAlgorithmResultDTO.getResult()); - experimentDTO.setResult((exaremeResult.getCode() >= 400) ? null : exaremeResult.getResults()); - experimentDTO.setStatus((exaremeResult.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success); + experimentDTO.setResult((exaremeAlgorithmResultDTO.getCode() >= 400) ? null : exaremeAlgorithmResultDTO.getResult()); + experimentDTO.setStatus((exaremeAlgorithmResultDTO.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success); return experimentDTO; } @@ -262,7 +229,7 @@ public class ExperimentService { UserDAO user = activeUserService.getActiveUser(); logger.LogUserAction("Updating experiment with uuid : " + uuid + "."); - experimentDAO = loadExperiment(uuid, logger); + experimentDAO = experimentRepository.loadExperiment(uuid, logger); //Verify (PATCH) /experiments non editable fields. verifyPatchExperimentNonEditableFields(experimentDTO, logger); @@ -290,7 +257,7 @@ public class ExperimentService { logger.LogUserAction("Updated experiment with uuid : " + uuid + "."); - experimentDTO = experimentDAO.convertToDTO(true); + experimentDTO = new ExperimentDTO(true, experimentDAO); return experimentDTO; } @@ -305,7 +272,7 @@ public class ExperimentService { UserDAO user = activeUserService.getActiveUser(); logger.LogUserAction("Deleting experiment with uuid : " + uuid + "."); - experimentDAO = loadExperiment(uuid, logger); + experimentDAO = experimentRepository.loadExperiment(uuid, logger); if (!experimentDAO.getCreatedBy().getUsername().equals(user.getUsername())) throw new UnauthorizedException("You don't have access to the experiment."); @@ -393,7 +360,7 @@ public class ExperimentService { .append("\n")); logger.LogUserAction("Executing " + algorithmName + parametersLogMessage); } - + /** * The getDatasetFromExperimentParameters will retrieve the dataset from the experiment parameters * @@ -404,7 +371,7 @@ public class ExperimentService { private String getDatasetFromExperimentParameters(ExperimentDTO experimentDTO, Logger logger) { String experimentDatasets = null; - for (AlgorithmDTO.AlgorithmParamDTO parameter : experimentDTO.getAlgorithm().getParameters()) { + for (ExaremeAlgorithmRequestParamDTO parameter : experimentDTO.getAlgorithm().getParameters()) { if (parameter.getLabel().equals("dataset")) { experimentDatasets = parameter.getValue(); break; @@ -418,100 +385,74 @@ public class ExperimentService { return experimentDatasets; } - /** - * The loadExperiment access the database and load the information of a specific experiment - * - * @param uuid is the id of the experiment to be retrieved - * @return the experiment information that was retrieved from database - */ - private ExperimentDAO loadExperiment(String uuid, Logger logger) { - UUID experimentUuid; - ExperimentDAO experimentDAO; - - try { - experimentUuid = UUID.fromString(uuid); - } catch (Exception e) { - logger.LogUserAction( e.getMessage()); - throw new BadRequestException(e.getMessage()); - } - - experimentDAO = experimentRepository.findByUuid(experimentUuid); - if (experimentDAO == null) { - logger.LogUserAction( "Experiment with uuid : " + uuid + "was not found."); - throw new ExperimentNotFoundException("Experiment with uuid : " + uuid + " was not found."); - } - - return experimentDAO; + private ExaremeAlgorithmResultDTO convertMIPEngineResultToExaremeAlgorithmResult(int code, String result) { + MIPEngineAlgorithmResultDTO mipVisualization = JsonConverters.convertJsonStringToObject(result, MIPEngineAlgorithmResultDTO.class); + LinkedTreeMap<String,Object> data = new LinkedTreeMap<>(); + data.put("data", new TabularVisualizationDTO(mipVisualization)); + data.put("type", "application/vnd.dataresource+json"); + List<Object> finalObject = new ArrayList<>(); + finalObject.add(data); + return new ExaremeAlgorithmResultDTO(code, finalObject); } /** - * The createExperimentInTheDatabase will insert a new experiment in the database according to the given experiment information + * The runExperiment will run the experiment to exareme or MIPEngine. * - * @param experimentDTO is the experiment information to inserted in the database - * @return the experiment information that was inserted into the database - * @Note In the database there will be stored Algorithm Details that is the whole information about the algorithm - * and an Algorithm column that is required for the filtering with algorithm name in the GET /experiments. + * @param experimentDTO is the request with the experiment information + * @return the result of experiment as well as the http status that was retrieved */ - private ExperimentDAO createExperimentInTheDatabase(ExperimentDTO experimentDTO, Logger logger) { - UserDAO user = activeUserService.getActiveUser(); + public ExaremeAlgorithmResultDTO runExperiment(ExperimentDTO experimentDTO, Logger logger) { - ExperimentDAO experimentDAO = new ExperimentDAO(); - experimentDAO.setUuid(UUID.randomUUID()); - experimentDAO.setCreatedBy(user); - experimentDAO.setAlgorithm(JsonConverters.convertObjectToJsonString(experimentDTO.getAlgorithm())); - experimentDAO.setAlgorithmId(experimentDTO.getAlgorithm().getName()); - experimentDAO.setName(experimentDTO.getName()); - experimentDAO.setStatus(ExperimentDAO.Status.pending); + // Algorithm type + String algorithmType = experimentDTO.getAlgorithm().getType(); - try { - experimentRepository.save(experimentDAO); - } catch (Exception e) { - logger.LogUserAction("Attempted to save changes to database but an error ocurred : " + e.getMessage() + "."); - throw new InternalServerError(e.getMessage()); - } + // Run the 1st algorithm from the list + String algorithmName = experimentDTO.getAlgorithm().getName(); - logger.LogUserAction(" id : " + experimentDAO.getUuid()); - logger.LogUserAction(" algorithm : " + experimentDAO.getAlgorithm()); - logger.LogUserAction(" name : " + experimentDAO.getName()); - return experimentDAO; + // Run with the appropriate engine + if (algorithmType.equals("mipengine")) { + MIPEngineAlgorithmRequestDTO mipEngineAlgorithmRequestDTO = new MIPEngineAlgorithmRequestDTO(experimentDTO.getAlgorithm().getParameters()); + String body = JsonConverters.convertObjectToJsonString(mipEngineAlgorithmRequestDTO); + String url = mipengineAlgorithmsUrl + "/" + algorithmName.toLowerCase(); + logger.LogUserAction("url: " + url + ", body: " + body); + logger.LogUserAction("Algorithm runs on MIPEngine."); + return runMIPEngineExperiment(url, body); + } else { + List<ExaremeAlgorithmRequestParamDTO> algorithmParameters + = experimentDTO.getAlgorithm().getParameters(); + String body = gson.toJson(algorithmParameters); + String url = queryExaremeUrl + "/" + algorithmName; + logger.LogUserAction("url: " + url + ", body: " + body); + logger.LogUserAction("Algorithm runs on Exareme."); + return runExaremeExperiment(url, body); + } } - private void saveExperiment(ExperimentDAO experimentDAO, Logger logger) { - logger.LogUserAction(" id : " + experimentDAO.getUuid()); - logger.LogUserAction(" algorithm : " + experimentDAO.getAlgorithm()); - logger.LogUserAction(" name : " + experimentDAO.getName()); - logger.LogUserAction(" historyId : " + experimentDAO.getWorkflowHistoryId()); - logger.LogUserAction(" status : " + experimentDAO.getStatus()); + /** + * The runExaremeExperiment will run to exareme the experiment + * + * @param url is the url that contain the results of the experiment + * @param body is the parameters of the algorithm + * @return the result of exareme as well as the http status that was retrieved + */ + public ExaremeAlgorithmResultDTO runExaremeExperiment(String url, String body) { + StringBuilder results = new StringBuilder(); + int code; try { - experimentRepository.save(experimentDAO); + code = HTTPUtil.sendPost(url, body, results); } catch (Exception e) { - logger.LogUserAction("Attempted to save changes to database but an error ocurred : " + e.getMessage() + "."); - throw new InternalServerError(e.getMessage()); + throw new InternalServerError("Error occurred : " + e.getMessage()); } - logger.LogUserAction("Saved experiment"); - } - - private void finishExperiment(ExperimentDAO experimentDAO, Logger logger) { - experimentDAO.setFinished(new Date()); - - try { - experimentRepository.save(experimentDAO); - } catch (Exception e) { - logger.LogUserAction( "Attempted to save changes to database but an error ocurred : " + e.getMessage() + "."); - throw new InternalServerError(e.getMessage()); - } - } + // Results are stored in the experiment object + ExaremeAlgorithmResultDTO exaremeResult = JsonConverters.convertJsonStringToObject( + String.valueOf(results), ExaremeAlgorithmResultDTO.class + ); + exaremeResult.setCode(code); - private String formattingGalaxyResult(String result) { - List<LinkedTreeMap<String,Object>> jsonObject = JsonConverters.convertJsonStringToObject(result, new ArrayList<ArrayList<Object>>().getClass()); - LinkedTreeMap<String,Object> firstResult = jsonObject.get(0); - jsonObject = (List<LinkedTreeMap<String, Object>>) firstResult.get("result"); - List<LinkedTreeMap<String,Object>> finalJsonObject = new ArrayList<>(); - finalJsonObject.add(jsonObject.get(0)); - return JsonConverters.convertObjectToJsonString(finalJsonObject); + return exaremeResult; } /** @@ -519,10 +460,9 @@ public class ExperimentService { * * @param url is the url that contain the results of the experiment * @param body is the parameters of the algorithm - * @param experimentDTO is the experiment information to be executed in the exareme * @return the result of exareme as well as the http status that was retrieved */ - public ExaremeResult runExaremeExperiment(String url, String body, ExperimentDTO experimentDTO) { + public ExaremeAlgorithmResultDTO runMIPEngineExperiment(String url, String body) { StringBuilder results = new StringBuilder(); int code; @@ -531,12 +471,9 @@ public class ExperimentService { } catch (Exception e) { throw new InternalServerError("Error occurred : " + e.getMessage()); } - + System.out.println(results); // Results are stored in the experiment object - ExperimentDTO experimentDTOWithOnlyResult = JsonConverters.convertJsonStringToObject(String.valueOf(results), ExperimentDTO.class); - List<Object> resultDTOS = experimentDTOWithOnlyResult.getResult(); - - return new ExaremeResult(code, resultDTOS); + return convertMIPEngineResultToExaremeAlgorithmResult(code, String.valueOf(results)); } /* -------------------------------------- EXAREME CALLS ---------------------------------------------------------*/ @@ -548,27 +485,14 @@ public class ExperimentService { * @param logger contains username and the endpoint. * @return the experiment information that was retrieved from exareme */ - public ExperimentDTO createExaremeExperiment(ExperimentDTO experimentDTO, Logger logger) { + public ExperimentDTO createExperiment(ExperimentDTO experimentDTO, Logger logger) { logger.LogUserAction("Running the algorithm..."); - ExperimentDAO experimentDAO = createExperimentInTheDatabase(experimentDTO, logger); + ExperimentDAO experimentDAO = experimentRepository.createExperimentInTheDatabase(experimentDTO, activeUserService.getActiveUser(), logger); logger.LogUserAction("Created experiment with uuid :" + experimentDAO.getUuid()); - // Run the 1st algorithm from the list - String algorithmName = experimentDTO.getAlgorithm().getName(); - - // Get the parameters - List<AlgorithmDTO.AlgorithmParamDTO> algorithmParameters - = experimentDTO.getAlgorithm().getParameters(); - - String body = gson.toJson(algorithmParameters); - String url = queryExaremeUrl + "/" + algorithmName; - logger.LogUserAction("url: " + url + ", body: " + body); - - logger.LogUserAction("Completed, returning: " + experimentDTO.toString()); - - logger.LogUserAction("Starting exareme execution thread"); + logger.LogUserAction("Starting execution in thread"); ExperimentDTO finalExperimentDTO = experimentDTO; new Thread(() -> { @@ -576,433 +500,27 @@ public class ExperimentService { Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Thread named :" + Thread.currentThread().getName() + " with id :" + Thread.currentThread().getId() + " started!"); try { + // Results are stored in the experiment object - ExaremeResult exaremeResult = runExaremeExperiment(url, body, finalExperimentDTO); + ExaremeAlgorithmResultDTO exaremeAlgorithmResultDTO = runExperiment(finalExperimentDTO, logger); - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Experiment with uuid: " + experimentDAO.getUuid() + "gave response code: " + exaremeResult.getCode() + " and result: " + exaremeResult.getResults()); + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Experiment with uuid: " + experimentDAO.getUuid() + "gave response code: " + exaremeAlgorithmResultDTO.getCode() + " and result: " + exaremeAlgorithmResultDTO.getResult()); - experimentDAO.setResult((exaremeResult.getCode() >= 400) ? null : JsonConverters.convertObjectToJsonString(exaremeResult.getResults())); - experimentDAO.setStatus((exaremeResult.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success); + experimentDAO.setResult((exaremeAlgorithmResultDTO.getCode() >= 400) ? null : JsonConverters.convertObjectToJsonString(exaremeAlgorithmResultDTO.getResult())); + experimentDAO.setStatus((exaremeAlgorithmResultDTO.getCode() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success); } catch (Exception e) { Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "There was an exception: " + e.getMessage()); experimentDAO.setStatus(ExperimentDAO.Status.error); } - finishExperiment(experimentDAO, logger); - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Finished the experiment: " + experimentDAO.toString()); + experimentRepository.finishExperiment(experimentDAO, logger); + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Finished the experiment: " + experimentDAO); }).start(); - experimentDTO = experimentDAO.convertToDTO(true); + experimentDTO = new ExperimentDTO(true, experimentDAO); return experimentDTO; } /* --------------------------------------- GALAXY CALLS ---------------------------------------------------------*/ - - /** - * The runWorkflow will POST the algorithm to the galaxy client - * - * @param experimentDTO is the request with the experiment information - * @return the response to be returned - */ - public ExperimentDTO runGalaxyWorkflow(ExperimentDTO experimentDTO, Logger logger) { - logger.LogUserAction("Running a workflow..."); - - ExperimentDAO experimentDAO = createExperimentInTheDatabase(experimentDTO, logger); - logger.LogUserAction("Created experiment with uuid :" + experimentDAO.getUuid()); - - - // Run the 1st algorithm from the list - String workflowId = experimentDTO.getAlgorithm().getName(); - - // Get the parameters - List<AlgorithmDTO.AlgorithmParamDTO> algorithmParameters - = experimentDTO.getAlgorithm().getParameters(); - - // Convert the parameters to workflow parameters - HashMap<String, String> algorithmParamsIncludingEmpty = new HashMap<>(); - if (algorithmParameters != null) { - for (AlgorithmDTO.AlgorithmParamDTO param : algorithmParameters) { - algorithmParamsIncludingEmpty.put(param.getName(), param.getValue()); - } - } - - // Get all the algorithm parameters because the frontend provides only the non-null - final GalaxyInstance instance = GalaxyInstanceFactory.get(galaxyUrl, galaxyApiKey); - final WorkflowsClient workflowsClient = instance.getWorkflowsClient(); - Workflow workflow = null; - for (Workflow curWorkflow : workflowsClient.getWorkflows()) { - if (curWorkflow.getId().equals(workflowId)) { - workflow = curWorkflow; - break; - } - } - if (workflow == null) { - logger.LogUserAction("Could not find algorithm code: " + workflowId); - throw new BadRequestException("Could not find galaxy algorithm."); - } - final WorkflowDetails workflowDetails = workflowsClient.showWorkflow(workflow.getId()); - for (Map.Entry<String, WorkflowInputDefinition> workflowParameter : workflowDetails.getInputs().entrySet()) { - if (!(algorithmParamsIncludingEmpty.containsKey(workflowParameter.getValue().getUuid()))) { - algorithmParamsIncludingEmpty.put(workflowParameter.getValue().getUuid(), ""); - } - } - - // Create the body of the request - HashMap<String, HashMap<String, String>> requestBody = new HashMap<>(); - requestBody.put("inputs", algorithmParamsIncludingEmpty); - JsonObject requestBodyJson = new JsonParser().parse(gson.toJson(requestBody)).getAsJsonObject(); - - // Create the request client - RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); - logger.LogUserAction("Running Galaxy workflow with id: " + workflow.getId()); - - // Call Galaxy to run the workflow - Call<PostWorkflowToGalaxyDtoResponse> call = service.postWorkflowToGalaxy(workflow.getId(), galaxyApiKey, requestBodyJson); - try { - Response<PostWorkflowToGalaxyDtoResponse> response = call.execute(); - - if (response.code() == 200) { // Call succeeded - String responseBody = gson.toJson(response.body()); - logger.LogUserAction("Response: " + responseBody); - - String historyId = (String) new JSONObject(responseBody).get("history_id"); - experimentDAO.setWorkflowHistoryId(historyId); - experimentDAO.setStatus(ExperimentDAO.Status.success); - - } else { // Something unexpected happened - String msgErr = gson.toJson(response.errorBody()); - logger.LogUserAction("Error Response: " + msgErr); - experimentDTO.setStatus((response.code() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success); - } - - } catch (Exception e) { - logger.LogUserAction("An exception occurred: " + e.getMessage()); - experimentDAO.setStatus(ExperimentDAO.Status.error); - } - saveExperiment(experimentDAO, logger); - - // Start the process of fetching the status - updateWorkflowExperiment(experimentDAO, logger); - - logger.LogUserAction("Run workflow completed!"); - - experimentDTO = experimentDAO.convertToDTO(true); - return experimentDTO; - } - - - /** - * This method creates a thread that will fetch the workflow result when it is ready - * - * @param experimentDAO will be used to fetch it's workflow status, it should have the workflowHistoryId initialized - * and the result should not already be fetched - */ - public void updateWorkflowExperiment(ExperimentDAO experimentDAO, Logger logger) { - - if (experimentDAO == null) { - logger.LogUserAction("The experiment does not exist."); - return; - } - - logger.LogUserAction(" Experiment id : " + experimentDAO.getUuid()); - if (experimentDAO.getWorkflowHistoryId() == null) { - logger.LogUserAction("History Id does not exist."); - return; - } - - logger.LogUserAction("Starting Thread..."); - new Thread(() -> { - while (true) { - // ATTENTION: Inside the Thread only LogExperimentAction should be used, not LogExperimentAction! - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Thread is running..."); - - try { - sleep(2000); - } catch (InterruptedException e) { - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Sleep was disrupted: " + e.getMessage()); - } - - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Fetching status for experiment Id: " + experimentDAO.getUuid()); - - String state = getWorkflowStatus(experimentDAO); - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "State is: " + state); - - switch (state) { - case "pending": - // Do nothing, when the experiment is created the status is set to running - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Workflow is still running."); - break; - - case "success": - // Get only the job result that is visible - List<GalaxyWorkflowResult> workflowJobsResults = getWorkflowResults(experimentDAO); - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Results are: " + workflowJobsResults.toString()); - - boolean resultFound = false; - for (GalaxyWorkflowResult jobResult : workflowJobsResults) { - if (jobResult.getVisible()) { - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Visible result are: " + jobResult.getId()); - - String result = getWorkflowResultBody(experimentDAO, jobResult.getId()); - - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "ResultDTO: " + result); - if (result == null) { - experimentDAO.setStatus(ExperimentDAO.Status.error); - } else { - experimentDAO.setResult(result); - experimentDAO.setStatus(ExperimentDAO.Status.success); - resultFound = true; - } - } - } - - if (!resultFound) { // If there is no visible result - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "No visible result"); - experimentDAO.setStatus(ExperimentDAO.Status.error); - } - - finishExperiment(experimentDAO, logger); - break; - - case "error": - // Get the job result that failed - workflowJobsResults = getWorkflowResults(experimentDAO); - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Error results are: " + workflowJobsResults.toString()); - - boolean failedJobFound = false; - for (GalaxyWorkflowResult jobResult : workflowJobsResults) { - if (jobResult.getState().equals("error")) { - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Failed job is: " + jobResult.getId()); - - String result = getWorkflowJobError(jobResult.getId(), experimentDAO); - - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Job result: " + result); - if (result == null) { - experimentDAO.setStatus(ExperimentDAO.Status.error); - } - experimentDAO.setStatus(ExperimentDAO.Status.error); - failedJobFound = true; - } - } - - if (!failedJobFound) { // If there is no visible failed job - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "No failed result"); - experimentDAO.setStatus(ExperimentDAO.Status.error); - } - finishExperiment(experimentDAO, logger); - break; - - default: // InternalError or unexpected result - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "An unexpected error occurred."); - experimentDAO.setStatus(ExperimentDAO.Status.error); - finishExperiment(experimentDAO, logger); - break; - } - - // If result exists return - if (experimentDAO.getResult() != null) { - Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "ResultDTO exists: " + experimentDAO.getResult()); - return; - } - } - }).start(); - } - - - /** - * @param experimentDAO The experiment of the workflow - * @return "pending" -> When the workflow is still running - * "internalError" -> When an exception or a bad request occurred - * "error" -> When the workflow produced an error - * "success" -> When the workflow completed successfully - */ - public String getWorkflowStatus(ExperimentDAO experimentDAO) { - String historyId = experimentDAO.getWorkflowHistoryId(); - String experimentName = experimentDAO.getName(); - UUID experimentId = experimentDAO.getUuid(); - - // ATTENTION: This function is used from a Thread. Only LogExperimentAction should be used, not LogUserAction! - Logger.LogExperimentAction(experimentName, experimentId, " History Id : " + historyId); - - // Create the request client - RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); - Call<Object> call = service.getWorkflowStatusFromGalaxy(historyId, galaxyApiKey); - - String result; - try { - Response<Object> response = call.execute(); - if (response.code() >= 400) { - Logger.LogExperimentAction(experimentName, experimentId, " Response code: " - + response.code() + "" + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); - return "internalError"; - } - result = new Gson().toJson(response.body()); - Logger.LogExperimentAction(experimentName, experimentId, " ResultDTO: " + result); - - } catch (IOException e) { - Logger.LogExperimentAction(experimentName, experimentId, " An exception happened: " + e.getMessage()); - return "internalError"; - } - - String state; - try { - JSONObject resultJson = new JSONObject(result); - state = resultJson.getString("state"); - } catch (JSONException e) { - Logger.LogExperimentAction(experimentName, experimentId, " An exception happened: " + e.getMessage()); - return "internalError"; - } - - Logger.LogExperimentAction(experimentName, experimentId, " Completed!"); - switch (state) { - case "ok": - return "success"; - case "error": - return "error"; - case "pending": - case "new": - case "waiting": - case "queued": - return "pending"; - default: - return "internalError"; - } - } - - /** - * @param experimentDAO The experiment of the workflow - * @return a List<GalaxyWorkflowResult> or null when an error occurred - */ - public List<GalaxyWorkflowResult> getWorkflowResults(ExperimentDAO experimentDAO) { - - String historyId = experimentDAO.getWorkflowHistoryId(); - String experimentName = experimentDAO.getName(); - UUID experimentId = experimentDAO.getUuid(); - Logger.LogExperimentAction(experimentName, experimentId, " historyId : " + historyId); - - RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); - Call<List<GalaxyWorkflowResult>> call = service.getWorkflowResultsFromGalaxy(historyId, galaxyApiKey); - - List<GalaxyWorkflowResult> getGalaxyWorkflowResultList; - try { - Response<List<GalaxyWorkflowResult>> response = call.execute(); - if (response.code() >= 400) { - Logger.LogExperimentAction(experimentName, experimentId, " Response code: " - + response.code() + "" + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); - return null; - } - getGalaxyWorkflowResultList = response.body(); - Logger.LogExperimentAction(experimentName, experimentId, " ResultDTO: " + response.body()); - - } catch (IOException e) { - Logger.LogExperimentAction(experimentName, experimentId, " An exception happened: " + e.getMessage()); - return null; - } - - Logger.LogExperimentAction(experimentName, experimentId, " Completed!"); - return getGalaxyWorkflowResultList; - - } - - /** - * @param experimentDAO The experiment of the workflow - * @param contentId the id of the job result that we want - * @return the result of the specific workflow job, null if there was an error - */ - public String getWorkflowResultBody(ExperimentDAO experimentDAO, String contentId) { - - String historyId = experimentDAO.getWorkflowHistoryId(); - String experimentName = experimentDAO.getName(); - UUID experimentId = experimentDAO.getUuid(); - - Logger.LogExperimentAction(experimentName, experimentId, " historyId : " + historyId); - - RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); - Call<Object> call = - service.getWorkflowResultsBodyFromGalaxy(historyId, contentId, galaxyApiKey); - - String resultJson; - try { - Response<Object> response = call.execute(); - if (response.code() >= 400) { - Logger.LogExperimentAction(experimentName, experimentId, " Response code: " - + response.code() + "" + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); - return null; - } - resultJson = new Gson().toJson(response.body()); - Logger.LogExperimentAction(experimentName, experimentId, " ResultDTO: " + resultJson); - - } catch (IOException e) { - Logger.LogExperimentAction(experimentName, experimentId, - " An exception happened: " + e.getMessage()); - return null; - } - - Logger.LogExperimentAction(experimentName, experimentId, " Completed!"); - return formattingGalaxyResult(resultJson); - } - - - /** - * @param jobId the id of the workflow job that failed - * @return the error that was produced or null if an error occurred - */ - public String getWorkflowJobError(String jobId, ExperimentDAO experimentDAO) { - String experimentName = experimentDAO.getName(); - UUID experimentId = experimentDAO.getUuid(); - - Logger.LogExperimentAction(experimentName, experimentId, " jobId : " + jobId); - RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); - Call<Object> callError = service.getErrorMessageOfWorkflowFromGalaxy(jobId, galaxyApiKey); - - String fullError; - String returnError; - try { - Response<Object> response = callError.execute(); - if (response.code() >= 400) { - Logger.LogExperimentAction(experimentName, experimentId, "Response code: " - + response.code() + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); - return null; - } - - // Parsing the stderr of the job that failed - String jsonString = new Gson().toJson(response.body()); - JsonElement jsonElement = new JsonParser().parse(jsonString); - JsonObject rootObject = jsonElement.getAsJsonObject(); - fullError = rootObject.get("stderr").getAsString(); - Logger.LogExperimentAction(experimentName, experimentId, "Error: " + fullError); - - String[] arrOfStr = fullError.split("ValueError", 0); - String specError = arrOfStr[arrOfStr.length - 1]; - returnError = specError.substring(1); - Logger.LogExperimentAction(experimentName, experimentId, "Parsed Error: " + returnError); - - } catch (IOException e) { - Logger.LogExperimentAction(experimentName, experimentId, "Exception: " + e.getMessage()); - return null; - } - - Logger.LogExperimentAction(experimentName, experimentId, "Completed successfully!"); - - return returnError; - } - - static class ExaremeResult { - private final int code; - private final List<Object> results; - - public ExaremeResult(int code, List<Object> results) { - this.code = code; - this.results = results; - } - - public int getCode() { - return code; - } - - public List<Object> getResults() { - return results; - } - } } diff --git a/src/main/java/eu/hbp/mip/services/GalaxyService.java b/src/main/java/eu/hbp/mip/services/GalaxyService.java new file mode 100644 index 0000000000000000000000000000000000000000..cc10afebdfdff26008947c6b24b7eb5865e6918b --- /dev/null +++ b/src/main/java/eu/hbp/mip/services/GalaxyService.java @@ -0,0 +1,456 @@ +package eu.hbp.mip.services; + +import com.github.jmchilton.blend4j.galaxy.GalaxyInstance; +import com.github.jmchilton.blend4j.galaxy.GalaxyInstanceFactory; +import com.github.jmchilton.blend4j.galaxy.WorkflowsClient; +import com.github.jmchilton.blend4j.galaxy.beans.Workflow; +import com.github.jmchilton.blend4j.galaxy.beans.WorkflowDetails; +import com.github.jmchilton.blend4j.galaxy.beans.WorkflowInputDefinition; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.internal.LinkedTreeMap; +import eu.hbp.mip.controllers.galaxy.retrofit.RetroFitGalaxyClients; +import eu.hbp.mip.controllers.galaxy.retrofit.RetrofitClientInstance; +import eu.hbp.mip.models.DAOs.ExperimentDAO; +import eu.hbp.mip.models.DTOs.ExaremeAlgorithmRequestParamDTO; +import eu.hbp.mip.models.DTOs.ExperimentDTO; +import eu.hbp.mip.models.galaxy.GalaxyWorkflowResult; +import eu.hbp.mip.models.galaxy.PostWorkflowToGalaxyDtoResponse; +import eu.hbp.mip.repositories.ExperimentRepository; +import eu.hbp.mip.utils.Exceptions.BadRequestException; +import eu.hbp.mip.utils.Exceptions.InternalServerError; +import eu.hbp.mip.utils.JsonConverters; +import eu.hbp.mip.utils.Logger; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import retrofit2.Call; +import retrofit2.Response; + +import java.io.IOException; +import java.util.*; + +import static java.lang.Thread.sleep; + +@Service +public class GalaxyService { + + private final ActiveUserService activeUserService; + private final ExperimentRepository experimentRepository; + public GalaxyService( + ActiveUserService activeUserService, + ExperimentRepository experimentRepository + ) { + this.activeUserService = activeUserService; + this.experimentRepository = experimentRepository; + } + @Value("#{'${services.galaxy.galaxyUrl}'}") + private String galaxyUrl; + + @Value("#{'${services.galaxy.galaxyApiKey}'}") + private String galaxyApiKey; + + private static final Gson gson = new Gson(); + + /** + * The runWorkflow will POST the algorithm to the galaxy client + * + * @param experimentDTO is the request with the experiment information + * @return the response to be returned + */ + public ExperimentDTO runGalaxyWorkflow(ExperimentDTO experimentDTO, Logger logger) { + logger.LogUserAction("Running a workflow..."); + + ExperimentDAO experimentDAO = experimentRepository.createExperimentInTheDatabase(experimentDTO, activeUserService.getActiveUser(), logger); + logger.LogUserAction("Created experiment with uuid :" + experimentDAO.getUuid()); + + + // Run the 1st algorithm from the list + String workflowId = experimentDTO.getAlgorithm().getName(); + + // Get the parameters + List<ExaremeAlgorithmRequestParamDTO> algorithmParameters + = experimentDTO.getAlgorithm().getParameters(); + + // Convert the parameters to workflow parameters + HashMap<String, String> algorithmParamsIncludingEmpty = new HashMap<>(); + if (algorithmParameters != null) { + for (ExaremeAlgorithmRequestParamDTO param : algorithmParameters) { + algorithmParamsIncludingEmpty.put(param.getName(), param.getValue()); + } + } + + // Get all the algorithm parameters because the frontend provides only the non-null + final GalaxyInstance instance = GalaxyInstanceFactory.get(galaxyUrl, galaxyApiKey); + final WorkflowsClient workflowsClient = instance.getWorkflowsClient(); + Workflow workflow = null; + for (Workflow curWorkflow : workflowsClient.getWorkflows()) { + if (curWorkflow.getId().equals(workflowId)) { + workflow = curWorkflow; + break; + } + } + if (workflow == null) { + logger.LogUserAction("Could not find algorithm code: " + workflowId); + throw new BadRequestException("Could not find galaxy algorithm."); + } + final WorkflowDetails workflowDetails = workflowsClient.showWorkflow(workflow.getId()); + for (Map.Entry<String, WorkflowInputDefinition> workflowParameter : workflowDetails.getInputs().entrySet()) { + if (!(algorithmParamsIncludingEmpty.containsKey(workflowParameter.getValue().getUuid()))) { + algorithmParamsIncludingEmpty.put(workflowParameter.getValue().getUuid(), ""); + } + } + + // Create the body of the request + HashMap<String, HashMap<String, String>> requestBody = new HashMap<>(); + requestBody.put("inputs", algorithmParamsIncludingEmpty); + JsonObject requestBodyJson = new JsonParser().parse(gson.toJson(requestBody)).getAsJsonObject(); + + // Create the request client + RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); + logger.LogUserAction("Running Galaxy workflow with id: " + workflow.getId()); + + // Call Galaxy to run the workflow + Call<PostWorkflowToGalaxyDtoResponse> call = service.postWorkflowToGalaxy(workflow.getId(), galaxyApiKey, requestBodyJson); + try { + Response<PostWorkflowToGalaxyDtoResponse> response = call.execute(); + + if (response.code() == 200) { // Call succeeded + String responseBody = gson.toJson(response.body()); + logger.LogUserAction("Response: " + responseBody); + + String historyId = (String) new JSONObject(responseBody).get("history_id"); + experimentDAO.setWorkflowHistoryId(historyId); + experimentDAO.setStatus(ExperimentDAO.Status.success); + + } else { // Something unexpected happened + String msgErr = gson.toJson(response.errorBody()); + logger.LogUserAction("Error Response: " + msgErr); + experimentDTO.setStatus((response.code() >= 400) ? ExperimentDAO.Status.error : ExperimentDAO.Status.success); + } + + } catch (Exception e) { + logger.LogUserAction("An exception occurred: " + e.getMessage()); + experimentDAO.setStatus(ExperimentDAO.Status.error); + } + experimentRepository.saveExperiment(experimentDAO, logger); + + // Start the process of fetching the status + updateWorkflowExperiment(experimentDAO, logger); + + logger.LogUserAction("Run workflow completed!"); + + experimentDTO = new ExperimentDTO(true, experimentDAO); + return experimentDTO; + } + + + + /** + * @param experimentDAO The experiment of the workflow + * @return "pending" -> When the workflow is still running + * "internalError" -> When an exception or a bad request occurred + * "error" -> When the workflow produced an error + * "success" -> When the workflow completed successfully + */ + public String getWorkflowStatus(ExperimentDAO experimentDAO) { + String historyId = experimentDAO.getWorkflowHistoryId(); + String experimentName = experimentDAO.getName(); + UUID experimentId = experimentDAO.getUuid(); + + // ATTENTION: This function is used from a Thread. Only LogExperimentAction should be used, not LogUserAction! + Logger.LogExperimentAction(experimentName, experimentId, " History Id : " + historyId); + + // Create the request client + RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); + Call<Object> call = service.getWorkflowStatusFromGalaxy(historyId, galaxyApiKey); + + String result; + try { + Response<Object> response = call.execute(); + if (response.code() >= 400) { + Logger.LogExperimentAction(experimentName, experimentId, " Response code: " + + response.code() + "" + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); + return "internalError"; + } + result = new Gson().toJson(response.body()); + Logger.LogExperimentAction(experimentName, experimentId, " ResultDTO: " + result); + + } catch (IOException e) { + Logger.LogExperimentAction(experimentName, experimentId, " An exception happened: " + e.getMessage()); + return "internalError"; + } + + String state; + try { + JSONObject resultJson = new JSONObject(result); + state = resultJson.getString("state"); + } catch (JSONException e) { + Logger.LogExperimentAction(experimentName, experimentId, " An exception happened: " + e.getMessage()); + return "internalError"; + } + + Logger.LogExperimentAction(experimentName, experimentId, " Completed!"); + switch (state) { + case "ok": + return "success"; + case "error": + return "error"; + case "pending": + case "new": + case "waiting": + case "queued": + return "pending"; + default: + return "internalError"; + } + } + + /** + * @param experimentDAO The experiment of the workflow + * @return a List<GalaxyWorkflowResult> or null when an error occurred + */ + public List<GalaxyWorkflowResult> getWorkflowResults(ExperimentDAO experimentDAO) { + + String historyId = experimentDAO.getWorkflowHistoryId(); + String experimentName = experimentDAO.getName(); + UUID experimentId = experimentDAO.getUuid(); + Logger.LogExperimentAction(experimentName, experimentId, " historyId : " + historyId); + + RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); + Call<List<GalaxyWorkflowResult>> call = service.getWorkflowResultsFromGalaxy(historyId, galaxyApiKey); + + List<GalaxyWorkflowResult> getGalaxyWorkflowResultList; + try { + Response<List<GalaxyWorkflowResult>> response = call.execute(); + if (response.code() >= 400) { + Logger.LogExperimentAction(experimentName, experimentId, " Response code: " + + response.code() + "" + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); + return null; + } + getGalaxyWorkflowResultList = response.body(); + Logger.LogExperimentAction(experimentName, experimentId, " ResultDTO: " + response.body()); + + } catch (IOException e) { + Logger.LogExperimentAction(experimentName, experimentId, " An exception happened: " + e.getMessage()); + return null; + } + + Logger.LogExperimentAction(experimentName, experimentId, " Completed!"); + return getGalaxyWorkflowResultList; + + } + + /** + * @param experimentDAO The experiment of the workflow + * @param contentId the id of the job result that we want + * @return the result of the specific workflow job, null if there was an error + */ + public String getWorkflowResultBody(ExperimentDAO experimentDAO, String contentId) { + + String historyId = experimentDAO.getWorkflowHistoryId(); + String experimentName = experimentDAO.getName(); + UUID experimentId = experimentDAO.getUuid(); + + Logger.LogExperimentAction(experimentName, experimentId, " historyId : " + historyId); + + RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); + Call<Object> call = + service.getWorkflowResultsBodyFromGalaxy(historyId, contentId, galaxyApiKey); + + String resultJson; + try { + Response<Object> response = call.execute(); + if (response.code() >= 400) { + Logger.LogExperimentAction(experimentName, experimentId, " Response code: " + + response.code() + "" + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); + return null; + } + resultJson = new Gson().toJson(response.body()); + Logger.LogExperimentAction(experimentName, experimentId, " ResultDTO: " + resultJson); + + } catch (IOException e) { + Logger.LogExperimentAction(experimentName, experimentId, + " An exception happened: " + e.getMessage()); + return null; + } + + Logger.LogExperimentAction(experimentName, experimentId, " Completed!"); + return formattingGalaxyResult(resultJson); + } + + private String formattingGalaxyResult(String result) { + List<LinkedTreeMap<String,Object>> jsonObject = JsonConverters.convertJsonStringToObject(result, new ArrayList<ArrayList<Object>>().getClass()); + LinkedTreeMap<String,Object> firstResult = jsonObject.get(0); + jsonObject = (List<LinkedTreeMap<String, Object>>) firstResult.get("result"); + List<LinkedTreeMap<String,Object>> finalJsonObject = new ArrayList<>(); + finalJsonObject.add(jsonObject.get(0)); + return JsonConverters.convertObjectToJsonString(finalJsonObject); + } + + + /** + * @param jobId the id of the workflow job that failed + * @return the error that was produced or null if an error occurred + */ + public String getWorkflowJobError(String jobId, ExperimentDAO experimentDAO) { + String experimentName = experimentDAO.getName(); + UUID experimentId = experimentDAO.getUuid(); + + Logger.LogExperimentAction(experimentName, experimentId, " jobId : " + jobId); + RetroFitGalaxyClients service = RetrofitClientInstance.getRetrofitInstance().create(RetroFitGalaxyClients.class); + Call<Object> callError = service.getErrorMessageOfWorkflowFromGalaxy(jobId, galaxyApiKey); + + String fullError; + String returnError; + try { + Response<Object> response = callError.execute(); + if (response.code() >= 400) { + Logger.LogExperimentAction(experimentName, experimentId, "Response code: " + + response.code() + " with body: " + (response.errorBody() != null ? response.errorBody().string() : " ")); + return null; + } + + // Parsing the stderr of the job that failed + String jsonString = new Gson().toJson(response.body()); + JsonElement jsonElement = new JsonParser().parse(jsonString); + JsonObject rootObject = jsonElement.getAsJsonObject(); + fullError = rootObject.get("stderr").getAsString(); + Logger.LogExperimentAction(experimentName, experimentId, "Error: " + fullError); + + String[] arrOfStr = fullError.split("ValueError", 0); + String specError = arrOfStr[arrOfStr.length - 1]; + returnError = specError.substring(1); + Logger.LogExperimentAction(experimentName, experimentId, "Parsed Error: " + returnError); + + } catch (IOException e) { + Logger.LogExperimentAction(experimentName, experimentId, "Exception: " + e.getMessage()); + return null; + } + + Logger.LogExperimentAction(experimentName, experimentId, "Completed successfully!"); + + return returnError; + } + + /** + * This method creates a thread that will fetch the workflow result when it is ready + * + * @param experimentDAO will be used to fetch it's workflow status, it should have the workflowHistoryId initialized + * and the result should not already be fetched + */ + public void updateWorkflowExperiment(ExperimentDAO experimentDAO, Logger logger) { + + if (experimentDAO == null) { + logger.LogUserAction("The experiment does not exist."); + return; + } + + logger.LogUserAction(" Experiment id : " + experimentDAO.getUuid()); + if (experimentDAO.getWorkflowHistoryId() == null) { + logger.LogUserAction("History Id does not exist."); + return; + } + + logger.LogUserAction("Starting Thread..."); + new Thread(() -> { + while (true) { + // ATTENTION: Inside the Thread only LogExperimentAction should be used, not LogExperimentAction! + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Thread is running..."); + + try { + sleep(2000); + } catch (InterruptedException e) { + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Sleep was disrupted: " + e.getMessage()); + throw new InternalServerError(e.getMessage()); + } + + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Fetching status for experiment Id: " + experimentDAO.getUuid()); + + String state = getWorkflowStatus(experimentDAO); + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "State is: " + state); + + switch (state) { + case "pending": + // Do nothing, when the experiment is created the status is set to running + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Workflow is still running."); + break; + + case "success": + // Get only the job result that is visible + List<GalaxyWorkflowResult> workflowJobsResults = getWorkflowResults(experimentDAO); + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Results are: " + workflowJobsResults.toString()); + + boolean resultFound = false; + for (GalaxyWorkflowResult jobResult : workflowJobsResults) { + if (jobResult.getVisible()) { + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Visible result are: " + jobResult.getId()); + + String result = getWorkflowResultBody(experimentDAO, jobResult.getId()); + + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "ResultDTO: " + result); + if (result == null) { + experimentDAO.setStatus(ExperimentDAO.Status.error); + } else { + experimentDAO.setResult(result); + experimentDAO.setStatus(ExperimentDAO.Status.success); + resultFound = true; + } + } + } + + if (!resultFound) { // If there is no visible result + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "No visible result"); + experimentDAO.setStatus(ExperimentDAO.Status.error); + } + + experimentRepository.finishExperiment(experimentDAO, logger); + break; + + case "error": + // Get the job result that failed + workflowJobsResults = getWorkflowResults(experimentDAO); + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Error results are: " + workflowJobsResults.toString()); + + boolean failedJobFound = false; + for (GalaxyWorkflowResult jobResult : workflowJobsResults) { + if (jobResult.getState().equals("error")) { + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Failed job is: " + jobResult.getId()); + + String result = getWorkflowJobError(jobResult.getId(), experimentDAO); + + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "Job result: " + result); + if (result == null) { + experimentDAO.setStatus(ExperimentDAO.Status.error); + } + experimentDAO.setStatus(ExperimentDAO.Status.error); + failedJobFound = true; + } + } + + if (!failedJobFound) { // If there is no visible failed job + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "No failed result"); + experimentDAO.setStatus(ExperimentDAO.Status.error); + } + experimentRepository.finishExperiment(experimentDAO, logger); + break; + + default: // InternalError or unexpected result + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "An unexpected error occurred."); + experimentDAO.setStatus(ExperimentDAO.Status.error); + experimentRepository.finishExperiment(experimentDAO, logger); + break; + } + + // If result exists return + if (experimentDAO.getResult() != null) { + Logger.LogExperimentAction(experimentDAO.getName(), experimentDAO.getUuid(), "ResultDTO exists: " + experimentDAO.getResult()); + return; + } + } + }).start(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 345fc4bfa1955560e8cc33b2fb7cf62bb3698f12..d8d542b6bb2d78a46f0f52d87057ab71acca04cd 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -34,6 +34,8 @@ spring: ### EXTERNAL SERVICES ### services: + mipengine: + algorithmsUrl: "http://127.0.0.1:5000/algorithms" exareme: queryExaremeUrl: "http://127.0.0.1:9090/mining/query" algorithmsUrl: "http://127.0.0.1:9090/mining/algorithms.json"