Skip to content
Snippets Groups Projects
Commit 3007ddc5 authored by Mirco Nasuti's avatar Mirco Nasuti
Browse files

add second datasource

parent 9d038ef9
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2161 additions and 262 deletions
.gitignore
+ 1
− 0
View file @ 3007ddc5
......@@ -4,3 +4,4 @@ target/
.DS_Store
.m2
*.jar
src/main/resources/data/
docker/runner/config/application.tmpl
+ 5
− 0
View file @ 3007ddc5
......@@ -7,6 +7,11 @@ spring:
username: {{ default .Env.DB_USER "postgres" }}
password: {{ .Env.DB_PASSWORD }}
driver-class-name: org.postgresql.Driver
variablesDatasource:
url: {{ default .Env.VARIABLES_DB_URL "jdbc:postgresql://172.22.0.1:5433/postgres" }}
username: {{ default .Env.VARIABLES_DB_USER "postgres" }}
password: {{ .Env.VARIABLES_DB_PASSWORD }}
driver-class-name: org.postgresql.Driver
jpa:
hibernate:
dialect: org.hibernate.dialect.PostgreSQL9Dialect
......
src/main/java/eu/hbp/mip/controllers/GroupsApi.java
+ 16
− 7
View file @ 3007ddc5
......@@ -4,19 +4,24 @@
package eu.hbp.mip.controllers;
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import eu.hbp.mip.model.Variable;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import eu.hbp.mip.model.Group;
import eu.hbp.mip.repositories.GroupRepository;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
@RestController
......@@ -26,18 +31,22 @@ public class GroupsApi {
private static final Logger LOGGER = Logger.getLogger(GroupsApi.class);
private static final String ROOT_CODE = "root";
private static final String VARIABLES_FILE = "data/variables.json";
@Autowired
GroupRepository groupRepository;
@ApiOperation(value = "Get the root group (containing all subgroups)", response = Group.class)
@ApiResponses(value = { @ApiResponse(code = 200, message = "Success") })
@RequestMapping(method = RequestMethod.GET)
public ResponseEntity<Group> getTheRootGroup() {
public ResponseEntity<Object> getTheRootGroup() {
LOGGER.info("Get root group and its whole sub-groups tree");
return ResponseEntity.ok(groupRepository.findOne(ROOT_CODE));
InputStream is = Variable.class.getClassLoader().getResourceAsStream(VARIABLES_FILE);
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
Object hierarchy = new Gson().fromJson(new JsonReader(br), Object.class);
return ResponseEntity.ok(hierarchy);
}
......
src/main/java/eu/hbp/mip/controllers/RequestsApi.java
+ 1
− 1
View file @ 3007ddc5
......@@ -32,7 +32,7 @@ public class RequestsApi {
) {
LOGGER.info("Post a request");
return ResponseEntity.ok(null);
return ResponseEntity.ok(null); // TODO: Get data from second datasource
}
}
src/main/java/eu/hbp/mip/controllers/StatsApi.java
+ 4
− 7
View file @ 3007ddc5
......@@ -4,14 +4,13 @@
package eu.hbp.mip.controllers;
import eu.hbp.mip.repositories.VariableRepository;
import eu.hbp.mip.model.GeneralStats;
import eu.hbp.mip.repositories.ArticleRepository;
import eu.hbp.mip.repositories.UserRepository;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import eu.hbp.mip.model.GeneralStats;
import eu.hbp.mip.repositories.ArticleRepository;
import eu.hbp.mip.repositories.UserRepository;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
......@@ -34,8 +33,6 @@ public class StatsApi {
@Autowired
ArticleRepository articleRepository;
@Autowired
VariableRepository variableRepository;
@ApiOperation(value = "Get general statistics", response = GeneralStats.class)
@ApiResponses(value = {@ApiResponse(code = 200, message = "Found"), @ApiResponse(code = 404, message = "Not found") })
......@@ -47,7 +44,7 @@ public class StatsApi {
stats.setUsers(userRepository.count());
stats.setArticles(articleRepository.count());
stats.setVariables(variableRepository.count());
stats.setVariables(0L); // TODO: compute from adni_merge DB
return ResponseEntity.ok(stats);
}
......
src/main/java/eu/hbp/mip/controllers/VariablesApi.java
+ 27
− 8
View file @ 3007ddc5
......@@ -5,15 +5,18 @@
package eu.hbp.mip.controllers;
import eu.hbp.mip.repositories.VariableRepository;
import io.swagger.annotations.*;
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import eu.hbp.mip.model.Value;
import eu.hbp.mip.model.Variable;
import io.swagger.annotations.*;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
......@@ -25,8 +28,8 @@ public class VariablesApi {
private static final Logger LOGGER = Logger.getLogger(VariablesApi.class);
@Autowired
VariableRepository variableRepository;
private static final String VARIABLES_FILE = "data/variables.json";
@ApiOperation(value = "Get variables", response = Variable.class, responseContainer = "List")
@ApiResponses(value = { @ApiResponse(code = 200, message = "Success") })
......@@ -41,7 +44,7 @@ public class VariablesApi {
) {
LOGGER.info("Get variables");
return ResponseEntity.ok(variableRepository.findAll());
return ResponseEntity.ok(null); // TODO : findall
}
@ApiOperation(value = "Get a variable", response = Variable.class)
......@@ -52,7 +55,7 @@ public class VariablesApi {
) {
LOGGER.info("Get a variable");
return ResponseEntity.ok(variableRepository.findOne(code));
return ResponseEntity.ok(null); // TODO findOne(code)
}
......@@ -65,7 +68,23 @@ public class VariablesApi {
) {
LOGGER.info("Get values from a variable");
return ResponseEntity.ok(variableRepository.findOne(code).getValues());
return ResponseEntity.ok(null); // TODO : findOne(code).getValues()
}
@ApiOperation(value = "Get groups and variables hierarchy", response = Variable.class)
@ApiResponses(value = { @ApiResponse(code = 200, message = "Found"), @ApiResponse(code = 404, message = "Not found") })
@RequestMapping(value = "/hierarchy", method = RequestMethod.GET)
public ResponseEntity<Object> getAVariable(
) {
LOGGER.info("Get groups and variables hierarchy");
InputStream is = Variable.class.getClassLoader().getResourceAsStream(VARIABLES_FILE);
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
Object hierarchy = new Gson().fromJson(new JsonReader(br), Object.class);
return ResponseEntity.ok(hierarchy);
}
......
src/main/java/eu/hbp/mip/repositories/GroupRepository.java deleted 100644 → 0
+ 0
− 11
View file @ 9d038ef9
package eu.hbp.mip.repositories;
import eu.hbp.mip.model.Group;
import org.springframework.data.repository.CrudRepository;
/**
* Created by mirco on 11.07.16.
*/
public interface GroupRepository extends CrudRepository<Group, String> {
}
src/main/java/eu/hbp/mip/repositories/VariableRepository.java deleted 100644 → 0
+ 0
− 11
View file @ 9d038ef9
package eu.hbp.mip.repositories;
import eu.hbp.mip.model.Variable;
import org.springframework.data.repository.CrudRepository;
/**
* Created by mirco on 11.07.16.
*/
public interface VariableRepository extends CrudRepository<Variable, String> {
}
src/main/java/eu/hbp/mip/utils/CSVUtil.java deleted 100644 → 0
+ 0
− 217
View file @ 9d038ef9
package eu.hbp.mip.utils;
import eu.hbp.mip.repositories.VariableRepository;
import org.apache.log4j.Logger;
import eu.hbp.mip.model.Dataset;
import eu.hbp.mip.model.Query;
import eu.hbp.mip.model.Variable;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Created by mirco on 29.01.16.
*/
public class CSVUtil {
private static final Logger LOGGER = Logger.getLogger(CSVUtil.class);
private static final String SEPARATOR = ",";
@Autowired
VariableRepository variableRepository;
public Dataset parseValues(String filename, Query query)
{
List<String[]> rows = getRows(filename);
Dataset result = new Dataset();
String code = generateDSCode(query);
Date date = new Date();
List<String> header = new LinkedList<>();
List<String> grouping = new LinkedList<>();
List<String> variable = new LinkedList<>();
Map<String, LinkedList<Object>> data = new HashMap<>();
List<Variable> covs = new LinkedList<>();
List<Variable> grps = new LinkedList<>();
List<Variable> vars = new LinkedList<>();
List<Variable> all = new LinkedList<>();
covs.addAll(query.getCovariables());
grps.addAll(query.getGrouping());
vars.addAll(query.getVariables());
all.addAll(query.getCovariables());
all.addAll(query.getGrouping());
all.addAll(query.getVariables());
header.addAll(covs.stream().map(Variable::getCode).collect(Collectors.toList()));
grouping.addAll(grps.stream().map(Variable::getCode).collect(Collectors.toList()));
variable.addAll(vars.stream().map(Variable::getCode).collect(Collectors.toList()));
try {
InputStream is = Dataset.class.getClassLoader().getResourceAsStream(filename);
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
String[] firstRow = br.readLine().split(SEPARATOR, -1);
br.close();
isr.close();
is.close();
for (Variable v : all) {
String type = getTypeFromDB(v);
String c = v.getCode();
int idx = find(c, firstRow);
List<Object> l = new LinkedList<>();
LinkedList<Object> ll = new LinkedList<>();
for (String[] row : rows) {
switch (type) {
case "T": {
String d = null;
String r = row[idx];
if (!r.isEmpty())
d = r;
l.add(d);
break;
}
case "I": {
Integer d;
try {
d = Integer.parseInt(row[idx]);
} catch (NumberFormatException e) {
try {
d = (int) Double.parseDouble(row[idx]); // Age
// for
// example
// has type
// I but is
// double
} catch (NumberFormatException e2) {
d = null; // No value found
}
}
l.add(d);
break;
}
case "N": {
Double d = null;
try {
d = Double.parseDouble(row[idx]);
} catch (NumberFormatException e) {
// should only happen when no data available ->
// we
// could have check this with
// !row[idx].isEmpty()
}
l.add(d);
break;
}
case "D": {
String d = null; // Formatted (ISO8601) String
// instead
// of Date
String r = row[idx];
if (!r.isEmpty())
d = r;
l.add(d);
break;
}
case "B": {
Boolean d;
d = !row[idx].isEmpty();
l.add(d);
break;
}
default: {
String d = null;
String r = row[idx];
if (!r.isEmpty())
d = r;
l.add(d);
break;
}
}
}
// TODO : should we remove this limit ???
if(l.size() > 100)
{
ll.addAll(l.subList(0,99));
}
data.put(c, ll);
}
} catch (IOException e) {
LOGGER.trace(e);
}
result.setCode(code);
result.setDate(date);
result.setHeader(header);
result.setGrouping(grouping);
result.setVariable(variable);
result.setData(data);
return result;
}
private List<String[]> getRows(String filename) {
List<String[]> rows = new LinkedList<>();
try {
InputStream is = CSVUtil.class.getClassLoader().getResourceAsStream(filename);
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
String[] firstRow = br.readLine().split(SEPARATOR, -1); // 1st row -> headers
for (String line = br.readLine(); line != null; line = br.readLine()) {
String[] row = line.split(SEPARATOR, -1);
rows.add(row);
}
br.close();
isr.close();
is.close();
} catch (IOException e) {
LOGGER.trace(e);
LOGGER.warn("A problem occured while trying to read a CSV file !");
}
return rows;
}
private int find(String code, String[] firstRow) {
for (int i = 0; i < firstRow.length; i++) {
if (firstRow[i].equals(code))
return i;
}
return -1;
}
private String generateDSCode(Query query) {
String prefix = "DS";
String queryStr = Integer.toString(query.hashCode());
String memId;
Pattern p = Pattern.compile("@(\\w+)");
Matcher m = p.matcher(queryStr);
if (m.find()) {
memId = m.group(1);
} else {
memId = Long.toString(new Date().getTime()); // In case a the regex fails (should not
// happen)
}
return prefix + memId;
}
private String getTypeFromDB(Variable v)
{
String type = variableRepository.findOne(v.getCode()).getType();
if(type == null)
{
type = "unknown";
}
return type;
}
}
src/main/resources/data/variables.json 0 → 100644
+ 2107
− 0
View file @ 3007ddc5
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment