From 4a35910dbd42f46aac8864db5aa3730514a8e761 Mon Sep 17 00:00:00 2001
From: Steve Reis <stevereis93@gmail.com>
Date: Tue, 21 Jun 2022 09:37:33 +0000
Subject: [PATCH] feat(datashield): Add datasets filter on histogram and
quantiles
---
.../connectors/datashield/main.connector.ts | 49 +++++++++++++------
.../connectors/datashield/transformations.ts | 2 +-
.../connectors/exareme/main.connector.ts | 1 +
api/src/engine/engine.interfaces.ts | 5 +-
api/src/engine/models/configuration.model.ts | 7 +++
api/src/schema.gql | 3 ++
6 files changed, 50 insertions(+), 17 deletions(-)
diff --git a/api/src/engine/connectors/datashield/main.connector.ts b/api/src/engine/connectors/datashield/main.connector.ts
index 487da9b..ef0a543 100644
--- a/api/src/engine/connectors/datashield/main.connector.ts
+++ b/api/src/engine/connectors/datashield/main.connector.ts
@@ -1,10 +1,5 @@
import { HttpService } from '@nestjs/axios';
-import {
- Inject,
- InternalServerErrorException,
- Logger,
- NotImplementedException,
-} from '@nestjs/common';
+import { Inject, InternalServerErrorException, Logger } from '@nestjs/common';
import { Request } from 'express';
import { catchError, firstValueFrom } from 'rxjs';
import {
@@ -45,7 +40,10 @@ export default class DataShieldService implements IEngineService {
) {}
getConfiguration(): IConfiguration {
- return {};
+ return {
+ hasGalaxy: false,
+ hasGrouping: false,
+ };
}
async login(username: string, password: string): Promise<User> {
@@ -81,12 +79,21 @@ export default class DataShieldService implements IEngineService {
}
async getAlgorithms(): Promise<Algorithm[]> {
- throw new NotImplementedException();
+ return [];
}
- async getHistogram(variable: string, cookie?: string): Promise<RawResult> {
- const path =
- this.options.baseurl + `histogram?var=${variable}&type=combine`;
+ async getHistogram(
+ variable: string,
+ datasets: string[],
+ cookie?: string,
+ ): Promise<RawResult> {
+ const url = new URL(this.options.baseurl + `histogram`);
+
+ url.searchParams.append('var', variable);
+ url.searchParams.append('type', 'combine');
+ url.searchParams.append('cohorts', datasets.join(','));
+
+ const path = url.href;
const response = await firstValueFrom(
this.httpService.get(path, {
@@ -97,7 +104,7 @@ export default class DataShieldService implements IEngineService {
);
if (response.data['global'] === undefined) {
- DataShieldService.logger.warn('Inconsistency on histogram result');
+ DataShieldService.logger.warn('Cannot parse histogram result');
DataShieldService.logger.verbose(path);
return {
rawdata: {
@@ -124,9 +131,16 @@ export default class DataShieldService implements IEngineService {
async getDescriptiveStats(
variable: string,
+ datasets: string[],
cookie?: string,
): Promise<TableResult> {
- const path = this.options.baseurl + `quantiles?var=${variable}&type=split`;
+ const url = new URL(this.options.baseurl + 'quantiles');
+
+ url.searchParams.append('var', variable);
+ url.searchParams.append('type', 'split');
+ url.searchParams.append('cohorts', datasets.join(','));
+
+ const path = url.href;
const response = await firstValueFrom(
this.httpService.get(path, {
@@ -167,14 +181,16 @@ export default class DataShieldService implements IEngineService {
switch (data.algorithm.id) {
case 'MULTIPLE_HISTOGRAMS': {
expResult.results = await Promise.all<RawResult>(
- data.variables.map((variable) => this.getHistogram(variable, cookie)),
+ data.variables.map((variable) =>
+ this.getHistogram(variable, expResult.datasets, cookie),
+ ),
);
break;
}
case 'DESCRIPTIVE_STATS': {
expResult.results = await Promise.all<TableResult>(
[...data.variables, ...data.coVariables].map((variable) =>
- this.getDescriptiveStats(variable, cookie),
+ this.getDescriptiveStats(variable, expResult.datasets, cookie),
),
);
break;
@@ -228,6 +244,9 @@ export default class DataShieldService implements IEngineService {
async getActiveUser(req: Request): Promise<User> {
const user = req.user as User;
+
+ if (!user) throw new InternalServerErrorException('User not found');
+
return {
username: user.id,
id: user.id,
diff --git a/api/src/engine/connectors/datashield/transformations.ts b/api/src/engine/connectors/datashield/transformations.ts
index db452ef..88c7abc 100644
--- a/api/src/engine/connectors/datashield/transformations.ts
+++ b/api/src/engine/connectors/datashield/transformations.ts
@@ -76,7 +76,7 @@ export const transformToTable = jsonata(`
},
"data": $.$each(function($v, $k) {
$not($k in $params) ? $append($k,$v.*) : undefined
- })
+ })[]
})
`);
diff --git a/api/src/engine/connectors/exareme/main.connector.ts b/api/src/engine/connectors/exareme/main.connector.ts
index 204ea2d..a2ca7a3 100644
--- a/api/src/engine/connectors/exareme/main.connector.ts
+++ b/api/src/engine/connectors/exareme/main.connector.ts
@@ -70,6 +70,7 @@ export default class ExaremeService implements IEngineService {
return {
contactLink: 'https://ebrains.eu/support/',
hasGalaxy: true,
+ hasGrouping: true,
};
}
diff --git a/api/src/engine/engine.interfaces.ts b/api/src/engine/engine.interfaces.ts
index f1cf66f..5edaca4 100644
--- a/api/src/engine/engine.interfaces.ts
+++ b/api/src/engine/engine.interfaces.ts
@@ -21,7 +21,10 @@ export interface IEngineOptions {
baseurl: string;
}
-export type IConfiguration = Pick<Configuration, 'contactLink' | 'hasGalaxy'>;
+export type IConfiguration = Pick<
+ Configuration,
+ 'contactLink' | 'hasGalaxy' | 'hasGrouping'
+>;
export interface IEngineService {
/**
diff --git a/api/src/engine/models/configuration.model.ts b/api/src/engine/models/configuration.model.ts
index 9471881..a7aeea0 100644
--- a/api/src/engine/models/configuration.model.ts
+++ b/api/src/engine/models/configuration.model.ts
@@ -8,6 +8,13 @@ export class Configuration {
@Field({ nullable: true, defaultValue: false })
hasGalaxy?: boolean;
+ @Field({
+ nullable: true,
+ defaultValue: false,
+ description: 'Indicates if histograms can handle grouping',
+ })
+ hasGrouping?: boolean;
+
@Field({ nullable: true })
contactLink?: string;
diff --git a/api/src/schema.gql b/api/src/schema.gql
index 4311e4d..57d7d6c 100644
--- a/api/src/schema.gql
+++ b/api/src/schema.gql
@@ -34,6 +34,9 @@ type Matomo {
type Configuration {
connectorId: String!
hasGalaxy: Boolean
+
+ """Indicates if histograms can handle grouping"""
+ hasGrouping: Boolean
contactLink: String
version: String!
skipAuth: Boolean
--
GitLab