diff --git a/api/src/engine/connectors/datashield/datashield.connector.ts b/api/src/engine/connectors/datashield/datashield.connector.ts
index 1f0cfd9000523cc293962565a26a4762fe6a1d19..0bbc8deb2e1ac039ae76f26b33ff79d982513a1a 100644
--- a/api/src/engine/connectors/datashield/datashield.connector.ts
+++ b/api/src/engine/connectors/datashield/datashield.connector.ts
@@ -15,10 +15,6 @@ import { Algorithm } from '../../../engine/models/experiment/algorithm.model';
import { AllowedLink } from '../../../engine/models/experiment/algorithm/nominal-parameter.model';
import { Experiment } from '../../../engine/models/experiment/experiment.model';
import { AlertLevel } from '../../../engine/models/result/alert-result.model';
-import {
- TableResult,
- TableStyle,
-} from '../../../engine/models/result/table-result.model';
import { Variable } from '../../../engine/models/variable.model';
import { ExperimentCreateInput } from '../../../experiments/models/input/experiment-create.input';
import { User } from '../../../users/models/user.model';
@@ -28,8 +24,6 @@ import {
dsGroup,
transformToDomain,
transformToHisto,
- transformToTable,
- transformToTableNominal,
transfoToHistoNominal as transformToHistoNominal,
} from './transformations';
@@ -184,59 +178,30 @@ export default class DataShieldConnector implements Connector {
}
async getDescriptiveStats(
- variable: Variable,
- datasets: string[],
- cookie?: string,
- ): Promise<TableResult> {
- const url = new URL(this.options.baseurl + 'quantiles');
+ experiment: Experiment,
+ vars: Variable[],
+ cookie: string,
+ ) {
+ const url = new URL(this.options.baseurl + 'descriptivestats');
+ const { variables, coVariables, datasets } = experiment;
- url.searchParams.append('var', variable.id);
- url.searchParams.append('type', 'split');
- url.searchParams.append('cohorts', datasets.join(','));
+ const inputData = {
+ variables,
+ covariables: coVariables,
+ datasets,
+ };
const path = url.href;
- const response = await firstValueFrom(
- this.httpService.get(path, {
+ const { data } = await firstValueFrom(
+ this.httpService.post(path, inputData, {
headers: {
cookie,
},
}),
);
- const title = variable.label ?? variable.id;
- const data = { ...response.data, title };
-
- const table = (
- variable.enumerations
- ? transformToTableNominal.evaluate(data)
- : transformToTable.evaluate(data)
- ) as TableResult;
-
- if (
- table &&
- table.headers &&
- variable.type === 'nominal' &&
- variable.enumerations
- ) {
- table.headers = table.headers.map((header) => {
- const category = variable.enumerations.find(
- (v) => v.value === header.name,
- );
-
- if (!category || !category.label) return header;
-
- return {
- ...header,
- name: category.label,
- };
- });
- }
-
- return {
- ...table,
- tableStyle: TableStyle.DEFAULT,
- };
+ handlers(experiment, data, vars);
}
async runExperiment(
@@ -258,6 +223,7 @@ export default class DataShieldConnector implements Connector {
name: data.name,
domain: data.domain,
datasets: data.datasets,
+ results: [],
algorithm: {
name: data.algorithm.id,
parameters: data.algorithm.parameters.map((p) => ({
@@ -285,23 +251,10 @@ export default class DataShieldConnector implements Connector {
break;
}
case 'DESCRIPTIVE_STATS': {
- // Cannot be done in parallel because Datashield API has an issue with parallel request (response mismatching)
- const results = [];
- for (const variable of allVariables) {
- const result = await this.getDescriptiveStats(
- variable,
- expResult.datasets,
- cookie,
- );
-
- results.push(result);
- }
-
- expResult.results = results;
+ await this.getDescriptiveStats(expResult, allVariables, cookie);
break;
}
default: {
- expResult.results = [];
await this.runAlgorithm(expResult, allVariables, cookie);
}
}
@@ -320,33 +273,33 @@ export default class DataShieldConnector implements Connector {
const path = new URL('/runAlgorithm', this.options.baseurl);
// Covariable and variable are inversed in Datashield API
- const coVariable =
+ const variable =
experiment.variables.length > 0 ? experiment.variables[0] : undefined;
const expToInput = {
algorithm: {
id: experiment.algorithm.name,
- coVariable,
- variables: experiment.coVariables,
+ variable,
+ covariables: experiment.coVariables,
},
datasets: experiment.datasets,
};
experiment.algorithm.parameters?.forEach((param) => {
if (!expToInput.algorithm[param.name]) {
- // FIXME: the parameter should be added in a specific key entry (e.g. expToInput.algorithm.parameters')
+ // FIXME: Parameters should be added in a specific key entry (e.g. expToInput.algorithm.parameters')
// Should be fixed inside the Datashield API
expToInput.algorithm[param.name] = param.value;
}
});
- const result = await firstValueFrom(
+ const { data } = await firstValueFrom(
this.httpService.post(path.href, expToInput, {
headers: { cookie, 'Content-Type': 'application/json' },
}),
);
- handlers(experiment, result.data, vars);
+ handlers(experiment, data, vars);
}
async logout(request: Request): Promise<void> {
diff --git a/api/src/engine/connectors/datashield/handlers/algorithms/descriptive.handler.spec.ts b/api/src/engine/connectors/datashield/handlers/algorithms/descriptive.handler.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..9e3620eb8175b19ae4e67bda4fdcc081eca9f24b
--- /dev/null
+++ b/api/src/engine/connectors/datashield/handlers/algorithms/descriptive.handler.spec.ts
@@ -0,0 +1,81 @@
+import { Experiment } from '../../../../models/experiment/experiment.model';
+import { HeatMapResult } from '../../../../models/result/heat-map-result.model';
+import DescriptiveHandler from './descriptive.handler';
+
+const data = {
+ quants: {
+ 'Albumin..Mass.volume..in.Serum.or.Plasma.global': [
+ 3.805, 3.91, 4.1, 4.2, 4.3, 4.6, 5.095, 4.2525,
+ ],
+ 'Alanine.aminotransferase..Enzymatic.activity.volume..in.Serum.or.Plasma.global':
+ [14.205, 14.9, 16.325, 20.5, 24.15, 35.08, 40.76, 22.4257],
+ },
+ heatmaps: [
+ [
+ {
+ '1': [
+ [0, 1, -2],
+ [-3, 4, 5],
+ [6, -7, 8],
+ ],
+ xlab: 'Albumin..Mass.volume..in.Serum.or.Plasma',
+ ylab: 'Alanine.aminotransferase..Enzymatic.activity.volume..in.Serum.or.Plasma',
+ x: [
+ 3.4371, 3.5494, 3.6617, 3.774, 3.8863, 3.9986, 4.1109, 4.2233, 4.3356,
+ 4.4479,
+ ],
+ y: [5.2062, 11.7548, 18.3034, 24.852, 31.4006, 37.9492, 44.4978],
+ },
+ ],
+ ],
+};
+
+const invMatrix = [
+ [0, -3, 6],
+ [1, 4, -7],
+ [-2, 5, 8],
+];
+
+const createExp = (): Experiment => ({
+ id: 'dummy-id',
+ name: 'Testing purpose',
+ algorithm: {
+ name: DescriptiveHandler.ALGO_NAME,
+ },
+ datasets: ['dataset1', 'dataset2'],
+ domain: 'sophia',
+ variables: [
+ 'Alanine.aminotransferase..Enzymatic.activity.volume..in.Serum.or.Plasma',
+ ],
+ coVariables: ['Albumin..Mass.volume..in.Serum.or.Plasma.global'],
+ results: [],
+});
+
+describe('Descriptive handler', () => {
+ let descriptiveHandler: DescriptiveHandler;
+ let exp: Experiment;
+
+ beforeEach(() => {
+ descriptiveHandler = new DescriptiveHandler();
+ exp = createExp();
+ });
+
+ describe('handle standard', () => {
+ it('should output 3 results', () => {
+ descriptiveHandler.handle(exp, data, []);
+ const heatmapResult = exp.results[2] as HeatMapResult;
+
+ expect(exp.results).toHaveLength(3);
+ expect(heatmapResult.matrix).toStrictEqual(invMatrix);
+ });
+ });
+
+ describe('handle without heatmaps', () => {
+ it('should output only 2 results', () => {
+ const specificData = { ...data, heatmaps: [] };
+ descriptiveHandler.handle(exp, specificData, []);
+
+ expect(exp.results).toHaveLength(2);
+ });
+ });
+});
diff --git a/api/src/engine/connectors/datashield/handlers/algorithms/descriptive.handler.ts b/api/src/engine/connectors/datashield/handlers/algorithms/descriptive.handler.ts
new file mode 100644
index 0000000000000000000000000000000000000000..f0f017443294f0fa89ad80c1f9bbcbabfafe231c
--- /dev/null
+++ b/api/src/engine/connectors/datashield/handlers/algorithms/descriptive.handler.ts
@@ -0,0 +1,123 @@
+import * as jsonata from 'jsonata';
+import { Experiment } from '../../../../models/experiment/experiment.model';
+import {
+ AlertLevel,
+ AlertResult,
+} from '../../../../models/result/alert-result.model';
+import { Variable } from '../../../../models/variable.model';
+import BaseHandler from '../base.handler';
+
+const transformToDescriptiveStats = jsonata(`
+(
+ $clearLabel := function($label) {
+ $trim($replace($label, '.', ' '))
+ };
+
+ $histoNumber := function($v, $k) {
+ {
+ "name": $clearLabel($k),
+ "headers": ['5%','10%','25%','50%','75%','90%','95%','Mean'].{
+ "name": $,
+ "type": "string"
+ },
+ "data": [[$v]]
+ }
+ };
+
+ $histoNominal := function($v, $k) {
+ {
+ "name": $clearLabel($k),
+ "headers": $keys($v).{
+ "name": $,
+ "type": "string"
+ },
+ "data": [[$v.*]]
+ }
+ };
+
+ $append(quants.$each(function($v, $k) {(
+ $t := $type($v);
+ $t = "array" ? $histoNumber($v, $k) : ($t = "object" ? $histoNominal($v, $k) : undefined)
+ )}), heatmaps.(
+ $.'1' ? {
+ "name": '',
+ "xAxis": {
+ "label": $clearLabel($.xlab),
+ "categories": $.x
+ },
+ "yAxis": {
+ "label": $clearLabel($.ylab),
+ "categories": $.y
+ },
+ "matrix": $transposeMat($.'1')
+ } : []))
+)
+`);
+
+transformToDescriptiveStats.registerFunction(
+ 'transposeMat',
+ (matrix) => {
+ if (!matrix) return [[]];
+
+ const invMatrix = [];
+
+ for (let i = 0; i < matrix[0].length; i++) {
+ invMatrix.push([]);
+ for (const elem of matrix) {
+ invMatrix[i].push(elem[i]);
+ }
+ }
+
+ return invMatrix;
+ },
+ '<a<a<n>:a<a<n>>',
+);
+
+export default class DescriptiveHandler extends BaseHandler {
+ public static readonly ALGO_NAME = 'descriptive_stats';
+
+ canHandle(algorithm: string, data: unknown): boolean {
+ return (
+ algorithm.toLowerCase() === DescriptiveHandler.ALGO_NAME &&
+ data &&
+ (data['quants'] || data['heatmaps'])
+ );
+ }
+
+ getErrors(data: unknown): AlertResult[] {
+ const errors = [];
+ if (
+ data['heatmaps'] &&
+ data['heatmaps'][0] &&
+ typeof data['heatmaps'][0][0] === 'string'
+ ) {
+ errors.push({
+ message: 'Heatmaps error: ' + data['heatmaps'][0][0],
+ level: AlertLevel.ERROR,
+ });
+ }
+
+ if (data['quants']) {
+ for (const [key, value] of Object.entries(data['quants'])) {
+ if (typeof value === 'string') {
+ errors.push({
+ message: `Table '${key}' error: ` + value,
+ level: AlertLevel.ERROR,
+ });
+ }
+ }
+ }
+
+ return errors;
+ }
+
+ handle(experiment: Experiment, data: unknown, vars: Variable[]): void {
+ if (!this.canHandle(experiment.algorithm.name, data))
+ return this.next?.handle(experiment, data, vars);
+
+ const results = transformToDescriptiveStats.evaluate(data);
+ const errors = this.getErrors(data);
+
+ experiment.results.push(...errors, ...results);
+ }
+}
diff --git a/api/src/engine/connectors/datashield/handlers/index.ts b/api/src/engine/connectors/datashield/handlers/index.ts
index 9948a2fcf9d6e9a9c6db071210202530a7a418e1..f15627552879b6f8755d3bee2403665e5036ffca 100644
--- a/api/src/engine/connectors/datashield/handlers/index.ts
+++ b/api/src/engine/connectors/datashield/handlers/index.ts
@@ -1,5 +1,6 @@
import { Variable } from 'src/engine/models/variable.model';
import { Experiment } from '../../../../engine/models/experiment/experiment.model';
+import DescriptiveHandler from './algorithms/descriptive.handler';
import ErrorAlgorithmHandler from './algorithms/error-algorithm.handler';
import LinearRegressionHandler from './algorithms/linear-regression.handler';
import LogisticRegressionHandler from './algorithms/logistic-regression.handler';
@@ -8,6 +9,7 @@ import TerminalAlgorithmHandler from './algorithms/terminal-algorithm.handler';
const start = new ErrorAlgorithmHandler();
start
+ .setNext(new DescriptiveHandler())
.setNext(new LinearRegressionHandler())
.setNext(new LogisticRegressionHandler())
.setNext(new TerminalAlgorithmHandler());