From 11c4dd7e05fb439d6332db5a322d3cb73c9e343f Mon Sep 17 00:00:00 2001
From: stevereis <stevereis93@gmail.com>
Date: Mon, 27 Sep 2021 17:24:53 +0200
Subject: [PATCH] Mutation for transient WIP : Converter for exareme
---
.../engine/connectors/exareme/converters.ts | 91 ++++++++++
.../transient-data-result.interface.ts | 42 +++++
.../connectors/exareme/main.connector.ts | 169 ++++++++++++++++--
api/src/engine/engine.controller.ts | 5 -
api/src/engine/engine.interfaces.ts | 2 -
api/src/engine/models/entity.model.ts | 5 +-
.../models/result/common/metadata.model.ts | 10 ++
.../models/result/table-result.model.ts | 14 ++
.../transient/transient-create.input.ts | 14 +-
.../models/transient/transient.model.ts | 12 +-
.../engine/models/utility/extrafield.input.ts | 12 ++
api/src/schema.gql | 21 ++-
12 files changed, 370 insertions(+), 27 deletions(-)
create mode 100644 api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts
create mode 100644 api/src/engine/models/result/common/metadata.model.ts
create mode 100644 api/src/engine/models/result/table-result.model.ts
create mode 100644 api/src/engine/models/utility/extrafield.input.ts
diff --git a/api/src/engine/connectors/exareme/converters.ts b/api/src/engine/connectors/exareme/converters.ts
index 72e6204..5a77920 100644
--- a/api/src/engine/connectors/exareme/converters.ts
+++ b/api/src/engine/connectors/exareme/converters.ts
@@ -4,6 +4,12 @@ import { Variable } from 'src/engine/models/variable.model';
import { Hierarchy } from './interfaces/hierarchy.interface';
import { VariableEntity } from './interfaces/variable-entity.interface';
import { Entity } from './interfaces/entity.interface';
+import { TransientCreateInput } from 'src/engine/models/transient/transient-create.input';
+import { TransientDataResult } from './interfaces/transient/transient-data-result.interface';
+import { Transient } from 'src/engine/models/transient/transient.model';
+import { MetaData } from 'src/engine/models/result/common/metadata.model';
+import { TableResult } from 'src/engine/models/result/table-result.model';
+import { Dictionary } from 'src/common/interfaces/utilities.interface';
export const dataToGroup = (data: Hierarchy): Group => {
return {
@@ -33,3 +39,88 @@ export const dataToVariable = (data: VariableEntity): Variable => {
groups: [],
};
};
+
+export const transientInputToData = (data: TransientCreateInput) => {
+ return {
+ algorithm: {
+ parameters: [
+ {
+ name: 'dataset',
+ value: data.datasets.join(','),
+ },
+ {
+ name: 'y',
+ value: data.variables.join(','),
+ },
+ {
+ name: 'filter',
+ value: data.filter,
+ },
+ {
+ name: 'pathology',
+ value: data.domain,
+ },
+ ],
+ type: 'string',
+ name: 'DESCRIPTIVE_STATS',
+ },
+ name: 'Descriptive statistics',
+ };
+};
+
+const dictToTable = (dict: Dictionary<string[]>, rows: number): string[][] => {
+ const keys = Object.keys(dict);
+
+ return keys.map((key) => {
+ const row = Array.from(Array(rows).keys())
+ .map((i) => dict[key][i])
+ .map((val) => val ?? '');
+ row.unshift(key);
+ return row;
+ });
+};
+
+export const dataToTransient = (data: TransientDataResult): Transient => {
+ const result = data.result[0];
+ const tables = Object.keys(result.data.single).map((varKey): TableResult => {
+ const variable = result.data.single[varKey];
+ const domains: MetaData[] = [];
+ const rows: Dictionary<string[]> = {};
+
+ let count = 0;
+
+ Object.keys(variable).map((domainKey) => {
+ domains.push({ name: domainKey, type: 'string' });
+ const data = variable[domainKey];
+
+ [
+ [varKey, 'num_total'],
+ ['datapoints', 'num_datapoints'],
+ ['nulls', 'num_nulls'],
+ ].forEach((keys) => {
+ if (!rows[keys[0]]) rows[keys[0]] = [];
+ rows[keys[0]][count] = data[keys[1]];
+ });
+
+ const properties = variable[domainKey].data;
+
+ Object.keys(properties).forEach((propKey) => {
+ if (!rows[propKey]) rows[propKey] = [];
+ rows[propKey][count] = properties[propKey].toString();
+ });
+
+ count++;
+ });
+
+ return {
+ data: dictToTable(rows, count),
+ metadatas: domains,
+ name: varKey,
+ };
+ });
+
+ return {
+ title: data.name,
+ result: tables,
+ };
+};
diff --git a/api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts b/api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts
new file mode 100644
index 0000000..37a62f9
--- /dev/null
+++ b/api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts
@@ -0,0 +1,42 @@
+export interface NumericalData {
+ [key: string]: number;
+}
+
+export interface CategoricalData {
+ [key: string]: {
+ count: number;
+ percentage: number;
+ };
+}
+
+export interface TransientDataResult {
+ name: string;
+ result: [
+ {
+ data: {
+ single: {
+ [variable: string]: {
+ [dataset: string]: {
+ data: NumericalData | CategoricalData;
+ num_datapoints: number;
+ num_total: number;
+ num_nulls: number;
+ };
+ };
+ };
+ model: {
+ [dataset: string]: {
+ data: {
+ [variable: string]: {
+ [key: string]: number;
+ };
+ };
+ num_datapoints: number;
+ num_total: number;
+ num_nulls: number;
+ };
+ };
+ };
+ },
+ ];
+}
diff --git a/api/src/engine/connectors/exareme/main.connector.ts b/api/src/engine/connectors/exareme/main.connector.ts
index 30a7a6e..a4ecf13 100644
--- a/api/src/engine/connectors/exareme/main.connector.ts
+++ b/api/src/engine/connectors/exareme/main.connector.ts
@@ -8,9 +8,16 @@ import { Group } from 'src/engine/models/group.model';
import { TransientCreateInput } from 'src/engine/models/transient/transient-create.input';
import { Transient } from 'src/engine/models/transient/transient.model';
import { Variable } from 'src/engine/models/variable.model';
-import { dataToCategory, dataToGroup, dataToVariable } from './converters';
+import {
+ dataToCategory,
+ dataToGroup,
+ dataToTransient,
+ dataToVariable,
+ transientInputToData,
+} from './converters';
import { Hierarchy } from './interfaces/hierarchy.interface';
import { Pathology } from './interfaces/pathology.interface';
+import { TransientDataResult } from './interfaces/transient/transient-data-result.interface';
export default class ExaremeService implements IEngineService {
constructor(
@@ -18,11 +25,157 @@ export default class ExaremeService implements IEngineService {
private readonly httpService: HttpService,
) {}
- createTransient(data: TransientCreateInput): Transient | Promise<Transient> {
- return {
- id: 'test',
- label: 'test',
- };
+ async createTransient(data: TransientCreateInput): Promise<Transient> {
+ const form = transientInputToData(data);
+
+ const path = this.options.baseurl + 'experiments/transient';
+
+ const result = {
+ name: 'Descriptive statistics',
+ result: [
+ {
+ data: {
+ single: {
+ 'Left inferior temporal gyrus': {
+ ppmi: {
+ data: {
+ std: 1.2048783713787277,
+ max: 15.0815,
+ min: 7.6335,
+ mean: 11.38076218487395,
+ },
+ num_datapoints: 714,
+ num_total: 714,
+ num_nulls: 0,
+ },
+ edsd: {
+ data: {
+ std: 1.3274694970555183,
+ max: 14.593,
+ min: 5.4301,
+ mean: 10.647539816933637,
+ },
+ num_datapoints: 437,
+ num_total: 474,
+ num_nulls: 37,
+ },
+ 'desd-synthdata': {
+ data: {
+ std: 1.3479276642860987,
+ max: 14.593,
+ min: 5.4301,
+ mean: 10.685619565217392,
+ },
+ num_datapoints: 920,
+ num_total: 1000,
+ num_nulls: 80,
+ },
+ },
+ 'Left posterior insula': {
+ ppmi: {
+ data: {
+ std: 0.25046887396228024,
+ max: 3.0882,
+ min: 1.7073,
+ mean: 2.358402521008403,
+ },
+ num_datapoints: 714,
+ num_total: 714,
+ num_nulls: 0,
+ },
+ edsd: {
+ data: {
+ std: 0.2716090949138581,
+ max: 3.1971,
+ min: 1.2675,
+ mean: 2.2726512585812357,
+ },
+ num_datapoints: 437,
+ num_total: 474,
+ num_nulls: 37,
+ },
+ 'desd-synthdata': {
+ data: {
+ std: 0.2619310561946756,
+ max: 3.1971,
+ min: 1.2675,
+ mean: 2.27014597826087,
+ },
+ num_datapoints: 920,
+ num_total: 1000,
+ num_nulls: 80,
+ },
+ },
+ },
+ model: {
+ ppmi: {
+ num_datapoints: 714,
+ data: {
+ 'Left inferior temporal gyrus': {
+ std: 1.2048783713787277,
+ max: 15.0815,
+ min: 7.6335,
+ mean: 11.38076218487395,
+ },
+ 'Left posterior insula': {
+ std: 0.25046887396228024,
+ max: 3.0882,
+ min: 1.7073,
+ mean: 2.358402521008403,
+ },
+ },
+ num_total: 714,
+ num_nulls: 0,
+ },
+ edsd: {
+ num_datapoints: 437,
+ data: {
+ 'Left inferior temporal gyrus': {
+ std: 1.3274694970555183,
+ max: 14.593,
+ min: 5.4301,
+ mean: 10.647539816933637,
+ },
+ 'Left posterior insula': {
+ std: 0.2716090949138581,
+ max: 3.1971,
+ min: 1.2675,
+ mean: 2.2726512585812357,
+ },
+ },
+ num_total: 474,
+ num_nulls: 37,
+ },
+ 'desd-synthdata': {
+ num_datapoints: 920,
+ data: {
+ 'Left inferior temporal gyrus': {
+ std: 1.3479276642860987,
+ max: 14.593,
+ min: 5.4301,
+ mean: 10.685619565217392,
+ },
+ 'Left posterior insula': {
+ std: 0.2619310561946756,
+ max: 3.1971,
+ min: 1.2675,
+ mean: 2.27014597826087,
+ },
+ },
+ num_total: 1000,
+ num_nulls: 80,
+ },
+ },
+ },
+ },
+ ],
+ } as TransientDataResult;
+
+ /*= await firstValueFrom(
+ this.httpService.post<TransientDataResult>(path, form),
+ );*/
+
+ return dataToTransient(result);
}
async getDomains(ids: string[]): Promise<Domain[]> {
@@ -57,10 +210,6 @@ export default class ExaremeService implements IEngineService {
}
}
- demo(): string {
- return 'exareme';
- }
-
getActiveUser(): Observable<string> {
const path = this.options.baseurl + 'activeUser';
diff --git a/api/src/engine/engine.controller.ts b/api/src/engine/engine.controller.ts
index a495a2c..c54f0ce 100644
--- a/api/src/engine/engine.controller.ts
+++ b/api/src/engine/engine.controller.ts
@@ -19,11 +19,6 @@ export class EngineController {
@Inject(ENGINE_SERVICE) private readonly engineService: IEngineService,
) {}
- @Get('/test')
- getTest(): string {
- return this.engineService.demo();
- }
-
@Get('/algorithms')
getAlgorithms(@Req() request: Request): Observable<string> {
return this.engineService.getAlgorithms(request);
diff --git a/api/src/engine/engine.interfaces.ts b/api/src/engine/engine.interfaces.ts
index ec9bcd7..323e65e 100644
--- a/api/src/engine/engine.interfaces.ts
+++ b/api/src/engine/engine.interfaces.ts
@@ -10,8 +10,6 @@ export interface IEngineOptions {
}
export interface IEngineService {
- demo(): string;
-
getDomains(ids: string[]): Domain[] | Promise<Domain[]>;
getAlgorithms(request: Request): Observable<string>;
diff --git a/api/src/engine/models/entity.model.ts b/api/src/engine/models/entity.model.ts
index 70fc950..32183fd 100644
--- a/api/src/engine/models/entity.model.ts
+++ b/api/src/engine/models/entity.model.ts
@@ -1,10 +1,11 @@
-import { Field, ObjectType } from '@nestjs/graphql';
+import { Field, InputType, ObjectType } from '@nestjs/graphql';
+@InputType()
@ObjectType()
export class Entity {
@Field()
id: string;
@Field({ nullable: true })
- label: string;
+ label?: string;
}
diff --git a/api/src/engine/models/result/common/metadata.model.ts b/api/src/engine/models/result/common/metadata.model.ts
new file mode 100644
index 0000000..0c35ddd
--- /dev/null
+++ b/api/src/engine/models/result/common/metadata.model.ts
@@ -0,0 +1,10 @@
+import { Field, ObjectType } from '@nestjs/graphql';
+
+@ObjectType()
+export class MetaData {
+ @Field()
+ name: string;
+
+ @Field()
+ type: string;
+}
diff --git a/api/src/engine/models/result/table-result.model.ts b/api/src/engine/models/result/table-result.model.ts
new file mode 100644
index 0000000..3ad5cc7
--- /dev/null
+++ b/api/src/engine/models/result/table-result.model.ts
@@ -0,0 +1,14 @@
+import { Field, ObjectType } from '@nestjs/graphql';
+import { MetaData } from './common/metadata.model';
+
+@ObjectType()
+export class TableResult {
+ @Field()
+ name: string;
+
+ @Field(() => [[String]])
+ data: string[][];
+
+ @Field(() => [MetaData])
+ metadatas: MetaData[];
+}
diff --git a/api/src/engine/models/transient/transient-create.input.ts b/api/src/engine/models/transient/transient-create.input.ts
index d4b012c..0a2827e 100644
--- a/api/src/engine/models/transient/transient-create.input.ts
+++ b/api/src/engine/models/transient/transient-create.input.ts
@@ -1,7 +1,17 @@
import { Field, InputType } from '@nestjs/graphql';
+import { Extrafield } from '../utility/extrafield.input';
@InputType()
-export class TransientCreateInput {
+export class TransientCreateInput extends Extrafield {
+ @Field(() => [String])
+ datasets: string[];
+
+ @Field(() => [String])
+ variables: string[];
+
+ @Field(() => String, { nullable: true })
+ filter: string;
+
@Field()
- name: string;
+ domain: string;
}
diff --git a/api/src/engine/models/transient/transient.model.ts b/api/src/engine/models/transient/transient.model.ts
index 4831c0f..54d8e10 100644
--- a/api/src/engine/models/transient/transient.model.ts
+++ b/api/src/engine/models/transient/transient.model.ts
@@ -1,5 +1,11 @@
-import { ObjectType } from '@nestjs/graphql';
-import { Entity } from '../entity.model';
+import { Field, ObjectType } from '@nestjs/graphql';
+import { TableResult } from '../result/table-result.model';
@ObjectType()
-export class Transient extends Entity {}
+export class Transient {
+ @Field()
+ title: string;
+
+ @Field(() => [TableResult])
+ result: TableResult[];
+}
diff --git a/api/src/engine/models/utility/extrafield.input.ts b/api/src/engine/models/utility/extrafield.input.ts
new file mode 100644
index 0000000..22f9172
--- /dev/null
+++ b/api/src/engine/models/utility/extrafield.input.ts
@@ -0,0 +1,12 @@
+import { Field, InputType } from '@nestjs/graphql';
+
+/**
+ * This class is used to add a common extra field to
+ * an input class in order to provide specific
+ * information for a connector
+ */
+@InputType()
+export class Extrafield {
+ @Field({ nullable: true })
+ extradata: string;
+}
diff --git a/api/src/schema.gql b/api/src/schema.gql
index 5657a0d..e0d4db8 100644
--- a/api/src/schema.gql
+++ b/api/src/schema.gql
@@ -34,9 +34,20 @@ type Domain {
rootGroup: Group!
}
+type MetaData {
+ name: String!
+ type: String!
+}
+
+type TableResult {
+ name: String!
+ data: [[String!]!]!
+ metadatas: [MetaData!]!
+}
+
type Transient {
- id: String!
- label: String
+ title: String!
+ result: [TableResult!]!
}
type Query {
@@ -48,5 +59,9 @@ type Mutation {
}
input TransientCreateInput {
- name: String!
+ extradata: String
+ datasets: [String!]!
+ variables: [String!]!
+ filter: String
+ domain: String!
}
--
GitLab