diff --git a/api/.eslintrc.js b/api/.eslintrc.js index f6c62bee279534a2a2ce7ee4424670655fbd2a4c..8e34f1ea92b73633f2be969865474d6ca4e5cb9c 100644 --- a/api/.eslintrc.js +++ b/api/.eslintrc.js @@ -20,5 +20,6 @@ module.exports = { '@typescript-eslint/explicit-function-return-type': 'off', '@typescript-eslint/explicit-module-boundary-types': 'off', '@typescript-eslint/no-explicit-any': 'off', + 'prettier/prettier': ['error', { "endOfLine": "auto"}, { usePrettierrc: true }], }, }; diff --git a/api/package-lock.json b/api/package-lock.json index a0fb8a8365088a38d2f372e8de42ac083a77386a..6f50c98fcb8f360b97b14a52408e77063f952e33 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -19,6 +19,7 @@ "apollo-server-express": "^3.3.0", "axios": "^0.21.1", "graphql": "^15.5.3", + "jsonata": "^1.8.5", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.2.0" @@ -7172,6 +7173,14 @@ "node": ">=6" } }, + "node_modules/jsonata": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/jsonata/-/jsonata-1.8.5.tgz", + "integrity": "sha512-ilDyTBkg6qhNoNVr8PUPzz5GYvRK+REKOM5MdOGzH2y6V4yvPRMegSvbZLpbTtI0QAgz09QM7drDhSHUlwp9pA==", + "engines": { + "node": ">= 8" + } + }, "node_modules/jsonc-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz", @@ -16079,6 +16088,11 @@ "minimist": "^1.2.5" } }, + "jsonata": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/jsonata/-/jsonata-1.8.5.tgz", + "integrity": "sha512-ilDyTBkg6qhNoNVr8PUPzz5GYvRK+REKOM5MdOGzH2y6V4yvPRMegSvbZLpbTtI0QAgz09QM7drDhSHUlwp9pA==" + }, "jsonc-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz", diff --git a/api/package.json b/api/package.json index 848e67878898649298ce812ddcef10ec95c47dc1..fd94af7dcea0a56fcf453b755939a3a373c8b459 100644 --- a/api/package.json +++ b/api/package.json @@ -32,6 +32,7 @@ "apollo-server-express": "^3.3.0", "axios": "^0.21.1", "graphql": "^15.5.3", + "jsonata": "^1.8.5", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.2.0" @@ -76,4 +77,4 @@ "coverageDirectory": "../coverage", "testEnvironment": "node" } -} \ No newline at end of file +} diff --git a/api/src/engine/connectors/datashield/main.connector.ts b/api/src/engine/connectors/datashield/main.connector.ts index c81cc65fbef911bcb160acc7bdb516ec3aa80e1d..1cdbb509c93a78794235c371c6fb3a62076b4792 100644 --- a/api/src/engine/connectors/datashield/main.connector.ts +++ b/api/src/engine/connectors/datashield/main.connector.ts @@ -1,8 +1,15 @@ import { Observable } from 'rxjs'; import { IEngineService } from 'src/engine/engine.interfaces'; import { Domain } from 'src/engine/models/domain.model'; +import { ExperimentCreateInput } from 'src/engine/models/experiment/experiment-create.input'; +import { Experiment } from 'src/engine/models/experiment/experiment.model'; export default class DataShieldService implements IEngineService { + createTransient( + data: ExperimentCreateInput, + ): Experiment | Promise<Experiment> { + throw new Error('Method not implemented.'); + } getDomains(): Domain[] { throw new Error('Method not implemented.'); } diff --git a/api/src/engine/connectors/exareme/converters.ts b/api/src/engine/connectors/exareme/converters.ts index 72e62044aa2241b824b46d376ff8c14c31b35779..cb1c884c6101bec8de814accda529690cb92896f 100644 --- a/api/src/engine/connectors/exareme/converters.ts +++ b/api/src/engine/connectors/exareme/converters.ts @@ -1,9 +1,14 @@ import { Category } from 'src/engine/models/category.model'; +import { ExperimentCreateInput } from 'src/engine/models/experiment/experiment-create.input'; +import { Experiment } from 'src/engine/models/experiment/experiment.model'; import { Group } from 'src/engine/models/group.model'; +import { TableResult } from 'src/engine/models/result/table-result.model'; import { Variable } from 'src/engine/models/variable.model'; +import { Entity } from './interfaces/entity.interface'; import { Hierarchy } from './interfaces/hierarchy.interface'; +import { TransientDataResult } from './interfaces/transient/transient-data-result.interface'; import { VariableEntity } from './interfaces/variable-entity.interface'; -import { Entity } from './interfaces/entity.interface'; +import { transientToTable } from './transformations'; export const dataToGroup = (data: Hierarchy): Group => { return { @@ -33,3 +38,40 @@ export const dataToVariable = (data: VariableEntity): Variable => { groups: [], }; }; + +export const experimentInputToData = (data: ExperimentCreateInput) => { + return { + algorithm: { + parameters: [ + { + name: 'dataset', + value: data.datasets.join(','), + }, + { + name: 'y', + value: data.variables.join(','), + }, + { + name: 'filter', + value: data.filter, + }, + { + name: 'pathology', + value: data.domain, + }, + ], + type: 'string', + name: data.algorithm, + }, + name: data.name, + }; +}; + +export const dataToTransient = (data: TransientDataResult): Experiment => { + const tabs: TableResult[] = transientToTable.evaluate(data); + + return { + title: data.name, + results: tabs, + }; +}; diff --git a/api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts b/api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts new file mode 100644 index 0000000000000000000000000000000000000000..37a62f96b071c594696f01bd3b27d75d2bc25f67 --- /dev/null +++ b/api/src/engine/connectors/exareme/interfaces/transient/transient-data-result.interface.ts @@ -0,0 +1,42 @@ +export interface NumericalData { + [key: string]: number; +} + +export interface CategoricalData { + [key: string]: { + count: number; + percentage: number; + }; +} + +export interface TransientDataResult { + name: string; + result: [ + { + data: { + single: { + [variable: string]: { + [dataset: string]: { + data: NumericalData | CategoricalData; + num_datapoints: number; + num_total: number; + num_nulls: number; + }; + }; + }; + model: { + [dataset: string]: { + data: { + [variable: string]: { + [key: string]: number; + }; + }; + num_datapoints: number; + num_total: number; + num_nulls: number; + }; + }; + }; + }, + ]; +} diff --git a/api/src/engine/connectors/exareme/main.connector.ts b/api/src/engine/connectors/exareme/main.connector.ts index e8d2bf84d664208c2de2747f7017a39113b00f79..aa0b30f4bb9cab2cb917062b5dcc222481bc042e 100644 --- a/api/src/engine/connectors/exareme/main.connector.ts +++ b/api/src/engine/connectors/exareme/main.connector.ts @@ -4,11 +4,20 @@ import { Request } from 'express'; import { firstValueFrom, map, Observable } from 'rxjs'; import { IEngineOptions, IEngineService } from 'src/engine/engine.interfaces'; import { Domain } from 'src/engine/models/domain.model'; +import { ExperimentCreateInput } from 'src/engine/models/experiment/experiment-create.input'; +import { Experiment } from 'src/engine/models/experiment/experiment.model'; import { Group } from 'src/engine/models/group.model'; import { Variable } from 'src/engine/models/variable.model'; -import { dataToCategory, dataToGroup, dataToVariable } from './converters'; +import { + dataToCategory, + dataToGroup, + dataToTransient, + dataToVariable, + experimentInputToData, +} from './converters'; import { Hierarchy } from './interfaces/hierarchy.interface'; import { Pathology } from './interfaces/pathology.interface'; +import { TransientDataResult } from './interfaces/transient/transient-data-result.interface'; export default class ExaremeService implements IEngineService { constructor( @@ -16,6 +25,18 @@ export default class ExaremeService implements IEngineService { private readonly httpService: HttpService, ) {} + async createTransient(data: ExperimentCreateInput): Promise<Experiment> { + const form = experimentInputToData(data); + + const path = this.options.baseurl + 'experiments/transient'; + + const resultAPI = await firstValueFrom( + this.httpService.post<TransientDataResult>(path, form), + ); + + return dataToTransient(resultAPI.data); + } + async getDomains(ids: string[]): Promise<Domain[]> { const path = this.options.baseurl + 'pathologies'; @@ -48,10 +69,6 @@ export default class ExaremeService implements IEngineService { } } - demo(): string { - return 'exareme'; - } - getActiveUser(): Observable<string> { const path = this.options.baseurl + 'activeUser'; diff --git a/api/src/engine/connectors/exareme/transformations.ts b/api/src/engine/connectors/exareme/transformations.ts new file mode 100644 index 0000000000000000000000000000000000000000..554bfda10c5e733cb7f533e1cda87ba461e605e8 --- /dev/null +++ b/api/src/engine/connectors/exareme/transformations.ts @@ -0,0 +1,42 @@ +// This file contains all transformation queries for JSONata +// see : https://docs.jsonata.org/ + +import * as jsonata from 'jsonata'; // old import style needed because of 'export = jsonata' + +export const transientToTable = jsonata(` +( + $e := function($x) {($x != null) ? $x : ''}; + + $fn := function($o, $prefix) { + $each($o, function($v, $k) {( + $name := $join([$prefix,$k], '/'); + $type($v) = 'object' ? $fn($v, $name): { + $name: $v + } + )}) ~> $merge() + }; + + result.data.[ + $.single.*@$p#$i.{ + 'groupBy' : 'single', + 'name': $keys(%)[$i], + 'metadatas': $append("", $keys(*)).{ + 'name': $, + 'type': 'string' + }, + 'data' : [ + [$keys(%)[$i], $p.*.($e(num_total))], + ['Datapoints', $p.*.($e(num_datapoints))], + ['Nulls', $p.*.($e(num_nulls))], + $p.*.data.($fn($)) ~> $reduce(function($a, $b) { + $each($a, function($v, $k) {( + { + $k : [$v, $e($lookup($b,$k))] + } + )}) ~> $merge() + }) ~> $each(function($v, $k) {$append($k,$v)}) + ] + } + ] +) +`); diff --git a/api/src/engine/engine.controller.ts b/api/src/engine/engine.controller.ts index a495a2c317efa7911dc3f2e43915c88f9fb13830..c54f0ce0fa44732ed4fc614caf5ff0509d6e0068 100644 --- a/api/src/engine/engine.controller.ts +++ b/api/src/engine/engine.controller.ts @@ -19,11 +19,6 @@ export class EngineController { @Inject(ENGINE_SERVICE) private readonly engineService: IEngineService, ) {} - @Get('/test') - getTest(): string { - return this.engineService.demo(); - } - @Get('/algorithms') getAlgorithms(@Req() request: Request): Observable<string> { return this.engineService.getAlgorithms(request); diff --git a/api/src/engine/engine.interfaces.ts b/api/src/engine/engine.interfaces.ts index 09b86b731d670a83b598936d834b8993f4a8b27f..0d90e92e62de3e342cc206f4579a2bf399df1f36 100644 --- a/api/src/engine/engine.interfaces.ts +++ b/api/src/engine/engine.interfaces.ts @@ -1,6 +1,8 @@ import { Request } from 'express'; import { Observable } from 'rxjs'; import { Domain } from './models/domain.model'; +import { ExperimentCreateInput } from './models/experiment/experiment-create.input'; +import { Experiment } from './models/experiment/experiment.model'; export interface IEngineOptions { type: string; @@ -8,8 +10,6 @@ export interface IEngineOptions { } export interface IEngineService { - demo(): string; - getDomains(ids: string[]): Domain[] | Promise<Domain[]>; getAlgorithms(request: Request): Observable<string>; @@ -24,6 +24,10 @@ export interface IEngineService { startExperimentTransient(request: Request): Observable<string>; + createTransient( + data: ExperimentCreateInput, + ): Promise<Experiment> | Experiment; + startExperiment(request: Request): Observable<string>; getActiveUser(request: Request): Observable<string>; diff --git a/api/src/engine/engine.resolver.ts b/api/src/engine/engine.resolver.ts index beeb589fc6de4f3294efffea16e4b55c1ac96f61..e3de433e113e0e13577372240cda4114aa02a193 100644 --- a/api/src/engine/engine.resolver.ts +++ b/api/src/engine/engine.resolver.ts @@ -1,8 +1,10 @@ import { Inject } from '@nestjs/common'; -import { Args, Query, Resolver } from '@nestjs/graphql'; +import { Args, Mutation, Query, Resolver } from '@nestjs/graphql'; import { ENGINE_SERVICE } from './engine.constants'; import { IEngineService } from './engine.interfaces'; import { Domain } from './models/domain.model'; +import { ExperimentCreateInput } from './models/experiment/experiment-create.input'; +import { Experiment } from './models/experiment/experiment.model'; @Resolver() export class EngineResolver { @@ -17,4 +19,11 @@ export class EngineResolver { ) { return this.engineService.getDomains(ids); } + + @Mutation(() => Experiment) + async createTransient( + @Args('data') experimentCreateInput: ExperimentCreateInput, + ) { + return this.engineService.createTransient(experimentCreateInput); + } } diff --git a/api/src/engine/models/category.model.ts b/api/src/engine/models/category.model.ts index 877acc5c3c4f5cb3eaf19b05f71879950b69dd76..921ebfce8d518a66a484bf44d04cf330b8091915 100644 --- a/api/src/engine/models/category.model.ts +++ b/api/src/engine/models/category.model.ts @@ -1,10 +1,5 @@ -import { Field, ObjectType } from '@nestjs/graphql'; +import { ObjectType } from '@nestjs/graphql'; +import { Entity } from './entity.model'; @ObjectType() -export class Category { - @Field() - id: string; - - @Field() - label: string; -} +export class Category extends Entity {} diff --git a/api/src/engine/models/entity.model.ts b/api/src/engine/models/entity.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..32183fd7e970410a1ee3f479cef4d691d56fb7a7 --- /dev/null +++ b/api/src/engine/models/entity.model.ts @@ -0,0 +1,11 @@ +import { Field, InputType, ObjectType } from '@nestjs/graphql'; + +@InputType() +@ObjectType() +export class Entity { + @Field() + id: string; + + @Field({ nullable: true }) + label?: string; +} diff --git a/api/src/engine/models/experiment/experiment-create.input.ts b/api/src/engine/models/experiment/experiment-create.input.ts new file mode 100644 index 0000000000000000000000000000000000000000..d871eead0d40690fda47ec6b8ee5ff76bd173e6d --- /dev/null +++ b/api/src/engine/models/experiment/experiment-create.input.ts @@ -0,0 +1,22 @@ +import { Field, InputType } from '@nestjs/graphql'; + +@InputType() +export class ExperimentCreateInput { + @Field(() => [String]) + datasets: string[]; + + @Field(() => [String]) + variables: string[]; + + @Field(() => String, { nullable: true }) + filter: string; + + @Field() + domain: string; + + @Field() + algorithm: string; + + @Field() + name: string; +} diff --git a/api/src/engine/models/experiment/experiment.model.ts b/api/src/engine/models/experiment/experiment.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..bf0881247afdf3b13f8e9e29e9a3653b1cb2d69d --- /dev/null +++ b/api/src/engine/models/experiment/experiment.model.ts @@ -0,0 +1,44 @@ +import { + createUnionType, + Field, + GraphQLISODateTime, + ObjectType, +} from '@nestjs/graphql'; +import { DummyResult } from '../result/dummy-result.model'; +import { TableResult } from '../result/table-result.model'; + +export const ResultUnion = createUnionType({ + name: 'ResultUnion', + types: () => [TableResult, DummyResult], + resolveType(value) { + if (value.metadatas) { + return TableResult; + } + if (value.listMax) { + return DummyResult; + } + + return null; + }, +}); + +@ObjectType() +export class Experiment { + @Field() + title: string; + + @Field({ nullable: true }) + uuid?: string; + + @Field(() => GraphQLISODateTime, { nullable: true }) + created_at?: Date; + + @Field(() => GraphQLISODateTime, { nullable: true }) + update_at?: Date; + + @Field(() => GraphQLISODateTime, { nullable: true }) + finished_at?: Date; + + @Field(() => [ResultUnion]) + results: Array<typeof ResultUnion>; +} diff --git a/api/src/engine/models/group.model.ts b/api/src/engine/models/group.model.ts index d4b45f741c1a9b4b089e8187a136154bd3b695d7..c2087ba2c6023a4a3d7f17110e3e739709d08a90 100644 --- a/api/src/engine/models/group.model.ts +++ b/api/src/engine/models/group.model.ts @@ -1,14 +1,9 @@ import { Field, ObjectType } from '@nestjs/graphql'; +import { Entity } from './entity.model'; import { Variable } from './variable.model'; @ObjectType() -export class Group { - @Field() - id: string; - - @Field() - label: string; - +export class Group extends Entity { @Field({ nullable: true }) description?: string; diff --git a/api/src/engine/models/result/common/metadata.model.ts b/api/src/engine/models/result/common/metadata.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..9cbeaf8b5967d42f6a01a8e4f5a55b262c305f4a --- /dev/null +++ b/api/src/engine/models/result/common/metadata.model.ts @@ -0,0 +1,10 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +@ObjectType() +export class Metadata { + @Field() + name: string; + + @Field() + type: string; +} diff --git a/api/src/engine/models/result/common/result.model.ts b/api/src/engine/models/result/common/result.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..ca86bfb4e109c44d0d4c06ea0e70ac05e3bf656a --- /dev/null +++ b/api/src/engine/models/result/common/result.model.ts @@ -0,0 +1,7 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +@ObjectType() +export abstract class Result { + @Field({ nullable: true }) + groupBy?: string; +} diff --git a/api/src/engine/models/result/dummy-result.model.ts b/api/src/engine/models/result/dummy-result.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..99e36f5c90b9100e1cd46ed9fa67f613a46864b5 --- /dev/null +++ b/api/src/engine/models/result/dummy-result.model.ts @@ -0,0 +1,14 @@ +import { Field, ObjectType } from '@nestjs/graphql'; +import { Result } from './common/result.model'; + +@ObjectType() +export class DummyResult extends Result { + @Field() + name: string; + + @Field(() => [[String]]) + data: string[][]; + + @Field(() => [String]) + listMax: string[]; +} diff --git a/api/src/engine/models/result/table-result.model.ts b/api/src/engine/models/result/table-result.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..1ebb6a1726aa51c7eeb3e566fbbdc09170bed95b --- /dev/null +++ b/api/src/engine/models/result/table-result.model.ts @@ -0,0 +1,15 @@ +import { Field, ObjectType } from '@nestjs/graphql'; +import { Metadata } from './common/metadata.model'; +import { Result } from './common/result.model'; + +@ObjectType() +export class TableResult extends Result { + @Field() + name: string; + + @Field(() => [[String]]) + data: string[][]; + + @Field(() => [Metadata]) + metadatas: Metadata[]; +} diff --git a/api/src/engine/models/variable.model.ts b/api/src/engine/models/variable.model.ts index 058feaf0be1e6a94b35c9f9d4ea98ac529ab0e83..e3c4098e31fccbcc4bd223c82ea55225559edaaf 100644 --- a/api/src/engine/models/variable.model.ts +++ b/api/src/engine/models/variable.model.ts @@ -1,15 +1,10 @@ import { Field, ObjectType } from '@nestjs/graphql'; import { Category } from './category.model'; +import { Entity } from './entity.model'; import { Group } from './group.model'; @ObjectType() -export class Variable { - @Field() - id: string; - - @Field({ nullable: true }) - label?: string; - +export class Variable extends Entity { @Field() type: string; diff --git a/api/src/schema.gql b/api/src/schema.gql index 023cdbeda9b5556163b2cd09140be601bd7cbc9c..a06bcf8e66d027c1c899668254b1459bc797c2ee 100644 --- a/api/src/schema.gql +++ b/api/src/schema.gql @@ -4,7 +4,7 @@ type Category { id: String! - label: String! + label: String } type Variable { @@ -18,7 +18,7 @@ type Variable { type Group { id: String! - label: String! + label: String description: String groups: [Group!]! variables: [Variable!]! @@ -26,7 +26,7 @@ type Group { type Domain { id: String! - label: String! + label: String description: String groups: [Group!]! variables: [Variable!]! @@ -34,6 +34,54 @@ type Domain { rootGroup: Group! } +type Metadata { + name: String! + type: String! +} + +type Experiment { + title: String! + uuid: String + created_at: DateTime + update_at: DateTime + finished_at: DateTime + results: [ResultUnion!]! +} + +""" +A date-time string at UTC, such as 2019-12-03T09:54:33Z, compliant with the date-time format. +""" +scalar DateTime + +union ResultUnion = TableResult | DummyResult + +type TableResult { + groupBy: String + name: String! + data: [[String!]!]! + metadatas: [Metadata!]! +} + +type DummyResult { + groupBy: String + name: String! + data: [[String!]!]! + listMax: [String!]! +} + type Query { domains(ids: [String!] = []): [Domain!]! } + +type Mutation { + createTransient(data: ExperimentCreateInput!): Experiment! +} + +input ExperimentCreateInput { + datasets: [String!]! + variables: [String!]! + filter: String + domain: String! + algorithm: String! + name: String! +}