Merge branch 'feat/datashield-integration' into 'develop'

feat: Add support for histograms and descriptives stats from Datashield API

See merge request sibmip/gateway!39

api/src/common/interfaces/utilities.interface.ts

+import { ResultUnion } from 'src/engine/models/result/common/result-union.model';
+
 export type Dictionary<T> = { [key: string]: T };
 
+export type ExperimentResult = typeof ResultUnion;
+
 export enum MIME_TYPES {
   ERROR = 'text/plain+error',
   WARNING = 'text/plain+warning',

api/src/engine/connectors/datashield/main.connector.ts

+import { HttpService } from '@nestjs/axios';
+import { Inject, Logger } from '@nestjs/common';
+import { REQUEST } from '@nestjs/core';
+import { Request } from 'express';
 import { firstValueFrom, Observable } from 'rxjs';
+import { MIME_TYPES } from 'src/common/interfaces/utilities.interface';
+import { ENGINE_MODULE_OPTIONS } from 'src/engine/engine.constants';
 import { IEngineOptions, IEngineService } from 'src/engine/engine.interfaces';
 import { Domain } from 'src/engine/models/domain.model';
-import { ExperimentCreateInput } from 'src/engine/models/experiment/input/experiment-create.input';
+import { Algorithm } from 'src/engine/models/experiment/algorithm.model';
 import {
   Experiment,
   PartialExperiment,
 } from 'src/engine/models/experiment/experiment.model';
-import { ListExperiments } from 'src/engine/models/experiment/list-experiments.model';
+import { ExperimentCreateInput } from 'src/engine/models/experiment/input/experiment-create.input';
 import { ExperimentEditInput } from 'src/engine/models/experiment/input/experiment-edit.input';
-import { Algorithm } from 'src/engine/models/experiment/algorithm.model';
-import { ENGINE_MODULE_OPTIONS } from 'src/engine/engine.constants';
-import { Inject } from '@nestjs/common';
-import { HttpService } from '@nestjs/axios';
-import { REQUEST } from '@nestjs/core';
-import { Request } from 'express';
-import { transformToDomains } from './transformations';
+import { ListExperiments } from 'src/engine/models/experiment/list-experiments.model';
+import { RawResult } from 'src/engine/models/result/raw-result.model';
+import { TableResult } from 'src/engine/models/result/table-result.model';
+import {
+  transformToDomains,
+  transformToHisto,
+  transformToTable,
+} from './transformations';
 
 export default class DataShieldService implements IEngineService {
+  private static readonly logger = new Logger(DataShieldService.name);
+  headers = {};
   constructor(
     @Inject(ENGINE_MODULE_OPTIONS) private readonly options: IEngineOptions,
     private readonly httpService: HttpService,
@@ -31,18 +40,104 @@ export default class DataShieldService implements IEngineService {
     throw new Error('Method not implemented.');
   }
 
-  createExperiment(
+  async getHistogram(variable: string): Promise<RawResult> {
+    const path =
+      this.options.baseurl + `histogram?var=${variable}&type=combine`;
+
+    const response = await firstValueFrom(
+      this.httpService.get(path, {
+        headers: {
+          cookie: this.req['req'].headers['cookie'],
+        },
+      }),
+    );
+
+    if (response.data['breaks'] === undefined) {
+      DataShieldService.logger.warn('Inconsistency on histogram result');
+      DataShieldService.logger.verbose(path);
+      return {
+        rawdata: {
+          data: response.data[0],
+          type: MIME_TYPES.ERROR,
+        },
+      };
+    }
+
+    const title = variable.replace(/\./g, ' ').trim();
+    const data = { ...response.data, title };
+    const chart = transformToHisto.evaluate(data);
+
+    return {
+      rawdata: {
+        data: chart,
+        type: 'application/vnd.highcharts+json',
+      },
+    };
+  }
+
+  async getDescriptiveStats(variable: string): Promise<TableResult> {
+    const path = this.options.baseurl + `quantiles?var=${variable}&type=split`;
+
+    const response = await firstValueFrom(
+      this.httpService.get(path, {
+        headers: {
+          cookie: this.req['req'].headers['cookie'],
+        },
+      }),
+    );
+
+    const title = variable.replace(/\./g, ' ').trim();
+    const data = { ...response.data, title };
+    return transformToTable.evaluate(data);
+  }
+
+  async createExperiment(
     data: ExperimentCreateInput,
     isTransient: boolean,
-  ): Experiment | Promise<Experiment> {
-    throw new Error('Method not implemented.');
+  ): Promise<Experiment> {
+    const expResult: Experiment = {
+      id: `${data.algorithm.id}-${Date.now()}`,
+      variables: data.variables,
+      name: data.name,
+      domain: data.domain,
+      datasets: data.datasets,
+      algorithm: {
+        id: data.algorithm.id,
+      },
+    };
+
+    switch (data.algorithm.id) {
+      case 'MULTIPLE_HISTOGRAMS': {
+        expResult.results = await Promise.all<RawResult>(
+          data.variables.map(
+            async (variable) => await this.getHistogram(variable),
+          ),
+        );
+        break;
+      }
+      case 'DESCRIPTIVE_STATS': {
+        expResult.results = await Promise.all<TableResult>(
+          [...data.variables, ...data.coVariables].map(
+            async (variable) => await this.getDescriptiveStats(variable),
+          ),
+        );
+        break;
+      }
+    }
+
+    return expResult;
   }
 
   listExperiments(
     page: number,
     name: string,
   ): ListExperiments | Promise<ListExperiments> {
-    throw new Error('Method not implemented.');
+    return {
+      totalExperiments: 0,
+      experiments: [],
+      totalPages: 0,
+      currentPage: 0,
+    };
   }
 
   getExperiment(id: string): Experiment | Promise<Experiment> {
@@ -63,12 +158,24 @@ export default class DataShieldService implements IEngineService {
   async getDomains(): Promise<Domain[]> {
     const path = this.options.baseurl + 'start';
 
-    const data = await firstValueFrom(
+    const response = await firstValueFrom(
       this.httpService.get(path, {
         auth: { username: 'guest', password: 'guest123' },
       }),
     );
-    return [transformToDomains.evaluate(data.data)];
+
+    if (response.headers && response.headers['set-cookie']) {
+      const cookies = response.headers['set-cookie'] as string[];
+      cookies.forEach((cookie) => {
+        const [key, value] = cookie.split(/={1}/);
+        this.req.res.cookie(key, value, {
+          httpOnly: true,
+          //sameSite: 'none',
+        });
+      });
+    }
+
+    return [transformToDomains.evaluate(response.data)];
   }
 
   getActiveUser(): string {

api/src/engine/connectors/datashield/transformations.ts

@@ -22,7 +22,91 @@ export const transformToDomains = jsonata(`
     },
     "variables": $distinct(groups.variables).{
         "id": $,
-        "label": $
+        "label": $trim($replace($ & '', '.', ' ')),
+        "type": "Number"
     }
 }
 `);
+
+export const transformToHisto = jsonata(`
+(
+    $nbBreaks := $count(breaks);
+
+    {
+    "chart": {
+      "type": 'column'
+    },
+    "legend": {
+      "enabled": false
+    },
+    "series": [
+      {
+        "data": counts,
+        "dataLabels": {
+          "enabled": true
+        }
+      }
+    ],
+    "title": {
+      "text": title ? title : ''
+    },
+    "tooltip": {
+      "enabled": true
+    },
+    "xAxis": {
+      "categories": breaks#$i[$i < $nbBreaks-1].[$ & ' - ' & %.*[$i+1]]
+    },
+    "yAxis": {
+      "min": 0,
+      "minRange": 0.1,
+      "allowDecimals": true
+    }
+})
+  `);
+
+export const transformToTable = jsonata(`
+{
+    "name": "Descriptive Statistics",
+    "headers": $append(title, ['5%','10%','25%','50%','75%','90%','95%','Mean']).{
+        "name": $,
+        "type": "string"
+    },
+    "data": $.message.$each(function($v, $k) {
+           $append($k,$v)
+        })
+}
+`);
+
+/*export const transformToTable = jsonata(`
+(
+    $params := ["xname", "equidist", "breaks"];
+    $concat := function($i, $j) {
+        $append($i, $j)
+    };
+
+    {
+        "name": "test",
+        "headers": $append('', $.*[0].breaks).{
+            "name": $,
+            "type": "string"
+        },
+        "data": $.$each(function($v, $k) {
+           [[$k],$v.$each(function($v, $k) {$not($k in $params) ? $append($k, $v) : undefined})]
+        }) ~> $reduce($concat, [])
+    }
+)
+`);*/
+
+/* export const transformToTable = jsonata(`
+(
+    $params := ["xname", "equidist", "breaks"];
+
+    $.$each(function($v, $k) {
+        {
+            "name": $k,
+            "headers": $v.breaks,
+            "data": $v.$each(function($v, $k) {$not($k in $params) ? $append($k, $v) : undefined})
+        }
+    })
+)
+`); */

api/src/engine/engine.module.ts

@@ -2,6 +2,7 @@ import { HttpModule, HttpService } from '@nestjs/axios';
 import { DynamicModule, Global, Logger, Module } from '@nestjs/common';
 import { REQUEST } from '@nestjs/core';
 import { GraphQLModule } from '@nestjs/graphql';
+import { Request } from 'express';
 import { join } from 'path';
 import { ENGINE_MODULE_OPTIONS, ENGINE_SERVICE } from './engine.constants';
 import { EngineController } from './engine.controller';
@@ -33,6 +34,14 @@ export class EngineModule {
         HttpModule,
         GraphQLModule.forRoot({
           autoSchemaFile: join(process.cwd(), 'src/schema.gql'),
+          context: ({ req, res }) => ({ req, res }),
+          cors: {
+            credentials: true,
+            origin: [
+              /http:\/\/localhost($|:\d*)/,
+              /http:\/\/127.0.0.1($|:\d*)/,
+            ],
+          },
         }),
       ],
       providers: [optionsProvider, engineProvider, EngineResolver],

api/src/main.ts

 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './main/app.module';
 
+const CORS_URL = process.env.CORS_URL ?? process.env.ENGINE_BASE_URL;
+
 async function bootstrap() {
-  const app = await NestFactory.create(AppModule, { cors: true });
+  const app = await NestFactory.create(AppModule, {
+    cors: {
+      credentials: true,
+      origin: [
+        /http:\/\/localhost($|:\d*)/,
+        /http:\/\/127.0.0.1($|:\d*)/,
+        CORS_URL,
+      ],
+    },
+  });
   await app.listen(process.env.GATEWAY_PORT);
 }
 bootstrap();