Merge branch 'feat/pearson-viz-integration' into 'develop'

Add Pearson viz integration

See merge request sibmip/gateway!47

api/src/common/interfaces/utilities.interface.ts

@@ -3,6 +3,7 @@ import { ResultUnion } from 'src/engine/models/result/common/result-union.model'
 export type Dictionary<T> = { [key: string]: T };
 
 export type ExperimentResult = typeof ResultUnion;
+export type AlgoResults = ExperimentResult[];
 
 export enum MIME_TYPES {
   ERROR = 'text/plain+error',

api/src/engine/connectors/exareme/handlers/algorithms/area.handler.ts

+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import { ResultChartExperiment } from '../../interfaces/experiment/result-chart-experiment.interface';
+import BaseHandler from '../base.handler';
+
+export default class AreaHandler extends BaseHandler {
+  static readonly transform = jsonata(`
+  ({
+      "name": data.title.text,
+      "xAxis": {
+          "label": data.xAxis.title.text
+      },
+      "yAxis": {
+          "label": data.yAxis.title.text
+      },
+      "lines": [
+          {
+              "label": "ROC curve",
+              "x": data.series.data.$[0],
+              "y": data.series.data.$[1],
+              "type": 0
+          }
+      ]
+  })
+  `);
+
+  canHandle(input: ResultChartExperiment): boolean {
+    return (
+      input.type === 'application/vnd.highcharts+json' &&
+      input.data.chart.type === 'area'
+    );
+  }
+
+  handle(algorithm: string, data: unknown, res: AlgoResults): void {
+    let req = data;
+    const inputs = data as ResultChartExperiment[];
+
+    if (inputs) {
+      inputs
+        .filter(this.canHandle)
+        .map((input) => AreaHandler.transform.evaluate(input))
+        .forEach((input) => res.push(input));
+
+      req = JSON.stringify(inputs.filter((input) => !this.canHandle(input)));
+    }
+
+    this.next?.handle(algorithm, req, res);
+  }
+}

api/src/engine/connectors/exareme/handlers/algorithms/descriptive.handler.ts

+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import {
+  GroupResult,
+  GroupsResult,
+} from 'src/engine/models/result/groups-result.model';
+import { ResultExperiment } from '../../interfaces/experiment/result-experiment.interface';
+import BaseHandler from '../base.handler';
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+
+export default class DescriptiveHandler extends BaseHandler {
+  private static readonly headerDescriptive = `
+$fnum := function($x) { $type($x) = 'number' ? $round($number($x),3) : $x };
+
+$e := function($x, $r) {($x != null) ? $fnum($x) : ($r ? $r : '')};
+
+$fn := function($o, $prefix) {
+    $type($o) = 'object' ? 
+    $each($o, function($v, $k) {(
+        $type($v) = 'object' ? { $k: $v.count & ' (' & $v.percentage & '%)' } : {
+            $k: $v
+        }
+    )}) ~> $merge()
+    : {}
+};`;
+
+  static readonly descriptiveModelToTables = jsonata(`
+(   
+    ${this.headerDescriptive}
+    
+    $vars := $count($keys(data.model.*.data))-1;
+    $varNames := $keys(data.model.*.data);
+    $model := data.model;
+
+    [[0..$vars].(
+        $i := $;
+        $varName := $varNames[$i];
+        $ks := $keys($model.*.data.*[$i][$type($) = 'object']);
+        {
+            'name': $varName,
+            'headers': $append("", $keys($$.data.model)).{
+                'name': $,
+                'type': 'string'
+            },
+            'data': [
+                [$varName, $model.*.($e(num_total))],
+                ['Datapoints', $model.*.($e(num_datapoints))],
+                ['Nulls', $model.*.($e(num_nulls))],
+                ($lookup($model.*.data, $varName).($fn($)) ~> $reduce(function($a, $b) {
+                    $map($ks, function($k) {(
+                        {
+                            $k : [$e($lookup($a,$k), "No data"), $e($lookup($b,$k), "No data")]
+                        }
+                    )}) ~> $merge()
+                })).$each(function($v, $k) {$append($k,$v)})[]
+            ]
+        }
+    )]  
+)`);
+
+  static readonly descriptiveSingleToTables = jsonata(`
+( 
+    ${this.headerDescriptive}
+
+    data.[
+        $.single.*@$p#$i.(
+            $ks := $keys($p.*.data[$type($) = 'object']);
+            {
+            'name': $keys(%)[$i],
+            'headers': $append("", $keys(*)).{
+                'name': $,
+                'type': 'string'
+            },
+            'data' : [
+                [$keys(%)[$i], $p.*.($e(num_total))],
+                ['Datapoints', $p.*.($e(num_datapoints))],
+                ['Nulls', $p.*.($e(num_nulls))],
+                ($p.*.data.($fn($)) ~> $reduce(function($a, $b) {
+                    $map($ks, function($k) {(
+                        {
+                            $k : [$e($lookup($a,$k), "No data"), $e($lookup($b,$k), "No data")]
+                        }
+                    )}) ~> $merge()
+                })).$each(function($v, $k) {$append($k,$v)})[]
+            ]
+        })
+    ]
+)
+`);
+
+  descriptiveDataToTableResult(data: ResultExperiment): GroupsResult {
+    const result = new GroupsResult();
+
+    result.groups = [
+      new GroupResult({
+        name: 'Variables',
+        description: 'Descriptive statistics for the variables of interest.',
+        results: DescriptiveHandler.descriptiveSingleToTables.evaluate(data),
+      }),
+    ];
+
+    result.groups.push(
+      new GroupResult({
+        name: 'Model',
+        description:
+          'Intersection table for the variables of interest as it appears in the experiment.',
+        results: DescriptiveHandler.descriptiveModelToTables.evaluate(data),
+      }),
+    );
+
+    return result;
+  }
+
+  handle(algorithm: string, data: unknown, res: AlgoResults): void {
+    let req = data;
+
+    if (algorithm.toLowerCase() === 'descriptive_stats') {
+      const inputs = data as ResultExperiment[];
+
+      inputs
+        .filter((input) => input.type === 'application/json')
+        .map((input) => this.descriptiveDataToTableResult(input))
+        .forEach((input) => res.push(input));
+
+      req = JSON.stringify(
+        inputs.filter((input) => input.type !== 'application/json'),
+      );
+    }
+
+    this.next?.handle(algorithm, req, res);
+  }
+}

api/src/engine/connectors/exareme/handlers/algorithms/heat-map.handler.ts

+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import { ResultChartExperiment } from '../../interfaces/experiment/result-chart-experiment.interface';
+import BaseHandler from '../base.handler';
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+
+export default class HeatMapHandler extends BaseHandler {
+  static readonly transform = jsonata(`
+  (
+      {
+          "name": data.title.text,
+          "xAxis": {
+              "categories": data.xAxis.categories,
+              "label": data.xAxis.label
+          },
+          "yAxis": {
+              "categories": data.yAxis.categories,
+              "label": data.yAxis.label
+          },
+          "matrix": $toMat(data.series.data)
+      }
+  )
+  `);
+
+  canHandle(input: ResultChartExperiment): boolean {
+    return (
+      input.type.toLowerCase() === 'application/vnd.highcharts+json' &&
+      input.data.chart.type.toLowerCase() === 'heatmap'
+    );
+  }
+
+  handle(algorithm: string, data: unknown, res: AlgoResults): void {
+    let req = data;
+    const inputs = data as ResultChartExperiment[];
+
+    if (inputs) {
+      inputs
+        .filter(this.canHandle)
+        .map((input) => HeatMapHandler.transform.evaluate(input))
+        .forEach((input) => res.push(input));
+
+      req = JSON.stringify(inputs.filter((input) => !this.canHandle(input)));
+    }
+
+    this.next?.handle(algorithm, req, res);
+  }
+}

api/src/engine/connectors/exareme/handlers/algorithms/pearson.handler.ts

+import { Expression } from 'jsonata';
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import { HeatMapResult } from 'src/engine/models/result/heat-map-result.model';
+import BaseHandler from '../base.handler';
+
+export default class PearsonHandler extends BaseHandler {
+  readonly transform: Expression = jsonata(`
+  (
+    $params := ['correlations', 'p-values', 'low_confidence_intervals', 'high_confidence_intervals'];
+
+    $.$sift(function($v, $k) {$k in $params}).$each(function($v, $k) {
+        {
+            'name': $k,
+            'xAxis': {
+                'categories': $v.variables
+            },
+            'yAxis': {
+                'categories': $keys($v.$sift(function($val, $key) {$key ~> /^(?!variables$)/}))
+            },
+            'matrix': $v.$sift(function($val, $key) {$key ~> /^(?!variables$)/}).$each(function($val, $key) {$val})[]
+            }
+    })
+  )`);
+
+  canHandle(algorithm: string): boolean {
+    return algorithm.toLocaleLowerCase() === 'pearson';
+  }
+
+  handle(algorithm: string, data: unknown, res: AlgoResults): void {
+    if (this.canHandle(algorithm)) {
+      try {
+        const results = this.transform.evaluate(data) as HeatMapResult[];
+        results
+          .filter((heatMap) => heatMap.matrix.length > 0 && heatMap.name)
+          .forEach((heatMap) => res.push(heatMap));
+      } catch (e) {
+        PearsonHandler.logger.warn(
+          'An error occur when converting result from Pearson',
+        );
+        PearsonHandler.logger.verbose(JSON.stringify(data));
+      }
+    }
+
+    this.next?.handle(algorithm, data, res);
+  }
+}

api/src/engine/connectors/exareme/handlers/algorithms/raw.handler.ts

+import {
+  AlgoResults,
+  MIME_TYPES,
+} from 'src/common/interfaces/utilities.interface';
+import { RawResult } from 'src/engine/models/result/raw-result.model';
+import { ResultExperiment } from '../../interfaces/experiment/result-experiment.interface';
+import BaseHandler from '../base.handler';
+
+export default class RawHandler extends BaseHandler {
+  dataToRaw = (algo: string, result: ResultExperiment): RawResult => {
+    let data = result;
+
+    if (algo === 'CART') {
+      data = { ...data, type: MIME_TYPES.JSONBTREE };
+    }
+
+    return { rawdata: data };
+  };
+
+  handle(algorithm: string, data: unknown, res: AlgoResults): void {
+    const inputs = data as ResultExperiment[];
+
+    inputs
+      .map((input) => this.dataToRaw(algorithm, input))
+      .forEach((input) => res.push(input));
+
+    this.next?.handle(algorithm, data, res);
+  }
+}

api/src/engine/connectors/exareme/handlers/base.handler.ts

+import { Logger } from '@nestjs/common';
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import ResultHandler from './result-handler.interface';
+
+export default abstract class BaseHandler implements ResultHandler {
+  protected static readonly logger = new Logger(this.name);
+
+  next: ResultHandler = null;
+
+  setNext(h: ResultHandler): ResultHandler {
+    this.next = h;
+    return h;
+  }
+
+  handle(algorithm: string, data: unknown, res: AlgoResults): void {
+    this.next?.handle(algorithm, data, res);
+  }
+}

api/src/engine/connectors/exareme/handlers/index.ts

+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import AreaHandler from './algorithms/area.handler';
+import DescriptiveHandler from './algorithms/descriptive.handler';
+import HeatMapHandler from './algorithms/heat-map.handler';
+import {
+  default as PearsonHandler,
+  default as RawHandler,
+} from './algorithms/raw.handler';
+
+const last = new RawHandler(); // should be last handler as it works as a fallback (if other handlers could not process the results)
+const start = new PearsonHandler()
+  .setNext(new AreaHandler())
+  .setNext(new DescriptiveHandler())
+  .setNext(new HeatMapHandler())
+  .setNext(last);
+
+export default (algo: string, data: unknown, res: AlgoResults) => {
+  start.handle(algo, data, res);
+};

api/src/engine/connectors/exareme/handlers/result-handler.interface.ts

+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+
+// produce algo handler
+export default interface ResultHandler {
+  setNext(h: ResultHandler): ResultHandler;
+  handle(algorithm: string, data: unknown, res: AlgoResults): void;
+}

api/src/engine/models/result/common/chart-axis.model.ts

@@ -2,9 +2,13 @@ import { ObjectType, Field } from '@nestjs/graphql';
 
 @ObjectType()
 export class ChartAxis {
-  @Field({ nullable: true, defaultValue: '' })
+  @Field({ nullable: true, defaultValue: '', description: 'label of the Axis' })
   label?: string;
 
-  @Field(() => [String], { nullable: true, defaultValue: [] })
+  @Field(() => [String], {
+    nullable: true,
+    defaultValue: [],
+    description: 'label of each element on this Axis',
+  })
   categories?: string[];
 }

api/src/schema.gql

@@ -133,7 +133,10 @@ type LineChartResult {
 }
 
 type ChartAxis {
+  """label of the Axis"""
   label: String
+
+  """label of each element on this Axis"""
   categories: [String!]
 }