From 890484be357a40def7bf6d4e4afb018b17628359 Mon Sep 17 00:00:00 2001
From: stevereis <stevereis93@gmail.com>
Date: Mon, 21 Feb 2022 18:32:36 +0100
Subject: [PATCH] fix: Add pearson (Exareme 2) integration
---
.../common/interfaces/utilities.interface.ts | 1 +
.../handlers/algorithms/area.handler.ts | 49 +++++++
.../algorithms/descriptive.handler.ts | 131 ++++++++++++++++++
.../handlers/algorithms/heat-map.handler.ts | 46 ++++++
.../handlers/algorithms/pearson.handler.ts | 34 +++--
.../handlers/algorithms/raw.handler.ts | 29 ++++
.../exareme/handlers/base.handler.ts | 12 +-
.../connectors/exareme/handlers/index.ts | 19 +++
.../handlers/result-handler.interface.ts | 6 +-
api/src/schema.gql | 3 +
10 files changed, 315 insertions(+), 15 deletions(-)
create mode 100644 api/src/engine/connectors/exareme/handlers/algorithms/area.handler.ts
create mode 100644 api/src/engine/connectors/exareme/handlers/algorithms/descriptive.handler.ts
create mode 100644 api/src/engine/connectors/exareme/handlers/algorithms/heat-map.handler.ts
create mode 100644 api/src/engine/connectors/exareme/handlers/algorithms/raw.handler.ts
create mode 100644 api/src/engine/connectors/exareme/handlers/index.ts
diff --git a/api/src/common/interfaces/utilities.interface.ts b/api/src/common/interfaces/utilities.interface.ts
index db02ef9..0a0978c 100644
--- a/api/src/common/interfaces/utilities.interface.ts
+++ b/api/src/common/interfaces/utilities.interface.ts
@@ -3,6 +3,7 @@ import { ResultUnion } from 'src/engine/models/result/common/result-union.model'
export type Dictionary<T> = { [key: string]: T };
export type ExperimentResult = typeof ResultUnion;
+export type AlgoResults = ExperimentResult[];
export enum MIME_TYPES {
ERROR = 'text/plain+error',
diff --git a/api/src/engine/connectors/exareme/handlers/algorithms/area.handler.ts b/api/src/engine/connectors/exareme/handlers/algorithms/area.handler.ts
new file mode 100644
index 0000000..198175a
--- /dev/null
+++ b/api/src/engine/connectors/exareme/handlers/algorithms/area.handler.ts
@@ -0,0 +1,49 @@
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import { ResultChartExperiment } from '../../interfaces/experiment/result-chart-experiment.interface';
+import BaseHandler from '../base.handler';
+
+export default class AreaHandler extends BaseHandler {
+ static readonly transform = jsonata(`
+ ({
+ "name": data.title.text,
+ "xAxis": {
+ "label": data.xAxis.title.text
+ },
+ "yAxis": {
+ "label": data.yAxis.title.text
+ },
+ "lines": [
+ {
+ "label": "ROC curve",
+ "x": data.series.data.$[0],
+ "y": data.series.data.$[1],
+ "type": 0
+ }
+ ]
+ })
+ `);
+
+ canHandle(input: ResultChartExperiment): boolean {
+ return (
+ input.type === 'application/vnd.highcharts+json' &&
+ input.data.chart.type === 'area'
+ );
+ }
+
+ handle(algorithm: string, data: unknown, res: AlgoResults): void {
+ let req = data;
+ const inputs = data as ResultChartExperiment[];
+
+ if (inputs) {
+ inputs
+ .filter(this.canHandle)
+ .map((input) => AreaHandler.transform.evaluate(input))
+ .forEach((input) => res.push(input));
+
+ req = JSON.stringify(inputs.filter((input) => !this.canHandle(input)));
+ }
+
+ this.next?.handle(algorithm, req, res);
+ }
+}
diff --git a/api/src/engine/connectors/exareme/handlers/algorithms/descriptive.handler.ts b/api/src/engine/connectors/exareme/handlers/algorithms/descriptive.handler.ts
new file mode 100644
index 0000000..a314a6b
--- /dev/null
+++ b/api/src/engine/connectors/exareme/handlers/algorithms/descriptive.handler.ts
@@ -0,0 +1,131 @@
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import {
+ GroupResult,
+ GroupsResult,
+} from 'src/engine/models/result/groups-result.model';
+import { ResultExperiment } from '../../interfaces/experiment/result-experiment.interface';
+import BaseHandler from '../base.handler';
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+
+export default class DescriptiveHandler extends BaseHandler {
+ private static readonly headerDescriptive = `
+$fnum := function($x) { $type($x) = 'number' ? $round($number($x),3) : $x };
+
+$e := function($x, $r) {($x != null) ? $fnum($x) : ($r ? $r : '')};
+
+$fn := function($o, $prefix) {
+ $type($o) = 'object' ?
+ $each($o, function($v, $k) {(
+ $type($v) = 'object' ? { $k: $v.count & ' (' & $v.percentage & '%)' } : {
+ $k: $v
+ }
+ )}) ~> $merge()
+ : {}
+};`;
+
+ static readonly descriptiveModelToTables = jsonata(`
+(
+ ${this.headerDescriptive}
+
+ $vars := $count($keys(data.model.*.data))-1;
+ $varNames := $keys(data.model.*.data);
+ $model := data.model;
+
+ [[0..$vars].(
+ $i := $;
+ $varName := $varNames[$i];
+ $ks := $keys($model.*.data.*[$i][$type($) = 'object']);
+ {
+ 'name': $varName,
+ 'headers': $append("", $keys($$.data.model)).{
+ 'name': $,
+ 'type': 'string'
+ },
+ 'data': [
+ [$varName, $model.*.($e(num_total))],
+ ['Datapoints', $model.*.($e(num_datapoints))],
+ ['Nulls', $model.*.($e(num_nulls))],
+ ($lookup($model.*.data, $varName).($fn($)) ~> $reduce(function($a, $b) {
+ $map($ks, function($k) {(
+ {
+ $k : [$e($lookup($a,$k), "No data"), $e($lookup($b,$k), "No data")]
+ }
+ )}) ~> $merge()
+ })).$each(function($v, $k) {$append($k,$v)})[]
+ ]
+ }
+ )]
+)`);
+
+ static readonly descriptiveSingleToTables = jsonata(`
+(
+ ${this.headerDescriptive}
+
+ data.[
+ $.single.*@$p#$i.(
+ $ks := $keys($p.*.data[$type($) = 'object']);
+ {
+ 'name': $keys(%)[$i],
+ 'headers': $append("", $keys(*)).{
+ 'name': $,
+ 'type': 'string'
+ },
+ 'data' : [
+ [$keys(%)[$i], $p.*.($e(num_total))],
+ ['Datapoints', $p.*.($e(num_datapoints))],
+ ['Nulls', $p.*.($e(num_nulls))],
+ ($p.*.data.($fn($)) ~> $reduce(function($a, $b) {
+ $map($ks, function($k) {(
+ {
+ $k : [$e($lookup($a,$k), "No data"), $e($lookup($b,$k), "No data")]
+ }
+ )}) ~> $merge()
+ })).$each(function($v, $k) {$append($k,$v)})[]
+ ]
+ })
+ ]
+)
+`);
+
+ descriptiveDataToTableResult(data: ResultExperiment): GroupsResult {
+ const result = new GroupsResult();
+
+ result.groups = [
+ new GroupResult({
+ name: 'Variables',
+ description: 'Descriptive statistics for the variables of interest.',
+ results: DescriptiveHandler.descriptiveSingleToTables.evaluate(data),
+ }),
+ ];
+
+ result.groups.push(
+ new GroupResult({
+ name: 'Model',
+ description:
+ 'Intersection table for the variables of interest as it appears in the experiment.',
+ results: DescriptiveHandler.descriptiveModelToTables.evaluate(data),
+ }),
+ );
+
+ return result;
+ }
+
+ handle(algorithm: string, data: unknown, res: AlgoResults): void {
+ let req = data;
+
+ if (algorithm.toLowerCase() === 'descriptive_stats') {
+ const inputs = data as ResultExperiment[];
+
+ inputs
+ .filter((input) => input.type === 'application/json')
+ .map((input) => this.descriptiveDataToTableResult(input))
+ .forEach((input) => res.push(input));
+
+ req = JSON.stringify(
+ inputs.filter((input) => input.type !== 'application/json'),
+ );
+ }
+
+ this.next?.handle(algorithm, req, res);
+ }
+}
diff --git a/api/src/engine/connectors/exareme/handlers/algorithms/heat-map.handler.ts b/api/src/engine/connectors/exareme/handlers/algorithms/heat-map.handler.ts
new file mode 100644
index 0000000..20591ad
--- /dev/null
+++ b/api/src/engine/connectors/exareme/handlers/algorithms/heat-map.handler.ts
@@ -0,0 +1,46 @@
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import { ResultChartExperiment } from '../../interfaces/experiment/result-chart-experiment.interface';
+import BaseHandler from '../base.handler';
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+
+export default class HeatMapHandler extends BaseHandler {
+ static readonly transform = jsonata(`
+ (
+ {
+ "name": data.title.text,
+ "xAxis": {
+ "categories": data.xAxis.categories,
+ "label": data.xAxis.label
+ },
+ "yAxis": {
+ "categories": data.yAxis.categories,
+ "label": data.yAxis.label
+ },
+ "matrix": $toMat(data.series.data)
+ }
+ )
+ `);
+
+ canHandle(input: ResultChartExperiment): boolean {
+ return (
+ input.type.toLowerCase() === 'application/vnd.highcharts+json' &&
+ input.data.chart.type.toLowerCase() === 'heatmap'
+ );
+ }
+
+ handle(algorithm: string, data: unknown, res: AlgoResults): void {
+ let req = data;
+ const inputs = data as ResultChartExperiment[];
+
+ if (inputs) {
+ inputs
+ .filter(this.canHandle)
+ .map((input) => HeatMapHandler.transform.evaluate(input))
+ .forEach((input) => res.push(input));
+
+ req = JSON.stringify(inputs.filter((input) => !this.canHandle(input)));
+ }
+
+ this.next?.handle(algorithm, req, res);
+ }
+}
diff --git a/api/src/engine/connectors/exareme/handlers/algorithms/pearson.handler.ts b/api/src/engine/connectors/exareme/handlers/algorithms/pearson.handler.ts
index 731cbad..1670265 100644
--- a/api/src/engine/connectors/exareme/handlers/algorithms/pearson.handler.ts
+++ b/api/src/engine/connectors/exareme/handlers/algorithms/pearson.handler.ts
@@ -1,9 +1,11 @@
-import jsonata, { Expression } from 'jsonata';
-import { Results } from 'src/common/interfaces/utilities.interface';
+import { Expression } from 'jsonata';
+import * as jsonata from 'jsonata'; // old import style needed due to 'export = jsonata'
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import { HeatMapResult } from 'src/engine/models/result/heat-map-result.model';
import BaseHandler from '../base.handler';
-class PearsonHandler extends BaseHandler {
- readonly pearsonCorellation: Expression = jsonata(`
+export default class PearsonHandler extends BaseHandler {
+ readonly transform: Expression = jsonata(`
(
$params := ['correlations', 'p-values', 'low_confidence_intervals', 'high_confidence_intervals'];
@@ -21,9 +23,25 @@ class PearsonHandler extends BaseHandler {
})
)`);
- handle(data: JSON, res: Results): void {
- try {
- const results = this.pearsonCorellation.evaluate(data);
- } catch (e) {}
+ canHandle(algorithm: string): boolean {
+ return algorithm.toLocaleLowerCase() === 'pearson';
+ }
+
+ handle(algorithm: string, data: unknown, res: AlgoResults): void {
+ if (this.canHandle(algorithm)) {
+ try {
+ const results = this.transform.evaluate(data) as HeatMapResult[];
+ results
+ .filter((heatMap) => heatMap.matrix.length > 0 && heatMap.name)
+ .forEach((heatMap) => res.push(heatMap));
+ } catch (e) {
+ PearsonHandler.logger.warn(
+ 'An error occur when converting result from Pearson',
+ );
+ PearsonHandler.logger.verbose(JSON.stringify(data));
+ }
+ }
+
+ this.next?.handle(algorithm, data, res);
}
}
diff --git a/api/src/engine/connectors/exareme/handlers/algorithms/raw.handler.ts b/api/src/engine/connectors/exareme/handlers/algorithms/raw.handler.ts
new file mode 100644
index 0000000..810be82
--- /dev/null
+++ b/api/src/engine/connectors/exareme/handlers/algorithms/raw.handler.ts
@@ -0,0 +1,29 @@
+import {
+ AlgoResults,
+ MIME_TYPES,
+} from 'src/common/interfaces/utilities.interface';
+import { RawResult } from 'src/engine/models/result/raw-result.model';
+import { ResultExperiment } from '../../interfaces/experiment/result-experiment.interface';
+import BaseHandler from '../base.handler';
+
+export default class RawHandler extends BaseHandler {
+ dataToRaw = (algo: string, result: ResultExperiment): RawResult => {
+ let data = result;
+
+ if (algo === 'CART') {
+ data = { ...data, type: MIME_TYPES.JSONBTREE };
+ }
+
+ return { rawdata: data };
+ };
+
+ handle(algorithm: string, data: unknown, res: AlgoResults): void {
+ const inputs = data as ResultExperiment[];
+
+ inputs
+ .map((input) => this.dataToRaw(algorithm, input))
+ .forEach((input) => res.push(input));
+
+ this.next?.handle(algorithm, data, res);
+ }
+}
diff --git a/api/src/engine/connectors/exareme/handlers/base.handler.ts b/api/src/engine/connectors/exareme/handlers/base.handler.ts
index 75c19cf..2f72f19 100644
--- a/api/src/engine/connectors/exareme/handlers/base.handler.ts
+++ b/api/src/engine/connectors/exareme/handlers/base.handler.ts
@@ -1,14 +1,18 @@
-import { Results } from 'src/common/interfaces/utilities.interface';
+import { Logger } from '@nestjs/common';
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
import ResultHandler from './result-handler.interface';
export default abstract class BaseHandler implements ResultHandler {
+ protected static readonly logger = new Logger(this.name);
+
next: ResultHandler = null;
- setNext(h: ResultHandler): void {
+ setNext(h: ResultHandler): ResultHandler {
this.next = h;
+ return h;
}
- handle(data: JSON, res: Results): void {
- this.next?.handle(data, res);
+ handle(algorithm: string, data: unknown, res: AlgoResults): void {
+ this.next?.handle(algorithm, data, res);
}
}
diff --git a/api/src/engine/connectors/exareme/handlers/index.ts b/api/src/engine/connectors/exareme/handlers/index.ts
new file mode 100644
index 0000000..7e153ec
--- /dev/null
+++ b/api/src/engine/connectors/exareme/handlers/index.ts
@@ -0,0 +1,19 @@
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
+import AreaHandler from './algorithms/area.handler';
+import DescriptiveHandler from './algorithms/descriptive.handler';
+import HeatMapHandler from './algorithms/heat-map.handler';
+import {
+ default as PearsonHandler,
+ default as RawHandler,
+} from './algorithms/raw.handler';
+
+const last = new RawHandler(); // should be last handler as it works as a fallback (if other handlers could not process the results)
+const start = new PearsonHandler()
+ .setNext(new AreaHandler())
+ .setNext(new DescriptiveHandler())
+ .setNext(new HeatMapHandler())
+ .setNext(last);
+
+export default (algo: string, data: unknown, res: AlgoResults) => {
+ start.handle(algo, data, res);
+};
diff --git a/api/src/engine/connectors/exareme/handlers/result-handler.interface.ts b/api/src/engine/connectors/exareme/handlers/result-handler.interface.ts
index da929b9..93f98a9 100644
--- a/api/src/engine/connectors/exareme/handlers/result-handler.interface.ts
+++ b/api/src/engine/connectors/exareme/handlers/result-handler.interface.ts
@@ -1,7 +1,7 @@
-import { Results } from 'src/common/interfaces/utilities.interface';
+import { AlgoResults } from 'src/common/interfaces/utilities.interface';
// produce algo handler
export default interface ResultHandler {
- setNext(h: ResultHandler): void;
- handle(data: JSON, res: Results): void;
+ setNext(h: ResultHandler): ResultHandler;
+ handle(algorithm: string, data: unknown, res: AlgoResults): void;
}
diff --git a/api/src/schema.gql b/api/src/schema.gql
index 4d248dc..b940e0e 100644
--- a/api/src/schema.gql
+++ b/api/src/schema.gql
@@ -133,7 +133,10 @@ type LineChartResult {
}
type ChartAxis {
+ """label of the Axis"""
label: String
+
+ """label of each element on this Axis"""
categories: [String!]
}
--
GitLab