From 671927d02a61d9c045df7f8bfda8e812afa9570b Mon Sep 17 00:00:00 2001
From: Xiao Gui <xgui3783@gmail.com>
Date: Tue, 30 Apr 2019 09:08:02 +0200
Subject: [PATCH] feat: migrate spatial search into the backend
---
deploy/datasets/index.js | 2 +
deploy/datasets/query.js | 59 ++++++++++++++--
deploy/datasets/spatialRouter.js | 47 +++++++++++++
.../databrowserModule/databrowser.service.ts | 70 +++++++++++++++++--
.../nehubaContainer.component.ts | 14 ++--
src/util/regionFlattener.ts | 2 +-
6 files changed, 176 insertions(+), 18 deletions(-)
create mode 100644 deploy/datasets/spatialRouter.js
diff --git a/deploy/datasets/index.js b/deploy/datasets/index.js
index a6f15dd43..01a7925bf 100644
--- a/deploy/datasets/index.js
+++ b/deploy/datasets/index.js
@@ -11,6 +11,8 @@ datasetsRouter.use((req, res, next) => {
next()
})
+datasetsRouter.use('/spatialSearch', require('./spatialRouter'))
+
datasetsRouter.get('/templateName/:templateName', (req, res, next) => {
const { templateName } = req.params
const { user } = req
diff --git a/deploy/datasets/query.js b/deploy/datasets/query.js
index 200d062e0..85ff46409 100644
--- a/deploy/datasets/query.js
+++ b/deploy/datasets/query.js
@@ -9,6 +9,7 @@ let cachedData = null
let otherQueryResult = null
const queryUrl = process.env.KG_DATASET_QUERY_URL || `https://kg.humanbrainproject.org/query/minds/core/dataset/v1.0.0/interactiveViewerKgQuery/instances?size=450&vocab=https%3A%2F%2Fschema.hbp.eu%2FmyQuery%2F`
const timeout = process.env.TIMEOUT || 5000
+const STORAGE_PATH = process.env.STORAGE_PATH || path.join(__dirname, 'data')
let getPublicAccessToken
@@ -84,8 +85,10 @@ const getDs = ({ user }) => user
*/
const flattenArray = (array) => {
- return array.filter(item => item.children.length === 0).concat(
- ...array.filter(item => item.children.length > 0).map(item => flattenArray(item.children))
+ return array.concat(
+ ...array.map(item => item.children && item.children instanceof Array
+ ? flattenArray(item.children)
+ : [])
)
}
@@ -105,6 +108,8 @@ const readConfigFile = (filename) => new Promise((resolve, reject) => {
let juBrain = null
let shortBundle = null
let longBundle = null
+let waxholm = null
+let allen = null
readConfigFile('colin.json')
.then(data => JSON.parse(data))
@@ -121,9 +126,16 @@ readConfigFile('MNI152.json')
})
.catch(console.error)
+readConfigFile('waxholmRatV2_0.json')
+ .then(data => JSON.parse(data))
+ .then(json => {
+ waxholm = flattenArray(json.parcellations[0].regions)
+ })
+ .catch(console.error)
+
const filterByPRs = (prs, atlasPr) => atlasPr
? prs.some(pr => {
- const regex = new RegExp(pr.name.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&'))
+ const regex = new RegExp(pr.name.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&'), 'i')
return atlasPr.some(aPr => regex.test(aPr.name))
})
: false
@@ -132,6 +144,8 @@ const manualFilter = require('./supplements/parcellation')
const filter = (datasets, {templateName, parcellationName}) => datasets
.filter(ds => {
+ if (/infant/.test(ds.name))
+ return false
if (templateName) {
return ds.referenceSpaces.some(rs => rs.name === templateName)
}
@@ -139,13 +153,15 @@ const filter = (datasets, {templateName, parcellationName}) => datasets
return ds.parcellationRegion.length > 0
? filterByPRs(
ds.parcellationRegion,
- parcellationName === 'JuBrain Cytoarchitectonic Atlas' && juBrain && !/infant/.test(ds.name)
+ parcellationName === 'JuBrain Cytoarchitectonic Atlas' && juBrain
? juBrain
: parcellationName === 'Fibre Bundle Atlas - Long Bundle' && longBundle
? longBundle
: parcellationName === 'Fibre Bundle Atlas - Short Bundle' && shortBundle
? shortBundle
- : null
+ : parcellationName === 'Whole Brain (v2.0)'
+ ? waxholm
+ : null
)
: manualFilter({ parcellationName, dataset: ds })
}
@@ -173,4 +189,35 @@ exports.init = () => fetchDatasetFromKg()
exports.getDatasets = ({ templateName, parcellationName, user }) => getDs({ user })
.then(json => filter(json, {templateName, parcellationName}))
-exports.getPreview = ({ datasetName }) => getPreviewFile({ datasetName })
\ No newline at end of file
+exports.getPreview = ({ datasetName }) => getPreviewFile({ datasetName })
+
+/**
+ * TODO
+ * change to real spatial query
+ */
+const cachedMap = new Map()
+const fetchSpatialDataFromKg = async ({ templateName }) => {
+ const cachedResult = cachedMap.get(templateName)
+ if (cachedResult)
+ return cachedResult
+
+ try {
+ const filename = path.join(STORAGE_PATH, templateName + '.json')
+ const exists = fs.existsSync(filename)
+
+ if (!exists)
+ return []
+
+ const data = fs.readFileSync(filename, 'utf-8')
+ const json = JSON.parse(data)
+ cachedMap.set(templateName, json)
+ return json
+ } catch (e) {
+ console.log('datasets#query.js#fetchSpatialDataFromKg', 'read file and parse json failed', e)
+ return []
+ }
+}
+
+exports.getSpatialDatasets = async ({ templateName, queryGeometry, queryArg }) => {
+ return await fetchSpatialDataFromKg({ templateName })
+}
\ No newline at end of file
diff --git a/deploy/datasets/spatialRouter.js b/deploy/datasets/spatialRouter.js
new file mode 100644
index 000000000..584eae6db
--- /dev/null
+++ b/deploy/datasets/spatialRouter.js
@@ -0,0 +1,47 @@
+const router = require('express').Router()
+const { getSpatialDatasets } = require('./query')
+
+const badRequestString = `spatialSearch endpoint uses param as follows:
+
+GET /templateName/<templateName>/<queryGeometry>/<queryArg>
+
+for example;
+
+GET /templateName/Colin/bbox/0_0_0__1_1_1`
+
+router.get('/templateName/:templateName/:queryGeometry/:queryArg', (req, res, next) => {
+ const { templateName, queryGeometry, queryArg } = req.params
+ let errorString = ``
+ if (!templateName)
+ errorString += `templateName is required\n`
+ if (!queryGeometry)
+ errorString += `queryGeometry is required\n`
+ if (!queryArg)
+ errorString += `queryArg is required\n`
+ if (errorString !== ``)
+ return next({
+ code: 400,
+ error: errorString,
+ trace: 'dataset#spatialRouter'
+ })
+
+ getSpatialDatasets({ templateName, queryGeometry, queryArg })
+ .then(arr => res.status(200).json(arr))
+ .catch(error => {
+ next({
+ code: 500,
+ error,
+ trace: 'dataset#spatialRouter#getSpatialDatasets'
+ })
+ })
+})
+
+router.use((req, res, next) => {
+ next({
+ code: 400,
+ error: badRequestString,
+ trace: 'dataset#spatialRouter#notFound'
+ })
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/src/ui/databrowserModule/databrowser.service.ts b/src/ui/databrowserModule/databrowser.service.ts
index 35adfb7c6..0d977cce1 100644
--- a/src/ui/databrowserModule/databrowser.service.ts
+++ b/src/ui/databrowserModule/databrowser.service.ts
@@ -1,10 +1,10 @@
import { Injectable, OnDestroy } from "@angular/core";
-import { Subscription, Observable, combineLatest, BehaviorSubject, fromEvent } from "rxjs";
+import { Subscription, Observable, combineLatest, BehaviorSubject, fromEvent, from, of } from "rxjs";
import { ViewerConfiguration } from "src/services/state/viewerConfig.store";
import { select, Store } from "@ngrx/store";
import { AtlasViewerConstantsServices } from "src/atlasViewer/atlasViewer.constantService.service";
-import { ADD_NG_LAYER, REMOVE_NG_LAYER, DataEntry, safeFilter, FETCHED_DATAENTRIES } from "src/services/stateStore.service";
-import { map, distinctUntilChanged, debounceTime, filter, tap } from "rxjs/operators";
+import { ADD_NG_LAYER, REMOVE_NG_LAYER, DataEntry, safeFilter, FETCHED_DATAENTRIES, FETCHED_SPATIAL_DATA, UPDATE_SPATIAL_DATA } from "src/services/stateStore.service";
+import { map, distinctUntilChanged, debounceTime, filter, tap, switchMap, catchError } from "rxjs/operators";
import { AtlasWorkerService } from "src/atlasViewer/atlasViewer.workerService.service";
import { FilterDataEntriesByRegion } from "./util/filterDataEntriesByRegion.pipe";
import { NO_METHODS } from "./util/filterDataEntriesByMethods.pipe";
@@ -14,6 +14,18 @@ import { WidgetUnit } from "src/atlasViewer/widgetUnit/widgetUnit.component";
const noMethodDisplayName = 'No methods described'
+/**
+ * param for .toFixed method
+ * 6: nm
+ * 3: um
+ * 0: mm
+ */
+const SPATIAL_SEARCH_PRECISION = 6
+/**
+ * in ms
+ */
+const SPATIAL_SEARCH_DEBOUNCE = 500
+
export function temporaryFilterDataentryName(name: string):string{
return /autoradiography/.test(name)
? 'autoradiography'
@@ -57,6 +69,9 @@ export class DatabrowserService implements OnDestroy{
public fetchDataObservable$: Observable<any>
public manualFetchDataset$: BehaviorSubject<null> = new BehaviorSubject(null)
+ private fetchSpatialData$: Observable<any>
+ public spatialDatasets$: Observable<any>
+
constructor(
private workerService: AtlasWorkerService,
private constantService: AtlasViewerConstantsServices,
@@ -80,8 +95,42 @@ export class DatabrowserService implements OnDestroy{
})
)
+ this.fetchSpatialData$ = combineLatest(
+ this.store.pipe(
+ select('viewerState'),
+ select('navigation')
+ ),
+ this.store.pipe(
+ select('viewerState'),
+ select('templateSelected')
+ )
+ ).pipe(
+ debounceTime(SPATIAL_SEARCH_DEBOUNCE)
+ )
- this.fetchDataObservable$ = combineLatest(
+ this.spatialDatasets$ = this.fetchSpatialData$.pipe(
+ switchMap(([navigation, templateSelected]) => {
+
+ /**
+ * templateSelected and templateSelected.name must be defined for spatial search
+ */
+ if (!templateSelected || !templateSelected.name)
+ return from(Promise.reject('templateSelected must not be empty'))
+ const encodedTemplateName = encodeURI(templateSelected.name)
+
+ // in mm
+ const center = navigation.position.map(n=>n/1e6)
+ const searchWidth = this.constantService.spatialWidth / 4 * navigation.zoom / 1e6
+ const pt1 = center.map(v => (v - searchWidth).toFixed(SPATIAL_SEARCH_PRECISION))
+ const pt2 = center.map(v => (v + searchWidth).toFixed(SPATIAL_SEARCH_PRECISION))
+
+ return from(fetch(`${this.constantService.backendUrl}datasets/spatialSearch/templateName/${encodedTemplateName}/bbox/${pt1.join('_')}__${pt2.join("_")}`)
+ .then(res => res.json()))
+ }),
+ catchError((err) => (console.log(err), of([])))
+ )
+
+ this.fetchDataObservable$ = combineLatest(
this.store.pipe(
select('viewerState'),
safeFilter('templateSelected'),
@@ -102,6 +151,19 @@ export class DatabrowserService implements OnDestroy{
this.fetchDataObservable$
)
+ this.subscriptions.push(
+ this.spatialDatasets$.subscribe(arr => {
+ this.store.dispatch({
+ type: FETCHED_SPATIAL_DATA,
+ fetchedDataEntries: arr
+ })
+ this.store.dispatch({
+ type : UPDATE_SPATIAL_DATA,
+ totalResults : arr.length
+ })
+ })
+ )
+
this.subscriptions.push(
this.fetchDataObservable$.pipe(
debounceTime(16)
diff --git a/src/ui/nehubaContainer/nehubaContainer.component.ts b/src/ui/nehubaContainer/nehubaContainer.component.ts
index 58a0fe4c1..2275936c2 100644
--- a/src/ui/nehubaContainer/nehubaContainer.component.ts
+++ b/src/ui/nehubaContainer/nehubaContainer.component.ts
@@ -840,13 +840,13 @@ export class NehubaContainer implements OnInit, OnDestroy{
const center = navigation.position.map(n=>n/1e6)
const searchWidth = this.constantService.spatialWidth / 4 * navigation.zoom / 1e6
- const templateSpace = this.selectedTemplate.name
- this.atlasViewerDataService.spatialSearch({
- center,
- searchWidth,
- templateSpace,
- pageNo : 0
- })
+ const { selectedTemplate } = this
+ // this.atlasViewerDataService.spatialSearch({
+ // center,
+ // searchWidth,
+ // selectedTemplate,
+ // pageNo : 0
+ // })
}
/* because the navigation can be changed from two sources,
diff --git a/src/util/regionFlattener.ts b/src/util/regionFlattener.ts
index 3d93fe488..d5ae471e1 100644
--- a/src/util/regionFlattener.ts
+++ b/src/util/regionFlattener.ts
@@ -1,6 +1,6 @@
export function regionFlattener(region:any){
return[
[ region ],
- ...region.children && region.children.map && region.children.map(regionFlattener)
+ ...((region.children && region.children.map && region.children.map(regionFlattener)) || [])
].reduce((acc, item) => acc.concat(item), [])
}
\ No newline at end of file
--
GitLab