From a9cf4c2d355df29da3f2d73c36c5107590554e03 Mon Sep 17 00:00:00 2001
From: xgui3783 <xgui3783@gmail.com>
Date: Mon, 29 Jun 2020 16:17:59 +0200
Subject: [PATCH] chore: adding atlas endpoint (#554)
---
deploy/app.js | 2 +
deploy/atlas/index.js | 73 ++++++++++++
deploy/atlas/index.spec.js | 79 +++++++++++++
deploy/atlas/query.js | 46 ++++++++
deploy/lruStore/index.js | 113 +++++++++++++++++++
deploy/package.json | 1 +
src/res/ext/MNI152.json | 1 +
src/res/ext/allenMouse.json | 3 +
src/res/ext/atlas/atlas_allenMouse.json | 8 ++
src/res/ext/atlas/atlas_multiLevelHuman.json | 14 +++
src/res/ext/atlas/atlas_waxholmRat.json | 8 ++
src/res/ext/bigbrain.json | 1 +
src/res/ext/colin.json | 1 +
src/res/ext/waxholmRatV2_0.json | 1 +
14 files changed, 351 insertions(+)
create mode 100644 deploy/atlas/index.js
create mode 100644 deploy/atlas/index.spec.js
create mode 100644 deploy/atlas/query.js
create mode 100644 deploy/lruStore/index.js
create mode 100644 src/res/ext/atlas/atlas_allenMouse.json
create mode 100644 src/res/ext/atlas/atlas_multiLevelHuman.json
create mode 100644 src/res/ext/atlas/atlas_waxholmRat.json
diff --git a/deploy/app.js b/deploy/app.js
index 02c1ec1fb..917bd1fdf 100644
--- a/deploy/app.js
+++ b/deploy/app.js
@@ -179,6 +179,7 @@ const jsonMiddleware = (req, res, next) => {
/**
* resources endpoints
*/
+const atlasesRouter = require('./atlas')
const templateRouter = require('./templates')
const nehubaConfigRouter = require('./nehubaConfig')
const datasetRouter = require('./datasets')
@@ -190,6 +191,7 @@ const setResLocalMiddleWare = routePathname => (req, res, next) => {
next()
}
+app.use('/atlases', setResLocalMiddleWare('atlases'), atlasesRouter)
app.use('/templates', setResLocalMiddleWare('templates'), jsonMiddleware, templateRouter)
app.use('/nehubaConfig', jsonMiddleware, nehubaConfigRouter)
app.use('/datasets', jsonMiddleware, datasetRouter)
diff --git a/deploy/atlas/index.js b/deploy/atlas/index.js
new file mode 100644
index 000000000..ef4c1317d
--- /dev/null
+++ b/deploy/atlas/index.js
@@ -0,0 +1,73 @@
+const router = require('express').Router()
+const url = require('url')
+const { detEncoding } = require('nomiseco')
+const { getAllAtlases, getAtlasById, isReady } = require('./query')
+
+const { getTemplate } = require('../templates/query')
+const { getHandleErrorFn } = require('../util/streamHandleError')
+
+router.get('/', async (req, res) => {
+ const allAtlases = await getAllAtlases()
+ const resolvedAtlases = allAtlases.map(v => {
+ return {
+ '@id': v,
+ url: res.locals.routePathname
+ ? url.resolve(`${res.locals.routePathname}/`, encodeURIComponent(v))
+ : encodeURIComponent(v)
+ }
+ })
+ res.status(200).json(resolvedAtlases)
+})
+
+router.get('/ready', (req, res) => {
+ if (isReady()) {
+ return res.status(200).end()
+ } else {
+ return res.status(503).end()
+ }
+})
+
+router.get('/:atlasId', async (req, res) => {
+ const { atlasId } = req.params
+ const { templateSpaces, ...rest } = await getAtlasById(atlasId)
+
+ return res.status(200).json({
+ ...rest,
+ templateSpaces: templateSpaces.map(tmpl => {
+ return {
+ ...tmpl,
+ url: res.locals.routePathname
+ ? url.resolve(`${res.locals.routePathname}/${encodeURIComponent(atlasId)}/`, encodeURIComponent(tmpl['@id']))
+ : `${encodeURIComponent(atlasId)}/${encodeURIComponent(tmpl['@id'])}`
+ }
+ })
+ })
+})
+
+const templateIdToStringMap = new Map([
+ ['minds/core/referencespace/v1.0.0/265d32a0-3d84-40a5-926f-bf89f68212b9', 'allenMouse'],
+ ['minds/core/referencespace/v1.0.0/d5717c4a-0fa1-46e6-918c-b8003069ade8', 'waxholmRatV2_0'],
+ ['minds/core/referencespace/v1.0.0/a1655b99-82f1-420f-a3c2-fe80fd4c8588', 'bigbrain'],
+ ['minds/core/referencespace/v1.0.0/dafcffc5-4826-4bf1-8ff6-46b8a31ff8e2', 'MNI152'],
+ ['minds/core/referencespace/v1.0.0/7f39f7be-445b-47c0-9791-e971c0b6d992', 'colin']
+])
+
+router.get('/:atlasId/:templateId', async (req, res) => {
+ const { atlasId, templateId } = req.params
+
+ const mappedString = templateIdToStringMap.get(templateId)
+ if (!mappedString) {
+ return res.status(404).end()
+ }
+
+ const header = req.get('Accept-Encoding')
+ const acceptedEncoding = detEncoding(header)
+
+ if (acceptedEncoding) res.set('Content-Encoding', acceptedEncoding)
+ getTemplate({ template: mappedString, acceptedEncoding, returnAsStream: true })
+ .pipe(res)
+ .on('error', getHandleErrorFn(req, res))
+})
+
+
+module.exports = router
\ No newline at end of file
diff --git a/deploy/atlas/index.spec.js b/deploy/atlas/index.spec.js
new file mode 100644
index 000000000..f30ca7bce
--- /dev/null
+++ b/deploy/atlas/index.spec.js
@@ -0,0 +1,79 @@
+const { expect, assert } = require('chai')
+const express = require('express')
+const router = require('./index')
+const { retry } = require('../../common/util')
+const got = require('got')
+
+let server, atlases, templates = []
+const PORT = 12345
+const baseUrl = `http://localhost:${PORT}`
+let routePathname
+describe('atlas/index.js', () => {
+ before(async () => {
+ const app = express()
+
+ app.use((req, res, next) => {
+ if (routePathname) {
+ res.locals.routePathname = routePathname
+ }
+ next()
+ })
+
+ app.use(router)
+
+ server = app.listen(PORT)
+
+ await retry(async () => {
+ console.log('retrying')
+ await got(`${baseUrl}/ready`)
+ }, { timeout: 500 })
+ })
+
+ beforeEach(() => {
+ routePathname = null
+ })
+
+ after(() => {
+ server.close()
+ })
+ it('> GET / works', async () => {
+ const { body } = await got(`${baseUrl}/`, { responseType: 'json' })
+ expect(body.length).to.equal(3)
+ assert(
+ body.every(({ url }) => !/undefined/.test(url)),
+ 'url pathname should not contain undefined'
+ )
+ atlases = body
+ })
+
+ it('> route pathname works', async () => {
+ routePathname = 'marshmellow'
+
+ const { body } = await got(`${baseUrl}/`, { responseType: 'json' })
+ expect(body.length).to.equal(3)
+ assert(
+ body.every(({ url }) => /^marshmellow/.test(url)),
+ 'url pathname should be as set'
+ )
+ })
+ it('> every end point works', async () => {
+ for (const { url } of atlases) {
+ expect(!!url).to.be.true
+ const { body } = await got(`${baseUrl}/${url}`, {responseType: 'json'})
+ expect(body.templateSpaces.length).to.be.greaterThan(0)
+ templates.push(
+ body.templateSpaces
+ )
+ }
+ })
+
+ it('> templates resolves fine', async () => {
+ for (const arrTmpl of templates) {
+ for (const { url } of arrTmpl) {
+ expect(!!url).to.equal(true)
+ const { body } = await got(`${baseUrl}/${url}`, { responseType: 'json' })
+ expect(body.parcellations.length).to.be.greaterThan(0)
+ }
+ }
+ })
+})
\ No newline at end of file
diff --git a/deploy/atlas/query.js b/deploy/atlas/query.js
new file mode 100644
index 000000000..6ecddcff3
--- /dev/null
+++ b/deploy/atlas/query.js
@@ -0,0 +1,46 @@
+const fs = require('fs')
+const path = require('path')
+const { promisify } = require('util')
+
+const readdirAsync = promisify(fs.readdir)
+const readFileAsync = promisify(fs.readFile)
+
+let ready = false
+
+const map = new Map()
+
+const getData = async () => {
+
+ let filepath
+ if (process.env.NODE_ENV === 'production') {
+ filepath = path.join(__dirname, '../res')
+ } else {
+ filepath = path.join(__dirname, '../../src/res/ext')
+ }
+
+ const files = await readdirAsync(path.join(filepath, 'atlas'))
+
+ for (const file of files) {
+
+ const data = await readFileAsync(path.join(filepath, 'atlas', file), 'utf-8')
+ const json = JSON.parse(data)
+ map.set(json['@id'], json)
+ }
+ ready = true
+}
+
+getData()
+
+const getAllAtlases = async () => {
+ return Array.from(map.keys())
+}
+
+const getAtlasById = async id => {
+ return map.get(id)
+}
+
+module.exports = {
+ getAllAtlases,
+ getAtlasById,
+ isReady: () => ready
+}
\ No newline at end of file
diff --git a/deploy/lruStore/index.js b/deploy/lruStore/index.js
new file mode 100644
index 000000000..e63e2e139
--- /dev/null
+++ b/deploy/lruStore/index.js
@@ -0,0 +1,113 @@
+/**
+ * Cache to allow for in memory response while data fetching/processing occur
+ */
+
+const {
+ REDIS_PROTO,
+ REDIS_ADDR,
+ REDIS_PORT,
+
+ REDIS_RATE_LIMITING_DB_EPHEMERAL_PORT_6379_TCP_PROTO,
+ REDIS_RATE_LIMITING_DB_EPHEMERAL_PORT_6379_TCP_ADDR,
+ REDIS_RATE_LIMITING_DB_EPHEMERAL_PORT_6379_TCP_PORT,
+
+ REDIS_USERNAME,
+ REDIS_PASSWORD,
+
+} = process.env
+
+const redisProto = REDIS_PROTO || REDIS_RATE_LIMITING_DB_EPHEMERAL_PORT_6379_TCP_PROTO || 'redis'
+const redisAddr = REDIS_ADDR || REDIS_RATE_LIMITING_DB_EPHEMERAL_PORT_6379_TCP_ADDR || null
+const redisPort = REDIS_PORT || REDIS_RATE_LIMITING_DB_EPHEMERAL_PORT_6379_TCP_PORT || 6379
+
+const userPass = `${REDIS_USERNAME || ''}${( REDIS_PASSWORD && (':' + REDIS_PASSWORD)) || ''}${ (REDIS_USERNAME || REDIS_PASSWORD) && '@'}`
+
+const redisURL = redisAddr && `${redisProto}://${userPass}${redisAddr}:${redisPort}`
+
+const crypto = require('crypto')
+
+let authKey
+
+const getAuthKey = () => {
+ crypto.randomBytes(128, (err, buf) => {
+ if (err) {
+ console.error(`generating random bytes error`, err)
+ return
+ }
+ authKey = buf.toString('base64')
+ console.log(`clear store key: ${authKey}`)
+ })
+}
+
+getAuthKey()
+
+const ensureString = val => {
+ if (typeof val !== 'string') throw new Error(`both key and val must be string`)
+}
+
+if (redisURL) {
+ const redis = require('redis')
+ const { promisify } = require('util')
+ const client = redis.createClient({
+ url: redisURL
+ })
+
+ const asyncGet = promisify(client.get).bind(client)
+ const asyncSet = promisify(client.set).bind(client)
+ const asyncDel = promisify(client.del).bind(client)
+
+ const keys = []
+
+ exports.store = {
+ set: async (key, val) => {
+ ensureString(key)
+ ensureString(val)
+ asyncSet(key, val)
+ keys.push(key)
+ },
+ get: async (key) => {
+ ensureString(key)
+ return asyncGet(key)
+ },
+ clear: async auth => {
+ if (auth !== authKey) {
+ getAuthKey()
+ throw new Error(`unauthorized`)
+ }
+ await asyncDel(keys.splice(0))
+ }
+ }
+
+ exports.StoreType = `redis`
+ console.log(`redis`)
+
+} else {
+ const LRU = require('lru-cache')
+ const store = new LRU({
+ max: 1024 * 1024 * 1024, // 1gb
+ length: (n, key) => n.length,
+ maxAge: Infinity, // never expires
+ })
+
+ exports.store = {
+ set: async (key, val) => {
+ ensureString(key)
+ ensureString(val)
+ store.set(key, val)
+ },
+ get: async (key) => {
+ ensureString(key)
+ return store.get(key)
+ },
+ clear: async auth => {
+ if (auth !== authKey) {
+ getAuthKey()
+ throw new Error(`unauthorized`)
+ }
+ store.reset()
+ }
+ }
+
+ exports.StoreType = `lru-cache`
+ console.log(`lru-cache`)
+}
diff --git a/deploy/package.json b/deploy/package.json
index 70050ceae..ebdc523ee 100644
--- a/deploy/package.json
+++ b/deploy/package.json
@@ -21,6 +21,7 @@
"hbp-seafile": "0.0.6",
"helmet-csp": "^2.8.0",
"jwt-decode": "^2.2.0",
+ "lru-cache": "^5.1.1",
"memorystore": "^1.6.1",
"nomiseco": "0.0.2",
"openid-client": "^2.4.5",
diff --git a/src/res/ext/MNI152.json b/src/res/ext/MNI152.json
index e72d8a90d..adb408c22 100644
--- a/src/res/ext/MNI152.json
+++ b/src/res/ext/MNI152.json
@@ -8,6 +8,7 @@
"nehubaConfigURL": "nehubaConfig/MNI152NehubaConfig",
"parcellations": [
{
+ "fullId": "minds/core/parcellationatlas/v1.0.0/94c1125b-b87e-45e4-901c-00daee7f2579",
"name": "JuBrain Cytoarchitectonic Atlas",
"ngId": "jubrain mni152 v18 left",
"auxillaryMeshIndices": [
diff --git a/src/res/ext/allenMouse.json b/src/res/ext/allenMouse.json
index dad3a22fb..724ccc849 100644
--- a/src/res/ext/allenMouse.json
+++ b/src/res/ext/allenMouse.json
@@ -1,5 +1,6 @@
{
"name": "Allen Mouse Common Coordinate Framework v3",
+ "fullId": "minds/core/referencespace/v1.0.0/265d32a0-3d84-40a5-926f-bf89f68212b9",
"type": "template",
"species": "Mouse",
"ngId": "stpt",
@@ -11,6 +12,7 @@
"parcellations": [
{
"ngId": "v3_2017",
+ "fullId": "minds/core/parcellationatlas/v1.0.0/05655b58-3b6f-49db-b285-64b5a0276f83",
"name": "Allen Mouse Common Coordinate Framework v3 2017",
"ngData": null,
"type": "parcellation",
@@ -19361,6 +19363,7 @@
}
},
{
+ "fullId": "minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
"ngId": "atlas",
"name": "Allen Mouse Common Coordinate Framework v3 2015",
"ngData": null,
diff --git a/src/res/ext/atlas/atlas_allenMouse.json b/src/res/ext/atlas/atlas_allenMouse.json
new file mode 100644
index 000000000..a882d0924
--- /dev/null
+++ b/src/res/ext/atlas/atlas_allenMouse.json
@@ -0,0 +1,8 @@
+{
+ "@id": "juelich/iav/atlas/v1.0.0/2",
+ "name": "Allen Mouse Common Coordinate Framework v3",
+ "templateSpaces": [{
+ "@id": "minds/core/referencespace/v1.0.0/265d32a0-3d84-40a5-926f-bf89f68212b9",
+ "name": "Allen Mouse Common Coordinate Framework v3"
+ }]
+}
\ No newline at end of file
diff --git a/src/res/ext/atlas/atlas_multiLevelHuman.json b/src/res/ext/atlas/atlas_multiLevelHuman.json
new file mode 100644
index 000000000..1fcaeb8c2
--- /dev/null
+++ b/src/res/ext/atlas/atlas_multiLevelHuman.json
@@ -0,0 +1,14 @@
+{
+ "@id": "juelich/iav/atlas/v1.0.0/1",
+ "name": "Multilevel Human Atlas",
+ "templateSpaces": [{
+ "@id": "minds/core/referencespace/v1.0.0/a1655b99-82f1-420f-a3c2-fe80fd4c8588",
+ "name": "Big Brain (Histology)"
+ }, {
+ "@id": "minds/core/referencespace/v1.0.0/7f39f7be-445b-47c0-9791-e971c0b6d992",
+ "name": "MNI Colin 27"
+ }, {
+ "@id": "minds/core/referencespace/v1.0.0/dafcffc5-4826-4bf1-8ff6-46b8a31ff8e2",
+ "name": "MNI 152 ICBM 2009c Nonlinear Asymmetric"
+ }]
+}
\ No newline at end of file
diff --git a/src/res/ext/atlas/atlas_waxholmRat.json b/src/res/ext/atlas/atlas_waxholmRat.json
new file mode 100644
index 000000000..f43a85377
--- /dev/null
+++ b/src/res/ext/atlas/atlas_waxholmRat.json
@@ -0,0 +1,8 @@
+{
+ "@id": "minds/core/parcellationatlas/v1.0.0/522b368e-49a3-49fa-88d3-0870a307974a",
+ "name": "Waxholm Space atlas of the Sprague Dawley rat brain",
+ "templateSpaces": [{
+ "@id": "minds/core/referencespace/v1.0.0/d5717c4a-0fa1-46e6-918c-b8003069ade8",
+ "name": "Waxholm Space rat brain MRI/DTI"
+ }]
+}
\ No newline at end of file
diff --git a/src/res/ext/bigbrain.json b/src/res/ext/bigbrain.json
index b49c34f64..ac63bdc45 100644
--- a/src/res/ext/bigbrain.json
+++ b/src/res/ext/bigbrain.json
@@ -14,6 +14,7 @@
"nehubaConfigURL": "nehubaConfig/bigbrainNehubaConfig",
"parcellations": [
{
+ "fullId": "minds/core/parcellationatlas/v1.0.0/94c1125b-b87e-45e4-901c-00daee7f2579",
"name": "Cytoarchitectonic Maps",
"properties": {
"description": "This dataset contains cytoarchitectonic maps of brain regions in the BigBrain space [Amunts et al. 2013]. The mappings were created using the semi-automatic method presented in Schleicher et al. 1999, based on coronal histological sections on 1 micron resolution. Mappings are available on approximately every 100th section for each region. They were then transformed to the sections of the 3D reconstructed BigBrain space using the transformations used in Amunts et al. 2013, which were provided by Claude Lepage (McGill). Only a few cytoarchitectonic maps in the Big Brain are currently **fully mapped**, based on a workflow that automatically fills in missing sections based on expert annotations. Other 3D maps are available in a preliminary version, in which the expert annotations in the Big Brain space were simply **interpolated**."
diff --git a/src/res/ext/colin.json b/src/res/ext/colin.json
index 0bfba5ad7..1ee5eedbb 100644
--- a/src/res/ext/colin.json
+++ b/src/res/ext/colin.json
@@ -8,6 +8,7 @@
"nehubaConfigURL": "nehubaConfig/colinNehubaConfig",
"parcellations": [
{
+ "fullId": "minds/core/parcellationatlas/v1.0.0/94c1125b-b87e-45e4-901c-00daee7f2579",
"name": "JuBrain Cytoarchitectonic Atlas",
"ngId": "jubrain colin v18 left",
"auxillaryMeshIndices": [
diff --git a/src/res/ext/waxholmRatV2_0.json b/src/res/ext/waxholmRatV2_0.json
index c1c5851eb..744753088 100644
--- a/src/res/ext/waxholmRatV2_0.json
+++ b/src/res/ext/waxholmRatV2_0.json
@@ -1,5 +1,6 @@
{
"name": "Waxholm Space rat brain MRI/DTI",
+ "fullId": "minds/core/referencespace/v1.0.0/d5717c4a-0fa1-46e6-918c-b8003069ade8",
"type": "template",
"species": "Rat",
"useTheme": "dark",
--
GitLab