From b97af9e4dc0a706e24e9cc994fa2ad1b42ccedeb Mon Sep 17 00:00:00 2001
From: Xiao Gui <xgui3783@gmail.com>
Date: Mon, 6 Jan 2020 11:38:10 +0100
Subject: [PATCH] WIP triage
---
deploy/auth/util.spec.js | 2 +-
deploy/datasets/query.js | 160 +-----------------
deploy/datasets/query.spec.js | 0
deploy/datasets/testData/allen2015.js | 113 +++++++++++++
deploy/datasets/testData/waxholmv2.js | 113 +++++++++++++
deploy/datasets/util.js | 233 ++++++++++++++++++++++++--
deploy/datasets/util.spec.js | 77 ++++++++-
deploy/package.json | 6 +-
8 files changed, 524 insertions(+), 180 deletions(-)
create mode 100644 deploy/datasets/query.spec.js
create mode 100644 deploy/datasets/testData/allen2015.js
create mode 100644 deploy/datasets/testData/waxholmv2.js
diff --git a/deploy/auth/util.spec.js b/deploy/auth/util.spec.js
index 7d05de884..f29c8b2d8 100644
--- a/deploy/auth/util.spec.js
+++ b/deploy/auth/util.spec.js
@@ -31,4 +31,4 @@ describe('util.js with env', (done) => {
const { getPublicAccessToken } = await util()
return getPublicAccessToken().should.be.fulfilled
})
-})
\ No newline at end of file
+})
diff --git a/deploy/datasets/query.js b/deploy/datasets/query.js
index 2cb5682ed..eb5571d75 100644
--- a/deploy/datasets/query.js
+++ b/deploy/datasets/query.js
@@ -3,9 +3,8 @@ const request = require('request')
const URL = require('url')
const path = require('path')
const archiver = require('archiver')
-const { getCommonSenseDsFilter } = require('./supplements/commonSense')
const { getPreviewFile, hasPreview } = require('./supplements/previewFile')
-const { init: kgQueryUtilInit, getUserKGRequestParam } = require('./util')
+const { init: kgQueryUtilInit, getUserKGRequestParam, filterDatasets } = require('./util')
let cachedData = null
let otherQueryResult = null
@@ -44,7 +43,6 @@ const fetchDatasetFromKg = async ({ user } = {}) => {
})
}
-
const cacheData = ({ results, ...rest }) => {
cachedData = results
otherQueryResult = rest
@@ -55,6 +53,7 @@ let fetchingPublicDataInProgress = false
let getPublicDsPr
const getPublicDs = async () => {
+ console.log(`fetching public ds ...`)
/**
* every request to public ds will trigger a refresh pull from master KG (throttled pending on resolved request)
@@ -63,6 +62,7 @@ const getPublicDs = async () => {
fetchingPublicDataInProgress = true
getPublicDsPr = fetchDatasetFromKg()
.then(_ => {
+ console.log(`public ds fetched!`)
fetchingPublicDataInProgress = false
getPublicDsPr = null
return _
@@ -80,158 +80,6 @@ const getDs = ({ user }) => user
? fetchDatasetFromKg({ user }).then(({ results }) => results)
: getPublicDs()
-/**
- * Needed by filter by parcellation
- */
-
-const flattenArray = (array) => {
- return array.concat(
- ...array.map(item => item.children && item.children instanceof Array
- ? flattenArray(item.children)
- : [])
- )
-}
-
-const readConfigFile = (filename) => new Promise((resolve, reject) => {
- let filepath
- if (process.env.NODE_ENV === 'production') {
- filepath = path.join(__dirname, '..', 'res', filename)
- } else {
- filepath = path.join(__dirname, '..', '..', 'src', 'res', 'ext', filename)
- }
- fs.readFile(filepath, 'utf-8', (err, data) => {
- if (err) reject(err)
- resolve(data)
- })
-})
-
-const populateSet = (flattenedRegions, set = new Set()) => {
- if (!(set instanceof Set)) throw `set needs to be an instance of Set`
- if (!(flattenedRegions instanceof Array)) throw `flattenedRegions needs to be an instance of Array`
- for (const region of flattenedRegions) {
- const { name, relatedAreas } = region
- if (name) set.add(name)
- if (relatedAreas && relatedAreas instanceof Array && relatedAreas.length > 0) {
- for (const relatedArea of relatedAreas) {
- if(typeof relatedArea === 'string') set.add(relatedArea)
- else console.warn(`related area not an instance of String. skipping`, relatedArea)
- }
- }
- }
- return set
-}
-
-let juBrainSet = new Set(),
- shortBundleSet = new Set(),
- longBundleSet = new Set(),
- waxholm1Set = new Set(),
- waxholm2Set = new Set(),
- waxholm3Set = new Set(),
- allen2015Set = new Set(),
- allen2017Set = new Set()
-
-readConfigFile('MNI152.json')
- .then(data => JSON.parse(data))
- .then(json => {
- const longBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Long Bundle').regions)
- const shortBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Short Bundle').regions)
- const jubrain = flattenArray(json.parcellations.find(({ name }) => 'JuBrain Cytoarchitectonic Atlas' === name).regions)
- longBundleSet = populateSet(longBundle)
- shortBundleSet = populateSet(shortBundle)
- juBrainSet = populateSet(jubrain)
- })
- .catch(console.error)
-
-readConfigFile('waxholmRatV2_0.json')
- .then(data => JSON.parse(data))
- .then(json => {
- const waxholm3 = flattenArray(json.parcellations[0].regions)
- const waxholm2 = flattenArray(json.parcellations[1].regions)
- const waxholm1 = flattenArray(json.parcellations[2].regions)
-
- waxholm1Set = populateSet(waxholm1)
- waxholm2Set = populateSet(waxholm2)
- waxholm3Set = populateSet(waxholm3)
- })
- .catch(console.error)
-
-readConfigFile('allenMouse.json')
- .then(data => JSON.parse(data))
- .then(json => {
- const flattenedAllen2017 = flattenArray(json.parcellations[0].regions)
- allen2017Set = populateSet(flattenedAllen2017)
-
- const flattenedAllen2015 = flattenArray(json.parcellations[1].regions)
- allen2015Set = populateSet(flattenedAllen2015)
- })
-
-const filterByPRSet = (prs, atlasPrSet = new Set()) => {
- if (!(atlasPrSet instanceof Set)) throw `atlasPrSet needs to be a set!`
- return prs.some(({ name, alias }) => atlasPrSet.has(alias) || atlasPrSet.has(name))
-}
-
-const filterByPRName = ({ parcellationName = null, dataset = {parcellationAtlas: []} } = {}) => parcellationName === null || dataset.parcellationAtlas.length === 0
- ? true
- : (dataset.parcellationAtlas || []).some(({ name }) => name === parcellationName)
-
-const filter = (datasets = [], { templateName, parcellationName }) => datasets
- .filter(getCommonSenseDsFilter({ templateName, parcellationName }))
- .filter(ds => {
- if (/infant/.test(ds.name))
- return false
- if (templateName) {
- return ds.referenceSpaces.some(rs => rs.name === templateName)
- }
- if (parcellationName) {
- if (ds.parcellationRegion.length === 0) return false
-
- let useSet
-
- // temporary measure
- // TODO ask curaion team re name of jubrain atlas
- let overwriteParcellationName
- switch (parcellationName) {
- case 'Cytoarchitectonic Maps':
- case 'JuBrain Cytoarchitectonic Atlas':
- useSet = juBrainSet
- overwriteParcellationName = 'Jülich Cytoarchitechtonic Brain Atlas (human)'
- break;
- case 'Fibre Bundle Atlas - Short Bundle':
- useSet = shortBundleSet
- break;
- case 'Fibre Bundle Atlas - Long Bundle':
- useSet = longBundleSet
- break;
- case 'Waxholm Space rat brain atlas v1':
- useSet = waxholm1Set
- break;
- case 'Waxholm Space rat brain atlas v2':
- useSet = waxholm2Set
- break;
- case 'Waxholm Space rat brain atlas v3':
- useSet = waxholm3Set
- break;
- case 'Allen Mouse Common Coordinate Framework v3 2015':
- useSet = allen2015Set
- break;
- case 'Allen Mouse Common Coordinate Framework v3 2017':
- useSet = allen2017Set
- break;
- default:
- useSet = new Set()
- }
- return filterByPRName({ dataset: ds, parcellationName: overwriteParcellationName || parcellationName }) && filterByPRSet(ds.parcellationRegion, useSet)
- }
-
- return false
- })
- .map(ds => {
- return {
- ...ds,
- preview: hasPreview({ datasetName: ds.name })
- }
- })
-
/**
* on init, populate the cached data
*/
@@ -241,7 +89,7 @@ const init = async () => {
}
const getDatasets = ({ templateName, parcellationName, user }) => getDs({ user })
- .then(json => filter(json, { templateName, parcellationName }))
+ .then(json => filterDatasets(json, { templateName, parcellationName }))
const getPreview = ({ datasetName, templateSelected }) => getPreviewFile({ datasetName, templateSelected })
diff --git a/deploy/datasets/query.spec.js b/deploy/datasets/query.spec.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/deploy/datasets/testData/allen2015.js b/deploy/datasets/testData/allen2015.js
new file mode 100644
index 000000000..a1fb58853
--- /dev/null
+++ b/deploy/datasets/testData/allen2015.js
@@ -0,0 +1,113 @@
+module.exports = [
+ {
+ "formats": [],
+ "datasetDOI": [
+ {
+ "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+ "doi": "10.1016/j.neuroscience.2009.03.048"
+ }
+ ],
+ "activity": [
+ {
+ "protocols": [
+ "Immunohistochemistry",
+ "Atlas",
+ "Brain-wide",
+ "Synaptic transmission",
+ "Neurtransmitter transport",
+ "Glutamate uptake",
+ "GLT1"
+ ],
+ "preparation": [
+ "Ex vivo"
+ ]
+ }
+ ],
+ "referenceSpaces": [],
+ "methods": [
+ "Immunohistochemistry"
+ ],
+ "custodians": [
+ "Danbolt, Niels C."
+ ],
+ "project": [
+ "Rodent brain neurotransporter atlas: GLT1"
+ ],
+ "description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
+ "parcellationAtlas": [
+ {
+ "name": "Allen Mouse Common Coordinate Framework v3 2015",
+ "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
+ "id": "39a1384b-8413-4d27-af8d-22432225401f"
+ },
+ {
+ "name": "Waxholm Space rat brain atlas v2",
+ "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
+ "id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
+ }
+ ],
+ "licenseInfo": [
+ {
+ "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+ "url": "https://creativecommons.org/licenses/by-sa/4.0"
+ }
+ ],
+ "embargoStatus": [
+ "Free"
+ ],
+ "license": [
+ {
+ "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+ "relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
+ }
+ ],
+ "parcellationRegion": [
+ {
+ "species": [],
+ "name": "Mouse Whole brain (v3 2015)",
+ "alias": "Whole brain"
+ },
+ {
+ "species": [
+ {
+ "identifier": [
+ "e9a384ea8a4edf817710b6edef5f2940",
+ "5401fdb1d638c2bc5b68241560cddac0"
+ ],
+ "name": "Rattus norvegicus",
+ "@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
+ }
+ ],
+ "name": "Whole Brain ",
+ "alias": "Whole brain"
+ }
+ ],
+ "species": [
+ "Rattus norvegicus",
+ "Mus musculus"
+ ],
+ "name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
+ "files": [],
+ "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
+ "contributors": [
+ "Danbolt, Nils C.",
+ "Bjaalie, Jan G.",
+ "Leergaard, Trygve B.",
+ "Lehre, K.P.",
+ "Real, Katia",
+ "Scott, Heather A.",
+ "Holmseth, Silvia"
+ ],
+ "id": "63bbb845ac6d2f1839f919c2ef0455bc",
+ "kgReference": [
+ "10.25493/Y147-2CE"
+ ],
+ "publications": [
+ {
+ "name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
+ "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+ "doi": "10.1016/j.neuroscience.2009.03.048"
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/deploy/datasets/testData/waxholmv2.js b/deploy/datasets/testData/waxholmv2.js
new file mode 100644
index 000000000..a1fb58853
--- /dev/null
+++ b/deploy/datasets/testData/waxholmv2.js
@@ -0,0 +1,113 @@
+module.exports = [
+ {
+ "formats": [],
+ "datasetDOI": [
+ {
+ "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+ "doi": "10.1016/j.neuroscience.2009.03.048"
+ }
+ ],
+ "activity": [
+ {
+ "protocols": [
+ "Immunohistochemistry",
+ "Atlas",
+ "Brain-wide",
+ "Synaptic transmission",
+ "Neurtransmitter transport",
+ "Glutamate uptake",
+ "GLT1"
+ ],
+ "preparation": [
+ "Ex vivo"
+ ]
+ }
+ ],
+ "referenceSpaces": [],
+ "methods": [
+ "Immunohistochemistry"
+ ],
+ "custodians": [
+ "Danbolt, Niels C."
+ ],
+ "project": [
+ "Rodent brain neurotransporter atlas: GLT1"
+ ],
+ "description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
+ "parcellationAtlas": [
+ {
+ "name": "Allen Mouse Common Coordinate Framework v3 2015",
+ "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
+ "id": "39a1384b-8413-4d27-af8d-22432225401f"
+ },
+ {
+ "name": "Waxholm Space rat brain atlas v2",
+ "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
+ "id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
+ }
+ ],
+ "licenseInfo": [
+ {
+ "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+ "url": "https://creativecommons.org/licenses/by-sa/4.0"
+ }
+ ],
+ "embargoStatus": [
+ "Free"
+ ],
+ "license": [
+ {
+ "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+ "relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
+ }
+ ],
+ "parcellationRegion": [
+ {
+ "species": [],
+ "name": "Mouse Whole brain (v3 2015)",
+ "alias": "Whole brain"
+ },
+ {
+ "species": [
+ {
+ "identifier": [
+ "e9a384ea8a4edf817710b6edef5f2940",
+ "5401fdb1d638c2bc5b68241560cddac0"
+ ],
+ "name": "Rattus norvegicus",
+ "@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
+ }
+ ],
+ "name": "Whole Brain ",
+ "alias": "Whole brain"
+ }
+ ],
+ "species": [
+ "Rattus norvegicus",
+ "Mus musculus"
+ ],
+ "name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
+ "files": [],
+ "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
+ "contributors": [
+ "Danbolt, Nils C.",
+ "Bjaalie, Jan G.",
+ "Leergaard, Trygve B.",
+ "Lehre, K.P.",
+ "Real, Katia",
+ "Scott, Heather A.",
+ "Holmseth, Silvia"
+ ],
+ "id": "63bbb845ac6d2f1839f919c2ef0455bc",
+ "kgReference": [
+ "10.25493/Y147-2CE"
+ ],
+ "publications": [
+ {
+ "name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
+ "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+ "doi": "10.1016/j.neuroscience.2009.03.048"
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/deploy/datasets/util.js b/deploy/datasets/util.js
index 983ddd931..186eaa9ff 100644
--- a/deploy/datasets/util.js
+++ b/deploy/datasets/util.js
@@ -1,4 +1,8 @@
const kgQueryUtil = require('./../auth/util')
+const { getCommonSenseDsFilter } = require('./supplements/commonSense')
+const { hasPreview } = require('./supplements/previewFile')
+const path = require('path')
+const fs = require('fs')
let getPublicAccessToken
@@ -25,6 +29,185 @@ const getUserKGRequestParam = async ({ user }) => {
}
}
+/**
+ * Needed by filter by parcellation
+ */
+
+const flattenArray = (array) => {
+ return array.concat(
+ ...array.map(item => item.children && item.children instanceof Array
+ ? flattenArray(item.children)
+ : [])
+ )
+}
+
+const readConfigFile = (filename) => new Promise((resolve, reject) => {
+ let filepath
+ if (process.env.NODE_ENV === 'production') {
+ filepath = path.join(__dirname, '..', 'res', filename)
+ } else {
+ filepath = path.join(__dirname, '..', '..', 'src', 'res', 'ext', filename)
+ }
+ fs.readFile(filepath, 'utf-8', (err, data) => {
+ if (err) reject(err)
+ resolve(data)
+ })
+})
+
+const populateSet = (flattenedRegions, set = new Set()) => {
+ if (!(set instanceof Set)) throw `set needs to be an instance of Set`
+ if (!(flattenedRegions instanceof Array)) throw `flattenedRegions needs to be an instance of Array`
+ for (const region of flattenedRegions) {
+ const { name, relatedAreas } = region
+ if (name) set.add(name)
+ if (relatedAreas && relatedAreas instanceof Array && relatedAreas.length > 0) {
+ for (const relatedArea of relatedAreas) {
+ if(typeof relatedArea === 'string') set.add(relatedArea)
+ else console.warn(`related area not an instance of String. skipping`, relatedArea)
+ }
+ }
+ }
+ return set
+}
+
+const initPrArray = []
+
+let juBrainSet = new Set(),
+ shortBundleSet = new Set(),
+ longBundleSet = new Set(),
+ waxholm1Set = new Set(),
+ waxholm2Set = new Set(),
+ waxholm3Set = new Set(),
+ allen2015Set = new Set(),
+ allen2017Set = new Set()
+
+initPrArray.push(
+ readConfigFile('MNI152.json')
+ .then(data => JSON.parse(data))
+ .then(json => {
+ const longBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Long Bundle').regions)
+ const shortBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Short Bundle').regions)
+ const jubrain = flattenArray(json.parcellations.find(({ name }) => 'JuBrain Cytoarchitectonic Atlas' === name).regions)
+ longBundleSet = populateSet(longBundle)
+ shortBundleSet = populateSet(shortBundle)
+ juBrainSet = populateSet(jubrain)
+ })
+ .catch(console.error)
+)
+
+initPrArray.push(
+ readConfigFile('waxholmRatV2_0.json')
+ .then(data => JSON.parse(data))
+ .then(json => {
+ const waxholm3 = flattenArray(json.parcellations[0].regions)
+ const waxholm2 = flattenArray(json.parcellations[1].regions)
+ const waxholm1 = flattenArray(json.parcellations[2].regions)
+
+ waxholm1Set = populateSet(waxholm1)
+ waxholm2Set = populateSet(waxholm2)
+ waxholm3Set = populateSet(waxholm3)
+ })
+ .catch(console.error)
+)
+
+initPrArray.push(
+ readConfigFile('allenMouse.json')
+ .then(data => JSON.parse(data))
+ .then(json => {
+ const flattenedAllen2017 = flattenArray(json.parcellations[0].regions)
+ allen2017Set = populateSet(flattenedAllen2017)
+
+ const flattenedAllen2015 = flattenArray(json.parcellations[1].regions)
+ allen2015Set = populateSet(flattenedAllen2015)
+ })
+ .catch(console.error)
+)
+
+const datasetRegionExistsInParcellationRegion = async (prs, atlasPrSet = new Set()) => {
+ if (!(atlasPrSet instanceof Set)) throw `atlasPrSet needs to be a set!`
+ await Promise.all(initPrArray)
+ return prs.some(({ name, alias }) => atlasPrSet.has(alias) || atlasPrSet.has(name))
+}
+
+const datasetBelongToParcellation = ({ parcellationName = null, dataset = {parcellationAtlas: []} } = {}) => parcellationName === null || dataset.parcellationAtlas.length === 0
+ ? true
+ : (dataset.parcellationAtlas || []).some(({ name }) => name === parcellationName)
+
+const filterDataset = async (dataset = null, { templateName, parcellationName }) => {
+
+ if (/infant/.test(dataset.name)) return false
+ if (templateName) {
+ return dataset.referenceSpaces.some(rs => rs.name === templateName)
+ }
+ if (parcellationName) {
+ if (dataset.parcellationRegion.length === 0) return false
+
+ let useSet
+
+ // temporary measure
+ // TODO ask curaion team re name of jubrain atlas
+ let overwriteParcellationName
+ switch (parcellationName) {
+ case 'Cytoarchitectonic Maps':
+ case 'JuBrain Cytoarchitectonic Atlas':
+ useSet = juBrainSet
+ overwriteParcellationName = 'Jülich Cytoarchitechtonic Brain Atlas (human)'
+ break;
+ case 'Fibre Bundle Atlas - Short Bundle':
+ useSet = shortBundleSet
+ break;
+ case 'Fibre Bundle Atlas - Long Bundle':
+ useSet = longBundleSet
+ break;
+ case 'Waxholm Space rat brain atlas v1':
+ useSet = waxholm1Set
+ break;
+ case 'Waxholm Space rat brain atlas v2':
+ useSet = waxholm2Set
+ break;
+ case 'Waxholm Space rat brain atlas v3':
+ useSet = waxholm3Set
+ break;
+ case 'Allen Mouse Common Coordinate Framework v3 2015':
+ useSet = allen2015Set
+ break;
+ case 'Allen Mouse Common Coordinate Framework v3 2017':
+ useSet = allen2017Set
+ break;
+ default:
+ useSet = new Set()
+ }
+ return datasetBelongToParcellation({ dataset, parcellationName: overwriteParcellationName || parcellationName })
+ && await datasetRegionExistsInParcellationRegion(dataset.parcellationRegion, useSet)
+ }
+
+ return false
+}
+
+const filterDatasets = async (datasets = [], { templateName, parcellationName }) => {
+
+ // filter by commonsense first (species)
+ const commonSenseFilteredDatasets = datasets.filter(getCommonSenseDsFilter({ templateName, parcellationName }))
+
+ // filter by parcellation name and if region is in parcellation
+ const filteredDatasets = []
+ for (const dataset of commonSenseFilteredDatasets) {
+ if (await filterDataset(dataset, { templateName, parcellationName })) {
+ filteredDatasets.push(dataset)
+ }
+ }
+
+ // append if preview is available
+ const filteredDatasetsAppendingPreview = filteredDatasets.map(ds => {
+ return {
+ ...ds,
+ preview: hasPreview({ datasetName: ds.name })
+ }
+ })
+
+ return filteredDatasetsAppendingPreview
+}
+
const init = async () => {
if (process.env.ACCESS_TOKEN) {
if (process.env.NODE_ENV === 'production') console.error(`ACCESS_TOKEN set in production!`)
@@ -35,23 +218,45 @@ const init = async () => {
getPublicAccessToken = getPublic
}
-const retry = (fn) => {
- let retryId
- retryId = setTimeout(() => {
- fn()
- .then(() => {
- console.log(`retry succeeded, clearing retryId`)
- clearTimeout(retryId)
- })
- .catch(e => {
- console.warn(`retry failed, retrying in 5sec`)
- retry(fn)
- })
- }, 5000)
+const defaultConfig = {
+ timeout: 5000,
+ retries: 3
+}
+
+const retry = async (fn, { timeout = defaultConfig.timeout, retries = defaultConfig.retries } = defaultConfig) => {
+ let retryNo = 0
+ while (retryNo < retries) {
+ retryNo ++
+ try {
+ const result = await fn()
+ return result
+ } catch (e) {
+ console.warn(`fn failed, retry after ${timeout} milliseconds`)
+ await (() => new Promise(rs => setTimeout(rs, timeout)))()
+ }
+ }
+
+ throw new Error(`fn failed ${retires} times. Aborting.`)
}
module.exports = {
init,
getUserKGRequestParam,
- retry
+ retry,
+ filterDatasets,
+ datasetBelongToParcellation,
+ datasetRegionExistsInParcellationRegion,
+ _getParcellations: async () => {
+ await Promise.all(initPrArray)
+ return {
+ juBrainSet,
+ shortBundleSet,
+ longBundleSet,
+ waxholm1Set,
+ waxholm2Set,
+ waxholm3Set,
+ allen2015Set,
+ allen2017Set
+ }
+ }
}
\ No newline at end of file
diff --git a/deploy/datasets/util.spec.js b/deploy/datasets/util.spec.js
index a4dbd0090..523f3e3e9 100644
--- a/deploy/datasets/util.spec.js
+++ b/deploy/datasets/util.spec.js
@@ -1,10 +1,73 @@
-const { retry } = require('./util')
+const { retry, filterDatasets, datasetRegionExistsInParcellationRegion, _getParcellations } = require('./util')
+const { fake } = require('sinon')
+const { assert, expect } = require('chai')
+const waxholmv2 = require('./testData/waxholmv2')
-let val = 0
+describe('datasets/util.js', () => {
-const prFn = () => {
- val++
- return val >=3 ? Promise.resolve() : Promise.reject()
-}
+ // describe('retry', () => {
-retry(() => prFn())
\ No newline at end of file
+ // let val = 0
+
+ // const failCall = fake()
+ // const succeedCall = fake()
+
+ // const prFn = () => {
+ // val++
+ // return val >=3
+ // ? (succeedCall(), Promise.resolve())
+ // : (failCall(), Promise.reject())
+ // }
+
+ // beforeEach(() => {
+ // val = 0
+ // succeedCall.resetHistory()
+ // failCall.resetHistory()
+ // })
+
+ // it('retry until succeed', async () => {
+ // await retry(prFn)
+ // assert(succeedCall.called)
+ // assert(failCall.calledTwice)
+ // })
+
+ // it('retry with shorter timeouts', async () => {
+ // await retry(prFn, { timeout: 100 })
+ // assert(succeedCall.called)
+ // assert(failCall.calledTwice)
+ // })
+
+ // it('when retries excceeded, retry fn throws', async () => {
+ // try {
+ // await retry(prFn, { timeout: 100, retries: 2 })
+ // assert(false, 'retry fn should throw if retries exceed')
+ // } catch (e) {
+ // assert(true)
+ // }
+ // })
+ // })
+
+ describe('datasetRegionExistsInParcellationRegion', () => {
+ it('should filter waxholm v2 properly', async () => {
+
+ const waxholmv2Pr = waxholmv2.map(dataset => {
+ return dataset.parcellationRegion
+ })
+
+ const { waxholm2Set } = await _getParcellations()
+ for (const pr of waxholmv2Pr){
+
+ const flag = await datasetRegionExistsInParcellationRegion(pr, waxholm2Set)
+ expect(flag).to.be.true
+ }
+ })
+ })
+
+ describe('filterDatasets', () => {
+ it('should filter waxholm v2 properly', async () => {
+ const filteredResult = await filterDatasets(waxholmv2, { parcellationName: 'Waxholm Space rat brain atlas v2' })
+ expect(filteredResult).to.have.length(1)
+ })
+ })
+
+})
diff --git a/deploy/package.json b/deploy/package.json
index e32f647bd..c0ac870c2 100644
--- a/deploy/package.json
+++ b/deploy/package.json
@@ -7,7 +7,8 @@
"start": "node server.js",
"test": "npm run testEnv && npm run testNoEnv",
"testEnv": "node -r dotenv/config ./node_modules/.bin/mocha ./test/mocha.test.js",
- "testNoEnv": "node ./node_modules/.bin/mocha ./test/mocha.test.noenv.js"
+ "testNoEnv": "node ./node_modules/.bin/mocha ./test/mocha.test.noenv.js",
+ "mocha": "mocha"
},
"keywords": [],
"author": "",
@@ -30,6 +31,7 @@
"chai-as-promised": "^7.1.1",
"cors": "^2.8.5",
"dotenv": "^6.2.0",
- "mocha": "^6.1.4"
+ "mocha": "^6.1.4",
+ "sinon": "^8.0.2"
}
}
--
GitLab