Skip to content
Snippets Groups Projects
Unverified Commit 3380b8b8 authored by xgui3783's avatar xgui3783 Committed by GitHub
Browse files

Merge pull request #6 from FZJ-INM1-BDA/chore/cleanUpDataset

Chore/clean up dataset
parents af955666 2619d2ad
No related branches found
No related tags found
No related merge requests found
name: Backend tests
on: [push]
env:
NODE_ENV: test
jobs:
install_dep:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [10.x, 12.x]
steps:
- uses: actions/checkout@v1
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: cd into deploy, npm install
run: |
cd deploy
npm i
- name: test no env
run: |
cd deploy
npm run testNoEnv
- name: test with env
env:
REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
HBP_CLIENTID: ${{ secrets.HBP_CLIENTID }}
HBP_CLIENTSECRET: ${{ secrets.HBP_CLIENTSECRET }}
run: |
cd deploy
npm run testEnv
......@@ -9,7 +9,7 @@ jobs:
strategy:
matrix:
node-version: [8.x, 10.x, 12.x]
node-version: [10.x, 12.x]
steps:
- uses: actions/checkout@v1
......
......@@ -23,7 +23,7 @@ describe('chai-as-promised.js', () => {
})
})
describe('util.js with env', (done) => {
describe('util.js with env', async () => {
it('when client id and client secret and refresh token is set, util should not throw', async () => {
......@@ -31,4 +31,4 @@ describe('util.js with env', (done) => {
const { getPublicAccessToken } = await util()
return getPublicAccessToken().should.be.fulfilled
})
})
\ No newline at end of file
})
......@@ -3,9 +3,8 @@ const request = require('request')
const URL = require('url')
const path = require('path')
const archiver = require('archiver')
const { getCommonSenseDsFilter } = require('./supplements/commonSense')
const { getPreviewFile, hasPreview } = require('./supplements/previewFile')
const { init: kgQueryUtilInit, getUserKGRequestParam } = require('./util')
const { init: kgQueryUtilInit, getUserKGRequestParam, filterDatasets } = require('./util')
let cachedData = null
let otherQueryResult = null
......@@ -44,7 +43,6 @@ const fetchDatasetFromKg = async ({ user } = {}) => {
})
}
const cacheData = ({ results, ...rest }) => {
cachedData = results
otherQueryResult = rest
......@@ -55,6 +53,7 @@ let fetchingPublicDataInProgress = false
let getPublicDsPr
const getPublicDs = async () => {
console.log(`fetching public ds ...`)
/**
* every request to public ds will trigger a refresh pull from master KG (throttled pending on resolved request)
......@@ -63,6 +62,7 @@ const getPublicDs = async () => {
fetchingPublicDataInProgress = true
getPublicDsPr = fetchDatasetFromKg()
.then(_ => {
console.log(`public ds fetched!`)
fetchingPublicDataInProgress = false
getPublicDsPr = null
return _
......@@ -80,158 +80,6 @@ const getDs = ({ user }) => user
? fetchDatasetFromKg({ user }).then(({ results }) => results)
: getPublicDs()
/**
* Needed by filter by parcellation
*/
const flattenArray = (array) => {
return array.concat(
...array.map(item => item.children && item.children instanceof Array
? flattenArray(item.children)
: [])
)
}
const readConfigFile = (filename) => new Promise((resolve, reject) => {
let filepath
if (process.env.NODE_ENV === 'production') {
filepath = path.join(__dirname, '..', 'res', filename)
} else {
filepath = path.join(__dirname, '..', '..', 'src', 'res', 'ext', filename)
}
fs.readFile(filepath, 'utf-8', (err, data) => {
if (err) reject(err)
resolve(data)
})
})
const populateSet = (flattenedRegions, set = new Set()) => {
if (!(set instanceof Set)) throw `set needs to be an instance of Set`
if (!(flattenedRegions instanceof Array)) throw `flattenedRegions needs to be an instance of Array`
for (const region of flattenedRegions) {
const { name, relatedAreas } = region
if (name) set.add(name)
if (relatedAreas && relatedAreas instanceof Array && relatedAreas.length > 0) {
for (const relatedArea of relatedAreas) {
if(typeof relatedArea === 'string') set.add(relatedArea)
else console.warn(`related area not an instance of String. skipping`, relatedArea)
}
}
}
return set
}
let juBrainSet = new Set(),
shortBundleSet = new Set(),
longBundleSet = new Set(),
waxholm1Set = new Set(),
waxholm2Set = new Set(),
waxholm3Set = new Set(),
allen2015Set = new Set(),
allen2017Set = new Set()
readConfigFile('MNI152.json')
.then(data => JSON.parse(data))
.then(json => {
const longBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Long Bundle').regions)
const shortBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Short Bundle').regions)
const jubrain = flattenArray(json.parcellations.find(({ name }) => 'JuBrain Cytoarchitectonic Atlas' === name).regions)
longBundleSet = populateSet(longBundle)
shortBundleSet = populateSet(shortBundle)
juBrainSet = populateSet(jubrain)
})
.catch(console.error)
readConfigFile('waxholmRatV2_0.json')
.then(data => JSON.parse(data))
.then(json => {
const waxholm3 = flattenArray(json.parcellations[0].regions)
const waxholm2 = flattenArray(json.parcellations[1].regions)
const waxholm1 = flattenArray(json.parcellations[2].regions)
waxholm1Set = populateSet(waxholm1)
waxholm2Set = populateSet(waxholm2)
waxholm3Set = populateSet(waxholm3)
})
.catch(console.error)
readConfigFile('allenMouse.json')
.then(data => JSON.parse(data))
.then(json => {
const flattenedAllen2017 = flattenArray(json.parcellations[0].regions)
allen2017Set = populateSet(flattenedAllen2017)
const flattenedAllen2015 = flattenArray(json.parcellations[1].regions)
allen2015Set = populateSet(flattenedAllen2015)
})
const filterByPRSet = (prs, atlasPrSet = new Set()) => {
if (!(atlasPrSet instanceof Set)) throw `atlasPrSet needs to be a set!`
return prs.some(({ name, alias }) => atlasPrSet.has(alias) || atlasPrSet.has(name))
}
const filterByPRName = ({ parcellationName = null, dataset = {parcellationAtlas: []} } = {}) => parcellationName === null || dataset.parcellationAtlas.length === 0
? true
: (dataset.parcellationAtlas || []).some(({ name }) => name === parcellationName)
const filter = (datasets = [], { templateName, parcellationName }) => datasets
.filter(getCommonSenseDsFilter({ templateName, parcellationName }))
.filter(ds => {
if (/infant/.test(ds.name))
return false
if (templateName) {
return ds.referenceSpaces.some(rs => rs.name === templateName)
}
if (parcellationName) {
if (ds.parcellationRegion.length === 0) return false
let useSet
// temporary measure
// TODO ask curaion team re name of jubrain atlas
let overwriteParcellationName
switch (parcellationName) {
case 'Cytoarchitectonic Maps':
case 'JuBrain Cytoarchitectonic Atlas':
useSet = juBrainSet
overwriteParcellationName = 'Jülich Cytoarchitechtonic Brain Atlas (human)'
break;
case 'Fibre Bundle Atlas - Short Bundle':
useSet = shortBundleSet
break;
case 'Fibre Bundle Atlas - Long Bundle':
useSet = longBundleSet
break;
case 'Waxholm Space rat brain atlas v1':
useSet = waxholm1Set
break;
case 'Waxholm Space rat brain atlas v2':
useSet = waxholm2Set
break;
case 'Waxholm Space rat brain atlas v3':
useSet = waxholm3Set
break;
case 'Allen Mouse Common Coordinate Framework v3 2015':
useSet = allen2015Set
break;
case 'Allen Mouse Common Coordinate Framework v3 2017':
useSet = allen2017Set
break;
default:
useSet = new Set()
}
return filterByPRName({ dataset: ds, parcellationName: overwriteParcellationName || parcellationName }) && filterByPRSet(ds.parcellationRegion, useSet)
}
return false
})
.map(ds => {
return {
...ds,
preview: hasPreview({ datasetName: ds.name })
}
})
/**
* on init, populate the cached data
*/
......@@ -241,7 +89,7 @@ const init = async () => {
}
const getDatasets = ({ templateName, parcellationName, user }) => getDs({ user })
.then(json => filter(json, { templateName, parcellationName }))
.then(json => filterDatasets(json, { templateName, parcellationName }))
const getPreview = ({ datasetName, templateSelected }) => getPreviewFile({ datasetName, templateSelected })
......
module.exports = [
{
"formats": [],
"datasetDOI": [
{
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
],
"activity": [
{
"protocols": [
"Immunohistochemistry",
"Atlas",
"Brain-wide",
"Synaptic transmission",
"Neurtransmitter transport",
"Glutamate uptake",
"GLT1"
],
"preparation": [
"Ex vivo"
]
}
],
"referenceSpaces": [],
"methods": [
"Immunohistochemistry"
],
"custodians": [
"Danbolt, Niels C."
],
"project": [
"Rodent brain neurotransporter atlas: GLT1"
],
"description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
"parcellationAtlas": [
{
"name": "Allen Mouse Common Coordinate Framework v3 2015",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
"id": "39a1384b-8413-4d27-af8d-22432225401f"
},
{
"name": "Waxholm Space rat brain atlas v2",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
"id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
}
],
"licenseInfo": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"url": "https://creativecommons.org/licenses/by-sa/4.0"
}
],
"embargoStatus": [
"Free"
],
"license": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
}
],
"parcellationRegion": [
{
"species": [],
"name": "Mouse Whole brain (v3 2015)",
"alias": "Whole brain"
},
{
"species": [
{
"identifier": [
"e9a384ea8a4edf817710b6edef5f2940",
"5401fdb1d638c2bc5b68241560cddac0"
],
"name": "Rattus norvegicus",
"@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
}
],
"name": "Whole Brain ",
"alias": "Whole brain"
}
],
"species": [
"Rattus norvegicus",
"Mus musculus"
],
"name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
"files": [],
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
"contributors": [
"Danbolt, Nils C.",
"Bjaalie, Jan G.",
"Leergaard, Trygve B.",
"Lehre, K.P.",
"Real, Katia",
"Scott, Heather A.",
"Holmseth, Silvia"
],
"id": "63bbb845ac6d2f1839f919c2ef0455bc",
"kgReference": [
"10.25493/Y147-2CE"
],
"publications": [
{
"name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
]
},
{
"formats": [],
"datasetDOI": [
{
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
],
"activity": [
{
"protocols": [
"Immunohistochemistry",
"Atlas",
"Brain-wide",
"Synaptic transmission",
"Neurtransmitter transport",
"Glutamate uptake",
"GLT1"
],
"preparation": [
"Ex vivo"
]
}
],
"referenceSpaces": [],
"methods": [
"Immunohistochemistry"
],
"custodians": [
"Danbolt, Niels C."
],
"project": [
"Rodent brain neurotransporter atlas: GLT1"
],
"description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
"parcellationAtlas": [
{
"name": "Allen Mouse Common Coordinate Framework v3 2015",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
"id": "39a1384b-8413-4d27-af8d-22432225401f"
},
{
"name": "Waxholm Space rat brain atlas v2",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
"id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
}
],
"licenseInfo": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"url": "https://creativecommons.org/licenses/by-sa/4.0"
}
],
"embargoStatus": [
"Free"
],
"license": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
}
],
"parcellationRegion": [
{
"species": [],
"name": "Mouse Whole brain (v3 2015)",
"alias": "Whole brain"
},
{
"species": [
{
"identifier": [
"e9a384ea8a4edf817710b6edef5f2940",
"5401fdb1d638c2bc5b68241560cddac0"
],
"name": "Rattus norvegicus",
"@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
}
],
"name": "Whole Brain ",
"alias": "Whole brain"
}
],
"species": [
"Rattus norvegicus",
"Mus musculus"
],
"name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
"files": [],
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
"contributors": [
"Danbolt, Nils C.",
"Bjaalie, Jan G.",
"Leergaard, Trygve B.",
"Lehre, K.P.",
"Real, Katia",
"Scott, Heather A.",
"Holmseth, Silvia"
],
"id": "63bbb845ac6d2f1839f919c2ef0455bc",
"kgReference": [
"10.25493/Y147-2CE"
],
"publications": [
{
"name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
]
}
]
\ No newline at end of file
module.exports = [
{
"formats": [],
"datasetDOI": [
{
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
],
"activity": [
{
"protocols": [
"Immunohistochemistry",
"Atlas",
"Brain-wide",
"Synaptic transmission",
"Neurtransmitter transport",
"Glutamate uptake",
"GLT1"
],
"preparation": [
"Ex vivo"
]
}
],
"referenceSpaces": [],
"methods": [
"Immunohistochemistry"
],
"custodians": [
"Danbolt, Niels C."
],
"project": [
"Rodent brain neurotransporter atlas: GLT1"
],
"description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
"parcellationAtlas": [
{
"name": "Allen Mouse Common Coordinate Framework v3 2015",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
"id": "39a1384b-8413-4d27-af8d-22432225401f"
},
{
"name": "Waxholm Space rat brain atlas v2",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
"id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
}
],
"licenseInfo": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"url": "https://creativecommons.org/licenses/by-sa/4.0"
}
],
"embargoStatus": [
"Free"
],
"license": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
}
],
"parcellationRegion": [
{
"species": [],
"name": "Mouse Whole brain (v3 2015)",
"alias": "Whole brain"
},
{
"species": [
{
"identifier": [
"e9a384ea8a4edf817710b6edef5f2940",
"5401fdb1d638c2bc5b68241560cddac0"
],
"name": "Rattus norvegicus",
"@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
}
],
"name": "Whole Brain ",
"alias": "Whole brain"
}
],
"species": [
"Rattus norvegicus",
"Mus musculus"
],
"name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
"files": [],
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
"contributors": [
"Danbolt, Nils C.",
"Bjaalie, Jan G.",
"Leergaard, Trygve B.",
"Lehre, K.P.",
"Real, Katia",
"Scott, Heather A.",
"Holmseth, Silvia"
],
"id": "63bbb845ac6d2f1839f919c2ef0455bc",
"kgReference": [
"10.25493/Y147-2CE"
],
"publications": [
{
"name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
]
},
{
"formats": [],
"datasetDOI": [
{
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
],
"activity": [
{
"protocols": [
"Immunohistochemistry",
"Atlas",
"Brain-wide",
"Synaptic transmission",
"Neurtransmitter transport",
"Glutamate uptake",
"GLT1"
],
"preparation": [
"Ex vivo"
]
}
],
"referenceSpaces": [],
"methods": [
"Immunohistochemistry"
],
"custodians": [
"Danbolt, Niels C."
],
"project": [
"Rodent brain neurotransporter atlas: GLT1"
],
"description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
"parcellationAtlas": [
{
"name": "Allen Mouse Common Coordinate Framework v3 2015",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
"id": "39a1384b-8413-4d27-af8d-22432225401f"
},
{
"name": "Waxholm Space rat brain atlas v2",
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
"id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
}
],
"licenseInfo": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"url": "https://creativecommons.org/licenses/by-sa/4.0"
}
],
"embargoStatus": [
"Free"
],
"license": [
{
"name": "Creative Commons Attribution-ShareAlike 4.0 International",
"relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
}
],
"parcellationRegion": [
{
"species": [],
"name": "Mouse Whole brain (v3 2015)",
"alias": "Whole brain"
},
{
"species": [
{
"identifier": [
"e9a384ea8a4edf817710b6edef5f2940",
"5401fdb1d638c2bc5b68241560cddac0"
],
"name": "Rattus norvegicus",
"@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
}
],
"name": "Whole Brain ",
"alias": "Whole brain"
}
],
"species": [
"Rattus norvegicus",
"Mus musculus"
],
"name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
"files": [],
"fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
"contributors": [
"Danbolt, Nils C.",
"Bjaalie, Jan G.",
"Leergaard, Trygve B.",
"Lehre, K.P.",
"Real, Katia",
"Scott, Heather A.",
"Holmseth, Silvia"
],
"id": "63bbb845ac6d2f1839f919c2ef0455bc",
"kgReference": [
"10.25493/Y147-2CE"
],
"publications": [
{
"name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
"cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
"doi": "10.1016/j.neuroscience.2009.03.048"
}
]
}
]
\ No newline at end of file
const kgQueryUtil = require('./../auth/util')
const { getCommonSenseDsFilter } = require('./supplements/commonSense')
const { hasPreview } = require('./supplements/previewFile')
const path = require('path')
const fs = require('fs')
let getPublicAccessToken
......@@ -25,6 +29,185 @@ const getUserKGRequestParam = async ({ user }) => {
}
}
/**
* Needed by filter by parcellation
*/
const flattenArray = (array) => {
return array.concat(
...array.map(item => item.children && item.children instanceof Array
? flattenArray(item.children)
: [])
)
}
const readConfigFile = (filename) => new Promise((resolve, reject) => {
let filepath
if (process.env.NODE_ENV === 'production') {
filepath = path.join(__dirname, '..', 'res', filename)
} else {
filepath = path.join(__dirname, '..', '..', 'src', 'res', 'ext', filename)
}
fs.readFile(filepath, 'utf-8', (err, data) => {
if (err) reject(err)
resolve(data)
})
})
const populateSet = (flattenedRegions, set = new Set()) => {
if (!(set instanceof Set)) throw `set needs to be an instance of Set`
if (!(flattenedRegions instanceof Array)) throw `flattenedRegions needs to be an instance of Array`
for (const region of flattenedRegions) {
const { name, relatedAreas } = region
if (name) set.add(name)
if (relatedAreas && relatedAreas instanceof Array && relatedAreas.length > 0) {
for (const relatedArea of relatedAreas) {
if(typeof relatedArea === 'string') set.add(relatedArea)
else console.warn(`related area not an instance of String. skipping`, relatedArea)
}
}
}
return set
}
const initPrArray = []
let juBrainSet = new Set(),
shortBundleSet = new Set(),
longBundleSet = new Set(),
waxholm1Set = new Set(),
waxholm2Set = new Set(),
waxholm3Set = new Set(),
allen2015Set = new Set(),
allen2017Set = new Set()
initPrArray.push(
readConfigFile('MNI152.json')
.then(data => JSON.parse(data))
.then(json => {
const longBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Long Bundle').regions)
const shortBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Short Bundle').regions)
const jubrain = flattenArray(json.parcellations.find(({ name }) => 'JuBrain Cytoarchitectonic Atlas' === name).regions)
longBundleSet = populateSet(longBundle)
shortBundleSet = populateSet(shortBundle)
juBrainSet = populateSet(jubrain)
})
.catch(console.error)
)
initPrArray.push(
readConfigFile('waxholmRatV2_0.json')
.then(data => JSON.parse(data))
.then(json => {
const waxholm3 = flattenArray(json.parcellations[0].regions)
const waxholm2 = flattenArray(json.parcellations[1].regions)
const waxholm1 = flattenArray(json.parcellations[2].regions)
waxholm1Set = populateSet(waxholm1)
waxholm2Set = populateSet(waxholm2)
waxholm3Set = populateSet(waxholm3)
})
.catch(console.error)
)
initPrArray.push(
readConfigFile('allenMouse.json')
.then(data => JSON.parse(data))
.then(json => {
const flattenedAllen2017 = flattenArray(json.parcellations[0].regions)
allen2017Set = populateSet(flattenedAllen2017)
const flattenedAllen2015 = flattenArray(json.parcellations[1].regions)
allen2015Set = populateSet(flattenedAllen2015)
})
.catch(console.error)
)
const datasetRegionExistsInParcellationRegion = async (prs, atlasPrSet = new Set()) => {
if (!(atlasPrSet instanceof Set)) throw `atlasPrSet needs to be a set!`
await Promise.all(initPrArray)
return prs.some(({ name, alias }) => atlasPrSet.has(alias) || atlasPrSet.has(name))
}
const datasetBelongToParcellation = ({ parcellationName = null, dataset = {parcellationAtlas: []} } = {}) => parcellationName === null || dataset.parcellationAtlas.length === 0
? true
: (dataset.parcellationAtlas || []).some(({ name }) => name === parcellationName)
const filterDataset = async (dataset = null, { templateName, parcellationName }) => {
if (/infant/.test(dataset.name)) return false
if (templateName) {
return dataset.referenceSpaces.some(rs => rs.name === templateName)
}
if (parcellationName) {
if (dataset.parcellationRegion.length === 0) return false
let useSet
// temporary measure
// TODO ask curaion team re name of jubrain atlas
let overwriteParcellationName
switch (parcellationName) {
case 'Cytoarchitectonic Maps':
case 'JuBrain Cytoarchitectonic Atlas':
useSet = juBrainSet
overwriteParcellationName = 'Jülich Cytoarchitechtonic Brain Atlas (human)'
break;
case 'Fibre Bundle Atlas - Short Bundle':
useSet = shortBundleSet
break;
case 'Fibre Bundle Atlas - Long Bundle':
useSet = longBundleSet
break;
case 'Waxholm Space rat brain atlas v1':
useSet = waxholm1Set
break;
case 'Waxholm Space rat brain atlas v2':
useSet = waxholm2Set
break;
case 'Waxholm Space rat brain atlas v3':
useSet = waxholm3Set
break;
case 'Allen Mouse Common Coordinate Framework v3 2015':
useSet = allen2015Set
break;
case 'Allen Mouse Common Coordinate Framework v3 2017':
useSet = allen2017Set
break;
default:
useSet = new Set()
}
return datasetBelongToParcellation({ dataset, parcellationName: overwriteParcellationName || parcellationName })
&& await datasetRegionExistsInParcellationRegion(dataset.parcellationRegion, useSet)
}
return false
}
const filterDatasets = async (datasets = [], { templateName, parcellationName }) => {
// filter by commonsense first (species)
const commonSenseFilteredDatasets = datasets.filter(getCommonSenseDsFilter({ templateName, parcellationName }))
// filter by parcellation name and if region is in parcellation
const filteredDatasets = []
for (const dataset of commonSenseFilteredDatasets) {
if (await filterDataset(dataset, { templateName, parcellationName })) {
filteredDatasets.push(dataset)
}
}
// append if preview is available
const filteredDatasetsAppendingPreview = filteredDatasets.map(ds => {
return {
...ds,
preview: hasPreview({ datasetName: ds.name })
}
})
return filteredDatasetsAppendingPreview
}
const init = async () => {
if (process.env.ACCESS_TOKEN) {
if (process.env.NODE_ENV === 'production') console.error(`ACCESS_TOKEN set in production!`)
......@@ -35,23 +218,45 @@ const init = async () => {
getPublicAccessToken = getPublic
}
const retry = (fn) => {
let retryId
retryId = setTimeout(() => {
fn()
.then(() => {
console.log(`retry succeeded, clearing retryId`)
clearTimeout(retryId)
})
.catch(e => {
console.warn(`retry failed, retrying in 5sec`)
retry(fn)
})
}, 5000)
const defaultConfig = {
timeout: 5000,
retries: 3
}
const retry = async (fn, { timeout = defaultConfig.timeout, retries = defaultConfig.retries } = defaultConfig) => {
let retryNo = 0
while (retryNo < retries) {
retryNo ++
try {
const result = await fn()
return result
} catch (e) {
console.warn(`fn failed, retry after ${timeout} milliseconds`)
await (() => new Promise(rs => setTimeout(rs, timeout)))()
}
}
throw new Error(`fn failed ${retires} times. Aborting.`)
}
module.exports = {
init,
getUserKGRequestParam,
retry
retry,
filterDatasets,
datasetBelongToParcellation,
datasetRegionExistsInParcellationRegion,
_getParcellations: async () => {
await Promise.all(initPrArray)
return {
juBrainSet,
shortBundleSet,
longBundleSet,
waxholm1Set,
waxholm2Set,
waxholm3Set,
allen2015Set,
allen2017Set
}
}
}
\ No newline at end of file
const { retry } = require('./util')
const { retry, filterDatasets, datasetRegionExistsInParcellationRegion, _getParcellations } = require('./util')
const { fake } = require('sinon')
const { assert, expect } = require('chai')
const waxholmv2 = require('./testData/waxholmv2')
const allen2015 = require('./testData/allen2015')
let val = 0
describe('datasets/util.js', () => {
const prFn = () => {
val++
return val >=3 ? Promise.resolve() : Promise.reject()
}
// describe('retry', () => {
retry(() => prFn())
\ No newline at end of file
// let val = 0
// const failCall = fake()
// const succeedCall = fake()
// const prFn = () => {
// val++
// return val >=3
// ? (succeedCall(), Promise.resolve())
// : (failCall(), Promise.reject())
// }
// beforeEach(() => {
// val = 0
// succeedCall.resetHistory()
// failCall.resetHistory()
// })
// it('retry until succeed', async () => {
// await retry(prFn)
// assert(succeedCall.called)
// assert(failCall.calledTwice)
// })
// it('retry with shorter timeouts', async () => {
// await retry(prFn, { timeout: 100 })
// assert(succeedCall.called)
// assert(failCall.calledTwice)
// })
// it('when retries excceeded, retry fn throws', async () => {
// try {
// await retry(prFn, { timeout: 100, retries: 2 })
// assert(false, 'retry fn should throw if retries exceed')
// } catch (e) {
// assert(true)
// }
// })
// })
describe('datasetRegionExistsInParcellationRegion', () => {
it('should filter waxholm v2 properly', async () => {
const waxholmv2Pr = waxholmv2.map(dataset => {
return dataset.parcellationRegion
})
const { waxholm2Set } = await _getParcellations()
for (const pr of waxholmv2Pr){
const flag = await datasetRegionExistsInParcellationRegion(pr, waxholm2Set)
expect(flag).to.be.true
}
})
})
describe('filterDatasets', () => {
it('should filter waxholm v1 properly', async () => {
const filteredResult = await filterDatasets(waxholmv2, { parcellationName: 'Waxholm Space rat brain atlas v1' })
expect(filteredResult).to.have.length(0)
})
it('should filter waxholm v2 properly', async () => {
const filteredResult = await filterDatasets(waxholmv2, { parcellationName: 'Waxholm Space rat brain atlas v2' })
expect(filteredResult).to.have.length(2)
})
it('should filter allen 2015 properly', async () => {
const filteredResult = await filterDatasets(allen2015, { parcellationName: 'Allen Mouse Common Coordinate Framework v3 2015' })
expect(filteredResult).to.have.length(2)
})
})
})
......@@ -6,8 +6,9 @@
"scripts": {
"start": "node server.js",
"test": "npm run testEnv && npm run testNoEnv",
"testEnv": "node -r dotenv/config ./node_modules/.bin/mocha ./test/mocha.test.js",
"testNoEnv": "node ./node_modules/.bin/mocha ./test/mocha.test.noenv.js"
"testEnv": "node -r dotenv/config ./node_modules/.bin/mocha ./test/mocha.test.js --timeout 10000",
"testNoEnv": "node ./node_modules/.bin/mocha ./test/mocha.test.noenv.js --timeout 10000",
"mocha": "mocha"
},
"keywords": [],
"author": "",
......@@ -30,6 +31,7 @@
"chai-as-promised": "^7.1.1",
"cors": "^2.8.5",
"dotenv": "^6.2.0",
"mocha": "^6.1.4"
"mocha": "^6.1.4",
"sinon": "^8.0.2"
}
}
require('../auth/util.spec')
require('../compression/index.spec')
\ No newline at end of file
require('../datasets/query.spec')
require('../datasets/util.spec')
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment