Merge pull request #6 from FZJ-INM1-BDA/chore/cleanUpDataset

Chore/clean up dataset

.github/workflows/backend.yml

+name: Backend tests
+
+on: [push]
+
+env:
+  NODE_ENV: test
+
+jobs:
+  install_dep:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [10.x, 12.x]
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: cd into deploy, npm install
+      run: |
+        cd deploy
+        npm i
+
+    - name: test no env
+      run: |
+        cd deploy
+        npm run testNoEnv
+
+    - name: test with env
+      env:
+        REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
+        HBP_CLIENTID: ${{ secrets.HBP_CLIENTID }}
+        HBP_CLIENTSECRET: ${{ secrets.HBP_CLIENTSECRET }}
+      run: |
+        cd deploy
+        npm run testEnv

.github/workflows/frontend.yml

@@ -9,7 +9,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [8.x, 10.x, 12.x]
+        node-version: [10.x, 12.x]
 
     steps:
     - uses: actions/checkout@v1

deploy/auth/util.spec.js

@@ -23,7 +23,7 @@ describe('chai-as-promised.js', () => {
   })
 })
 
-describe('util.js with env', (done) => {
+describe('util.js with env', async () => {
 
   it('when client id and client secret and refresh token is set, util should not throw', async () => {
     
@@ -31,4 +31,4 @@ describe('util.js with env', (done) => {
     const { getPublicAccessToken } = await util()
     return getPublicAccessToken().should.be.fulfilled
   })
-})
\ No newline at end of file
+})

deploy/datasets/query.js

@@ -3,9 +3,8 @@ const request = require('request')
 const URL = require('url')
 const path = require('path')
 const archiver = require('archiver')
-const { getCommonSenseDsFilter } = require('./supplements/commonSense')
 const { getPreviewFile, hasPreview } = require('./supplements/previewFile')
-const { init: kgQueryUtilInit, getUserKGRequestParam } = require('./util')
+const { init: kgQueryUtilInit, getUserKGRequestParam, filterDatasets } = require('./util')
 
 let cachedData = null
 let otherQueryResult = null
@@ -44,7 +43,6 @@ const fetchDatasetFromKg = async ({ user } = {}) => {
   })
 }
 
-
 const cacheData = ({ results, ...rest }) => {
   cachedData = results
   otherQueryResult = rest
@@ -55,6 +53,7 @@ let fetchingPublicDataInProgress = false
 let getPublicDsPr
 
 const getPublicDs = async () => {
+  console.log(`fetching public ds ...`)
 
   /**
    * every request to public ds will trigger a refresh pull from master KG (throttled pending on resolved request)
@@ -63,6 +62,7 @@ const getPublicDs = async () => {
     fetchingPublicDataInProgress = true
     getPublicDsPr = fetchDatasetFromKg()
       .then(_ => {
+        console.log(`public ds fetched!`)
         fetchingPublicDataInProgress = false
         getPublicDsPr = null
         return _
@@ -80,158 +80,6 @@ const getDs = ({ user }) => user
   ? fetchDatasetFromKg({ user }).then(({ results }) => results)
   : getPublicDs()
 
-/**
- * Needed by filter by parcellation
- */
-
-const flattenArray = (array) => {
-  return array.concat(
-    ...array.map(item => item.children && item.children instanceof Array
-      ? flattenArray(item.children)
-      : [])
-  )
-}
-
-const readConfigFile = (filename) => new Promise((resolve, reject) => {
-  let filepath
-  if (process.env.NODE_ENV === 'production') {
-    filepath = path.join(__dirname, '..', 'res', filename)
-  } else {
-    filepath = path.join(__dirname, '..', '..', 'src', 'res', 'ext', filename)
-  }
-  fs.readFile(filepath, 'utf-8', (err, data) => {
-    if (err) reject(err)
-    resolve(data)
-  })
-})
-
-const populateSet = (flattenedRegions, set = new Set()) => {
-  if (!(set instanceof Set)) throw `set needs to be an instance of Set`
-  if (!(flattenedRegions instanceof Array)) throw `flattenedRegions needs to be an instance of Array`
-  for (const region of flattenedRegions) {
-    const { name, relatedAreas } = region
-    if (name) set.add(name)
-    if (relatedAreas && relatedAreas instanceof Array && relatedAreas.length > 0) {
-      for (const relatedArea of relatedAreas) {
-        if(typeof relatedArea === 'string') set.add(relatedArea)
-        else console.warn(`related area not an instance of String. skipping`, relatedArea)
-      }
-    }
-  }
-  return set
-}
-
-let juBrainSet = new Set(),
-  shortBundleSet = new Set(),
-  longBundleSet = new Set(),
-  waxholm1Set = new Set(),
-  waxholm2Set = new Set(),
-  waxholm3Set = new Set(),
-  allen2015Set = new Set(),
-  allen2017Set = new Set()
-
-readConfigFile('MNI152.json')
-  .then(data => JSON.parse(data))
-  .then(json => {
-    const longBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Long Bundle').regions)
-    const shortBundle = flattenArray(json.parcellations.find(({ name }) =>  name === 'Fibre Bundle Atlas - Short Bundle').regions)
-    const jubrain = flattenArray(json.parcellations.find(({ name }) => 'JuBrain Cytoarchitectonic Atlas' === name).regions)
-    longBundleSet = populateSet(longBundle)
-    shortBundleSet = populateSet(shortBundle)
-    juBrainSet = populateSet(jubrain)
-  })
-  .catch(console.error)
-
-readConfigFile('waxholmRatV2_0.json')
-  .then(data => JSON.parse(data))
-  .then(json => {
-    const waxholm3 = flattenArray(json.parcellations[0].regions)
-    const waxholm2 = flattenArray(json.parcellations[1].regions)
-    const waxholm1 = flattenArray(json.parcellations[2].regions)
-
-    waxholm1Set = populateSet(waxholm1)
-    waxholm2Set = populateSet(waxholm2)
-    waxholm3Set = populateSet(waxholm3)
-  })
-  .catch(console.error)
-
-readConfigFile('allenMouse.json')
-  .then(data => JSON.parse(data))
-  .then(json => {
-    const flattenedAllen2017 = flattenArray(json.parcellations[0].regions)
-    allen2017Set = populateSet(flattenedAllen2017)
-
-    const flattenedAllen2015 = flattenArray(json.parcellations[1].regions)
-    allen2015Set = populateSet(flattenedAllen2015)
-  })
-
-const filterByPRSet = (prs, atlasPrSet = new Set()) => {
-  if (!(atlasPrSet instanceof Set)) throw `atlasPrSet needs to be a set!`
-  return prs.some(({ name, alias }) => atlasPrSet.has(alias) || atlasPrSet.has(name))
-}
-
-const filterByPRName = ({ parcellationName = null, dataset = {parcellationAtlas: []} } = {}) => parcellationName === null || dataset.parcellationAtlas.length === 0
-  ? true
-  : (dataset.parcellationAtlas || []).some(({ name }) => name === parcellationName)
-
-const filter = (datasets = [], { templateName, parcellationName }) => datasets
-  .filter(getCommonSenseDsFilter({ templateName, parcellationName }))
-  .filter(ds => {
-    if (/infant/.test(ds.name))
-      return false
-    if (templateName) {
-      return ds.referenceSpaces.some(rs => rs.name === templateName)
-    }
-    if (parcellationName) {
-      if (ds.parcellationRegion.length === 0) return false
-
-      let useSet
-
-      // temporary measure
-      // TODO ask curaion team re name of jubrain atlas
-      let overwriteParcellationName
-      switch (parcellationName) {
-        case 'Cytoarchitectonic Maps':
-        case 'JuBrain Cytoarchitectonic Atlas': 
-          useSet = juBrainSet
-          overwriteParcellationName = 'Jülich Cytoarchitechtonic Brain Atlas (human)'
-          break;
-        case 'Fibre Bundle Atlas - Short Bundle':
-          useSet = shortBundleSet
-          break;
-        case 'Fibre Bundle Atlas - Long Bundle':
-          useSet = longBundleSet
-          break;
-        case 'Waxholm Space rat brain atlas v1':
-          useSet = waxholm1Set
-          break;
-        case 'Waxholm Space rat brain atlas v2':
-          useSet = waxholm2Set
-          break;
-        case 'Waxholm Space rat brain atlas v3':
-          useSet = waxholm3Set
-          break;
-        case 'Allen Mouse Common Coordinate Framework v3 2015':
-          useSet = allen2015Set
-          break;
-        case 'Allen Mouse Common Coordinate Framework v3 2017':
-          useSet = allen2017Set
-          break;
-        default:
-          useSet = new Set()
-      }
-      return filterByPRName({ dataset: ds, parcellationName: overwriteParcellationName || parcellationName }) && filterByPRSet(ds.parcellationRegion, useSet)
-    }
-
-    return false
-  })
-  .map(ds => {
-    return {
-      ...ds,
-      preview: hasPreview({ datasetName: ds.name })
-    }
-  })
-
 /**
  * on init, populate the cached data
  */
@@ -241,7 +89,7 @@ const init = async () => {
 }
 
 const getDatasets = ({ templateName, parcellationName, user }) => getDs({ user })
-  .then(json => filter(json, { templateName, parcellationName }))
+  .then(json => filterDatasets(json, { templateName, parcellationName }))
 
 const getPreview = ({ datasetName, templateSelected }) => getPreviewFile({ datasetName, templateSelected })
 

deploy/datasets/testData/allen2015.js

+module.exports = [
+  {
+    "formats": [],
+    "datasetDOI": [
+      {
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ],
+    "activity": [
+      {
+        "protocols": [
+          "Immunohistochemistry",
+          "Atlas",
+          "Brain-wide",
+          "Synaptic transmission",
+          "Neurtransmitter transport",
+          "Glutamate uptake",
+          "GLT1"
+        ],
+        "preparation": [
+          "Ex vivo"
+        ]
+      }
+    ],
+    "referenceSpaces": [],
+    "methods": [
+      "Immunohistochemistry"
+    ],
+    "custodians": [
+      "Danbolt,  Niels C."
+    ],
+    "project": [
+      "Rodent brain neurotransporter atlas: GLT1"
+    ],
+    "description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
+    "parcellationAtlas": [
+      {
+        "name": "Allen Mouse Common Coordinate Framework v3 2015",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
+        "id": "39a1384b-8413-4d27-af8d-22432225401f"
+      },
+      {
+        "name": "Waxholm Space rat brain atlas v2",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
+        "id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
+      }
+    ],
+    "licenseInfo": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "url": "https://creativecommons.org/licenses/by-sa/4.0"
+      }
+    ],
+    "embargoStatus": [
+      "Free"
+    ],
+    "license": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
+      }
+    ],
+    "parcellationRegion": [
+      {
+        "species": [],
+        "name": "Mouse Whole brain (v3 2015)",
+        "alias": "Whole brain"
+      },
+      {
+        "species": [
+          {
+            "identifier": [
+              "e9a384ea8a4edf817710b6edef5f2940",
+              "5401fdb1d638c2bc5b68241560cddac0"
+            ],
+            "name": "Rattus norvegicus",
+            "@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
+          }
+        ],
+        "name": "Whole Brain ",
+        "alias": "Whole brain"
+      }
+    ],
+    "species": [
+      "Rattus norvegicus",
+      "Mus musculus"
+    ],
+    "name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
+    "files": [],
+    "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
+    "contributors": [
+      "Danbolt, Nils C.",
+      "Bjaalie, Jan G.",
+      "Leergaard, Trygve B.",
+      "Lehre, K.P.",
+      "Real, Katia",
+      "Scott, Heather A.",
+      "Holmseth, Silvia"
+    ],
+    "id": "63bbb845ac6d2f1839f919c2ef0455bc",
+    "kgReference": [
+      "10.25493/Y147-2CE"
+    ],
+    "publications": [
+      {
+        "name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ]
+  },
+  {
+    "formats": [],
+    "datasetDOI": [
+      {
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ],
+    "activity": [
+      {
+        "protocols": [
+          "Immunohistochemistry",
+          "Atlas",
+          "Brain-wide",
+          "Synaptic transmission",
+          "Neurtransmitter transport",
+          "Glutamate uptake",
+          "GLT1"
+        ],
+        "preparation": [
+          "Ex vivo"
+        ]
+      }
+    ],
+    "referenceSpaces": [],
+    "methods": [
+      "Immunohistochemistry"
+    ],
+    "custodians": [
+      "Danbolt,  Niels C."
+    ],
+    "project": [
+      "Rodent brain neurotransporter atlas: GLT1"
+    ],
+    "description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
+    "parcellationAtlas": [
+      {
+        "name": "Allen Mouse Common Coordinate Framework v3 2015",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
+        "id": "39a1384b-8413-4d27-af8d-22432225401f"
+      },
+      {
+        "name": "Waxholm Space rat brain atlas v2",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
+        "id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
+      }
+    ],
+    "licenseInfo": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "url": "https://creativecommons.org/licenses/by-sa/4.0"
+      }
+    ],
+    "embargoStatus": [
+      "Free"
+    ],
+    "license": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
+      }
+    ],
+    "parcellationRegion": [
+      {
+        "species": [],
+        "name": "Mouse Whole brain (v3 2015)",
+        "alias": "Whole brain"
+      },
+      {
+        "species": [
+          {
+            "identifier": [
+              "e9a384ea8a4edf817710b6edef5f2940",
+              "5401fdb1d638c2bc5b68241560cddac0"
+            ],
+            "name": "Rattus norvegicus",
+            "@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
+          }
+        ],
+        "name": "Whole Brain ",
+        "alias": "Whole brain"
+      }
+    ],
+    "species": [
+      "Rattus norvegicus",
+      "Mus musculus"
+    ],
+    "name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
+    "files": [],
+    "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
+    "contributors": [
+      "Danbolt, Nils C.",
+      "Bjaalie, Jan G.",
+      "Leergaard, Trygve B.",
+      "Lehre, K.P.",
+      "Real, Katia",
+      "Scott, Heather A.",
+      "Holmseth, Silvia"
+    ],
+    "id": "63bbb845ac6d2f1839f919c2ef0455bc",
+    "kgReference": [
+      "10.25493/Y147-2CE"
+    ],
+    "publications": [
+      {
+        "name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ]
+  }
+]
\ No newline at end of file

deploy/datasets/testData/waxholmv2.js

+module.exports = [
+  {
+    "formats": [],
+    "datasetDOI": [
+      {
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ],
+    "activity": [
+      {
+        "protocols": [
+          "Immunohistochemistry",
+          "Atlas",
+          "Brain-wide",
+          "Synaptic transmission",
+          "Neurtransmitter transport",
+          "Glutamate uptake",
+          "GLT1"
+        ],
+        "preparation": [
+          "Ex vivo"
+        ]
+      }
+    ],
+    "referenceSpaces": [],
+    "methods": [
+      "Immunohistochemistry"
+    ],
+    "custodians": [
+      "Danbolt,  Niels C."
+    ],
+    "project": [
+      "Rodent brain neurotransporter atlas: GLT1"
+    ],
+    "description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
+    "parcellationAtlas": [
+      {
+        "name": "Allen Mouse Common Coordinate Framework v3 2015",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
+        "id": "39a1384b-8413-4d27-af8d-22432225401f"
+      },
+      {
+        "name": "Waxholm Space rat brain atlas v2",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
+        "id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
+      }
+    ],
+    "licenseInfo": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "url": "https://creativecommons.org/licenses/by-sa/4.0"
+      }
+    ],
+    "embargoStatus": [
+      "Free"
+    ],
+    "license": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
+      }
+    ],
+    "parcellationRegion": [
+      {
+        "species": [],
+        "name": "Mouse Whole brain (v3 2015)",
+        "alias": "Whole brain"
+      },
+      {
+        "species": [
+          {
+            "identifier": [
+              "e9a384ea8a4edf817710b6edef5f2940",
+              "5401fdb1d638c2bc5b68241560cddac0"
+            ],
+            "name": "Rattus norvegicus",
+            "@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
+          }
+        ],
+        "name": "Whole Brain ",
+        "alias": "Whole brain"
+      }
+    ],
+    "species": [
+      "Rattus norvegicus",
+      "Mus musculus"
+    ],
+    "name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
+    "files": [],
+    "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
+    "contributors": [
+      "Danbolt, Nils C.",
+      "Bjaalie, Jan G.",
+      "Leergaard, Trygve B.",
+      "Lehre, K.P.",
+      "Real, Katia",
+      "Scott, Heather A.",
+      "Holmseth, Silvia"
+    ],
+    "id": "63bbb845ac6d2f1839f919c2ef0455bc",
+    "kgReference": [
+      "10.25493/Y147-2CE"
+    ],
+    "publications": [
+      {
+        "name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ]
+  },
+  {
+    "formats": [],
+    "datasetDOI": [
+      {
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ],
+    "activity": [
+      {
+        "protocols": [
+          "Immunohistochemistry",
+          "Atlas",
+          "Brain-wide",
+          "Synaptic transmission",
+          "Neurtransmitter transport",
+          "Glutamate uptake",
+          "GLT1"
+        ],
+        "preparation": [
+          "Ex vivo"
+        ]
+      }
+    ],
+    "referenceSpaces": [],
+    "methods": [
+      "Immunohistochemistry"
+    ],
+    "custodians": [
+      "Danbolt,  Niels C."
+    ],
+    "project": [
+      "Rodent brain neurotransporter atlas: GLT1"
+    ],
+    "description": "Glutamate is the major excitatory transmitter in the central nervous system (Danbolt, Prog. Neurobiol. 65:1-105, 2001). It is inactivated by cellular uptake, mostly catalyzed by the glutamate transporters GLT1 (slc1a2, excitatory amino acid transporter [EAAT2]) subtype expressed at high levels in brain astrocytes and at lower levels in neurons. Three C-terminal variants of EAAT2 exist: GLT1a (Pines et al., Nature 360:464-467, 1992), GLT1b (Utsunomiya-Tate et al., FEBS Lett 416:312-326,1997), and GLT1c (Rauen et al., Neurochem. Int. 45:1095-1106, 2004). This dataset is brain-wide collection of microscopic images showing the brain-wide distribution of GLT1 in the mouse and rat brain, visualized by immunohistochemistry using antibodies against GLT1a and GLT1b. To facilitate identification of anatomical location adjacent section were stained to reveal cyto- and myeloarchitecture.",
+    "parcellationAtlas": [
+      {
+        "name": "Allen Mouse Common Coordinate Framework v3 2015",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/39a1384b-8413-4d27-af8d-22432225401f",
+        "id": "39a1384b-8413-4d27-af8d-22432225401f"
+      },
+      {
+        "name": "Waxholm Space rat brain atlas v2",
+        "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/parcellationatlas/v1.0.0/2449a7f0-6dd0-4b5a-8f1e-aec0db03679d",
+        "id": "2449a7f0-6dd0-4b5a-8f1e-aec0db03679d"
+      }
+    ],
+    "licenseInfo": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "url": "https://creativecommons.org/licenses/by-sa/4.0"
+      }
+    ],
+    "embargoStatus": [
+      "Free"
+    ],
+    "license": [
+      {
+        "name": "Creative Commons Attribution-ShareAlike 4.0 International",
+        "relativeUrl": "minds/core/licensetype/v1.0.0/78a3bfb2-f4b9-40f0-869c-34b5e48a45bd"
+      }
+    ],
+    "parcellationRegion": [
+      {
+        "species": [],
+        "name": "Mouse Whole brain (v3 2015)",
+        "alias": "Whole brain"
+      },
+      {
+        "species": [
+          {
+            "identifier": [
+              "e9a384ea8a4edf817710b6edef5f2940",
+              "5401fdb1d638c2bc5b68241560cddac0"
+            ],
+            "name": "Rattus norvegicus",
+            "@id": "https://nexus.humanbrainproject.org/v0/data/minds/core/species/v1.0.0/f3490d7f-8f7f-4b40-b238-963dcac84412"
+          }
+        ],
+        "name": "Whole Brain ",
+        "alias": "Whole brain"
+      }
+    ],
+    "species": [
+      "Rattus norvegicus",
+      "Mus musculus"
+    ],
+    "name": "Brain-wide distribution of glutamate type 1 transporter protein (GLT1)",
+    "files": [],
+    "fullId": "https://nexus.humanbrainproject.org/v0/data/minds/core/dataset/v1.0.0/f7a7d460-8724-4cd1-a06e-457eb8954fbd",
+    "contributors": [
+      "Danbolt, Nils C.",
+      "Bjaalie, Jan G.",
+      "Leergaard, Trygve B.",
+      "Lehre, K.P.",
+      "Real, Katia",
+      "Scott, Heather A.",
+      "Holmseth, Silvia"
+    ],
+    "id": "63bbb845ac6d2f1839f919c2ef0455bc",
+    "kgReference": [
+      "10.25493/Y147-2CE"
+    ],
+    "publications": [
+      {
+        "name": "The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation",
+        "cite": "Holmseth, S., Scott, H. A., Real, K., Lehre, K. P., Leergaard, T. B., Bjaalie, J. G., & Danbolt, N. C. (2009). The concentrations and distributions of three C-terminal variants of the GLT1 (EAAT2; slc1a2) glutamate transporter protein in rat brain tissue suggest differential regulation. Neuroscience, 162(4), 1055–1071. ",
+        "doi": "10.1016/j.neuroscience.2009.03.048"
+      }
+    ]
+  }
+]
\ No newline at end of file

deploy/datasets/util.js

 const kgQueryUtil = require('./../auth/util')
+const { getCommonSenseDsFilter } = require('./supplements/commonSense')
+const { hasPreview } = require('./supplements/previewFile')
+const path = require('path')
+const fs = require('fs')
 
 let getPublicAccessToken
 
@@ -25,6 +29,185 @@ const getUserKGRequestParam = async ({ user }) => {
   }
 }
 
+/**
+ * Needed by filter by parcellation
+ */
+
+const flattenArray = (array) => {
+  return array.concat(
+    ...array.map(item => item.children && item.children instanceof Array
+      ? flattenArray(item.children)
+      : [])
+  )
+}
+
+const readConfigFile = (filename) => new Promise((resolve, reject) => {
+  let filepath
+  if (process.env.NODE_ENV === 'production') {
+    filepath = path.join(__dirname, '..', 'res', filename)
+  } else {
+    filepath = path.join(__dirname, '..', '..', 'src', 'res', 'ext', filename)
+  }
+  fs.readFile(filepath, 'utf-8', (err, data) => {
+    if (err) reject(err)
+    resolve(data)
+  })
+})
+
+const populateSet = (flattenedRegions, set = new Set()) => {
+  if (!(set instanceof Set)) throw `set needs to be an instance of Set`
+  if (!(flattenedRegions instanceof Array)) throw `flattenedRegions needs to be an instance of Array`
+  for (const region of flattenedRegions) {
+    const { name, relatedAreas } = region
+    if (name) set.add(name)
+    if (relatedAreas && relatedAreas instanceof Array && relatedAreas.length > 0) {
+      for (const relatedArea of relatedAreas) {
+        if(typeof relatedArea === 'string') set.add(relatedArea)
+        else console.warn(`related area not an instance of String. skipping`, relatedArea)
+      }
+    }
+  }
+  return set
+}
+
+const initPrArray = []
+
+let juBrainSet = new Set(),
+  shortBundleSet = new Set(),
+  longBundleSet = new Set(),
+  waxholm1Set = new Set(),
+  waxholm2Set = new Set(),
+  waxholm3Set = new Set(),
+  allen2015Set = new Set(),
+  allen2017Set = new Set()
+
+initPrArray.push(
+  readConfigFile('MNI152.json')
+    .then(data => JSON.parse(data))
+    .then(json => {
+      const longBundle = flattenArray(json.parcellations.find(({ name }) => name === 'Fibre Bundle Atlas - Long Bundle').regions)
+      const shortBundle = flattenArray(json.parcellations.find(({ name }) =>  name === 'Fibre Bundle Atlas - Short Bundle').regions)
+      const jubrain = flattenArray(json.parcellations.find(({ name }) => 'JuBrain Cytoarchitectonic Atlas' === name).regions)
+      longBundleSet = populateSet(longBundle)
+      shortBundleSet = populateSet(shortBundle)
+      juBrainSet = populateSet(jubrain)
+    })
+    .catch(console.error)
+)
+
+initPrArray.push(
+  readConfigFile('waxholmRatV2_0.json')
+    .then(data => JSON.parse(data))
+    .then(json => {
+      const waxholm3 = flattenArray(json.parcellations[0].regions)
+      const waxholm2 = flattenArray(json.parcellations[1].regions)
+      const waxholm1 = flattenArray(json.parcellations[2].regions)
+
+      waxholm1Set = populateSet(waxholm1)
+      waxholm2Set = populateSet(waxholm2)
+      waxholm3Set = populateSet(waxholm3)
+    })
+    .catch(console.error)
+)
+
+initPrArray.push(
+  readConfigFile('allenMouse.json')
+    .then(data => JSON.parse(data))
+    .then(json => {
+      const flattenedAllen2017 = flattenArray(json.parcellations[0].regions)
+      allen2017Set = populateSet(flattenedAllen2017)
+
+      const flattenedAllen2015 = flattenArray(json.parcellations[1].regions)
+      allen2015Set = populateSet(flattenedAllen2015)
+    })
+    .catch(console.error)
+)
+
+const datasetRegionExistsInParcellationRegion = async (prs, atlasPrSet = new Set()) => {
+  if (!(atlasPrSet instanceof Set)) throw `atlasPrSet needs to be a set!`
+  await Promise.all(initPrArray)
+  return prs.some(({ name, alias }) => atlasPrSet.has(alias) || atlasPrSet.has(name))
+}
+
+const datasetBelongToParcellation = ({ parcellationName = null, dataset = {parcellationAtlas: []} } = {}) => parcellationName === null || dataset.parcellationAtlas.length === 0
+  ? true
+  : (dataset.parcellationAtlas || []).some(({ name }) => name === parcellationName)
+
+const filterDataset = async (dataset = null, { templateName, parcellationName }) => {
+
+  if (/infant/.test(dataset.name)) return false
+  if (templateName) {
+    return dataset.referenceSpaces.some(rs => rs.name === templateName)
+  }
+  if (parcellationName) {
+    if (dataset.parcellationRegion.length === 0) return false
+
+    let useSet
+
+    // temporary measure
+    // TODO ask curaion team re name of jubrain atlas
+    let overwriteParcellationName
+    switch (parcellationName) {
+      case 'Cytoarchitectonic Maps':
+      case 'JuBrain Cytoarchitectonic Atlas': 
+        useSet = juBrainSet
+        overwriteParcellationName = 'Jülich Cytoarchitechtonic Brain Atlas (human)'
+        break;
+      case 'Fibre Bundle Atlas - Short Bundle':
+        useSet = shortBundleSet
+        break;
+      case 'Fibre Bundle Atlas - Long Bundle':
+        useSet = longBundleSet
+        break;
+      case 'Waxholm Space rat brain atlas v1':
+        useSet = waxholm1Set
+        break;
+      case 'Waxholm Space rat brain atlas v2':
+        useSet = waxholm2Set
+        break;
+      case 'Waxholm Space rat brain atlas v3':
+        useSet = waxholm3Set
+        break;
+      case 'Allen Mouse Common Coordinate Framework v3 2015':
+        useSet = allen2015Set
+        break;
+      case 'Allen Mouse Common Coordinate Framework v3 2017':
+        useSet = allen2017Set
+        break;
+      default:
+        useSet = new Set()
+    }
+    return datasetBelongToParcellation({ dataset, parcellationName: overwriteParcellationName || parcellationName })
+      && await datasetRegionExistsInParcellationRegion(dataset.parcellationRegion, useSet)
+  }
+
+  return false
+}
+
+const filterDatasets = async (datasets = [], { templateName, parcellationName }) => {
+
+  // filter by commonsense first (species)
+  const commonSenseFilteredDatasets = datasets.filter(getCommonSenseDsFilter({ templateName, parcellationName }))
+  
+  // filter by parcellation name and if region is in parcellation
+  const filteredDatasets = []
+  for (const dataset of commonSenseFilteredDatasets) {
+    if (await filterDataset(dataset, { templateName, parcellationName })) {
+      filteredDatasets.push(dataset)
+    }
+  }
+
+  // append if preview is available
+  const filteredDatasetsAppendingPreview = filteredDatasets.map(ds => {
+    return {
+      ...ds,
+      preview: hasPreview({ datasetName: ds.name })
+    }
+  })
+
+  return filteredDatasetsAppendingPreview
+}
+
 const init = async () => {
   if (process.env.ACCESS_TOKEN) {
     if (process.env.NODE_ENV === 'production') console.error(`ACCESS_TOKEN set in production!`)
@@ -35,23 +218,45 @@ const init = async () => {
   getPublicAccessToken = getPublic
 }
 
-const retry = (fn) => {
-  let retryId
-  retryId = setTimeout(() => {
-    fn()
-      .then(() => {
-        console.log(`retry succeeded, clearing retryId`)
-        clearTimeout(retryId)
-      })
-      .catch(e => {
-        console.warn(`retry failed, retrying in 5sec`)
-        retry(fn)
-      })
-  }, 5000)
+const defaultConfig = {
+  timeout: 5000,
+  retries: 3
+}
+
+const retry = async (fn, { timeout = defaultConfig.timeout, retries = defaultConfig.retries } = defaultConfig) => {
+  let retryNo = 0
+  while (retryNo < retries) {
+    retryNo ++
+    try {
+      const result = await fn()
+      return result
+    } catch (e) {
+      console.warn(`fn failed, retry after ${timeout} milliseconds`)
+      await (() => new Promise(rs => setTimeout(rs, timeout)))()
+    }
+  }
+
+  throw new Error(`fn failed ${retires} times. Aborting.`)
 }
 
 module.exports = {
   init,
   getUserKGRequestParam,
-  retry
+  retry,
+  filterDatasets,
+  datasetBelongToParcellation,
+  datasetRegionExistsInParcellationRegion,
+  _getParcellations: async () => {
+    await Promise.all(initPrArray)
+    return {
+      juBrainSet,
+      shortBundleSet,
+      longBundleSet,
+      waxholm1Set,
+      waxholm2Set,
+      waxholm3Set,
+      allen2015Set,
+      allen2017Set
+    }
+  }
 }
\ No newline at end of file

deploy/datasets/util.spec.js

-const { retry } = require('./util')
+const { retry, filterDatasets, datasetRegionExistsInParcellationRegion, _getParcellations } = require('./util')
+const { fake } = require('sinon')
+const { assert, expect } = require('chai')
+const waxholmv2 = require('./testData/waxholmv2')
+const allen2015 = require('./testData/allen2015')
 
-let val = 0
+describe('datasets/util.js', () => {
 
-const prFn = () => {
-  val++
-  return val >=3 ? Promise.resolve() : Promise.reject()
-} 
+  // describe('retry', () => {
 
-retry(() => prFn())
\ No newline at end of file
+  //   let val = 0
+  
+  //   const failCall = fake()
+  //   const succeedCall = fake()
+  
+  //   const prFn = () => {
+  //     val++
+  //     return val >=3
+  //       ? (succeedCall(), Promise.resolve())
+  //       : (failCall(), Promise.reject())
+  //   }
+  
+  //   beforeEach(() => {
+  //     val = 0
+  //     succeedCall.resetHistory()
+  //     failCall.resetHistory()
+  //   })
+  
+  //   it('retry until succeed', async () => {
+  //     await retry(prFn)
+  //     assert(succeedCall.called)
+  //     assert(failCall.calledTwice)
+  //   })
+  
+  //   it('retry with shorter timeouts', async () => {
+  //     await retry(prFn, { timeout: 100 })
+  //     assert(succeedCall.called)
+  //     assert(failCall.calledTwice)
+  //   })
+  
+  //   it('when retries excceeded, retry fn throws', async () => {
+  //     try {
+  //       await retry(prFn, { timeout: 100, retries: 2 })
+  //       assert(false, 'retry fn should throw if retries exceed')
+  //     } catch (e) {
+  //       assert(true)
+  //     }
+  //   })
+  // })
+  
+  describe('datasetRegionExistsInParcellationRegion', () => {
+    it('should filter waxholm v2 properly', async () => {
+
+      const waxholmv2Pr = waxholmv2.map(dataset => {
+        return dataset.parcellationRegion
+      })
+
+      const { waxholm2Set } = await _getParcellations()
+      for (const pr of waxholmv2Pr){
+
+      const flag = await datasetRegionExistsInParcellationRegion(pr, waxholm2Set)
+        expect(flag).to.be.true
+      }
+    })
+  })
+  
+  describe('filterDatasets', () => {
+    it('should filter waxholm v1 properly', async () => {
+      const filteredResult = await filterDatasets(waxholmv2, { parcellationName: 'Waxholm Space rat brain atlas v1' })
+      expect(filteredResult).to.have.length(0)
+    })
+
+    it('should filter waxholm v2 properly', async () => {
+      const filteredResult = await filterDatasets(waxholmv2, { parcellationName: 'Waxholm Space rat brain atlas v2' })
+      expect(filteredResult).to.have.length(2)
+    })
+
+    it('should filter allen 2015 properly', async () => {
+
+      const filteredResult = await filterDatasets(allen2015, { parcellationName: 'Allen Mouse Common Coordinate Framework v3 2015' })
+      expect(filteredResult).to.have.length(2)
+    })
+  })
+  
+})

deploy/package.json

@@ -6,8 +6,9 @@
   "scripts": {
     "start": "node server.js",
     "test": "npm run testEnv && npm run testNoEnv",
-    "testEnv": "node -r dotenv/config ./node_modules/.bin/mocha ./test/mocha.test.js",
-    "testNoEnv": "node ./node_modules/.bin/mocha ./test/mocha.test.noenv.js"
+    "testEnv": "node -r dotenv/config ./node_modules/.bin/mocha ./test/mocha.test.js --timeout 10000",
+    "testNoEnv": "node ./node_modules/.bin/mocha ./test/mocha.test.noenv.js --timeout 10000",
+    "mocha": "mocha"
   },
   "keywords": [],
   "author": "",
@@ -30,6 +31,7 @@
     "chai-as-promised": "^7.1.1",
     "cors": "^2.8.5",
     "dotenv": "^6.2.0",
-    "mocha": "^6.1.4"
+    "mocha": "^6.1.4",
+    "sinon": "^8.0.2"
   }
 }

deploy/test/mocha.test.js

 require('../auth/util.spec')
-require('../compression/index.spec')
\ No newline at end of file
+require('../datasets/query.spec')
+require('../datasets/util.spec')