From 1ab057cb64e9bc09382c6566ae46798e701aed65 Mon Sep 17 00:00:00 2001
From: Xiao Gui <xgui3783@gmail.com>
Date: Mon, 1 Jul 2019 15:51:15 +0200
Subject: [PATCH] feat; directly stream file from KG
---
deploy/datasets/index.js | 55 +++++++++---
deploy/datasets/query.js | 85 ++++++++++++++++---
deploy/package.json | 1 +
src/atlasViewer/atlasViewer.template.html | 3 +-
src/res/ext/MNI152.json | 4 +-
src/services/zipFileDownload.service.ts | 13 +++
.../referenceToast.component.ts | 9 ++
.../referenceToast.template.html | 2 +-
.../signinBanner/signinBanner.template.html | 4 +-
9 files changed, 149 insertions(+), 27 deletions(-)
diff --git a/deploy/datasets/index.js b/deploy/datasets/index.js
index 80557fa3e..e97436626 100644
--- a/deploy/datasets/index.js
+++ b/deploy/datasets/index.js
@@ -2,7 +2,7 @@ const express = require('express')
const path = require('path')
const fs = require('fs')
const datasetsRouter = express.Router()
-const { init, getDatasets, getPreview } = require('./query')
+const { init, getDatasets, getPreview, getDatasetFromId, getDatasetFileAsZip } = require('./query')
const url = require('url')
const qs = require('querystring')
@@ -10,21 +10,24 @@ const bodyParser = require('body-parser')
datasetsRouter.use(bodyParser.urlencoded({ extended: false }))
datasetsRouter.use(bodyParser.json())
-
init().catch(e => {
console.warn(`dataset init failed`, e)
})
-datasetsRouter.use((req, res, next) => {
- res.setHeader('Cache-Control', 'no-cache')
+const cacheMaxAge24Hr = (_req, res, next) => {
+ const oneDay = 24 * 60 * 60
+ res.setHeader('Cache-Control', `max-age=${oneDay}`)
next()
-})
-
+}
+const noCacheMiddleWare = (_req, res, next) => {
+ res.setHeader('Cache-Control', 'no-cache')
+ next()
+}
-datasetsRouter.use('/spatialSearch', require('./spatialRouter'))
+datasetsRouter.use('/spatialSearch', noCacheMiddleWare, require('./spatialRouter'))
-datasetsRouter.get('/templateName/:templateName', (req, res, next) => {
+datasetsRouter.get('/templateName/:templateName', noCacheMiddleWare, (req, res, next) => {
const { templateName } = req.params
const { user } = req
getDatasets({ templateName, user })
@@ -40,7 +43,7 @@ datasetsRouter.get('/templateName/:templateName', (req, res, next) => {
})
})
-datasetsRouter.get('/parcellationName/:parcellationName', (req, res, next) => {
+datasetsRouter.get('/parcellationName/:parcellationName', noCacheMiddleWare, (req, res, next) => {
const { parcellationName } = req.params
const { user } = req
getDatasets({ parcellationName, user })
@@ -56,7 +59,7 @@ datasetsRouter.get('/parcellationName/:parcellationName', (req, res, next) => {
})
})
-datasetsRouter.get('/preview/:datasetName', (req, res, next) => {
+datasetsRouter.get('/preview/:datasetName', cacheMaxAge24Hr, (req, res, next) => {
const { datasetName } = req.params
const ref = url.parse(req.headers.referer)
const { templateSelected, parcellationSelected } = qs.parse(ref.query)
@@ -97,7 +100,7 @@ fs.readdir(RECEPTOR_PATH, (err, files) => {
files.forEach(file => previewFileMap.set(`res/image/receptor/${file}`, path.join(RECEPTOR_PATH, file)))
})
-datasetsRouter.get('/previewFile', (req, res) => {
+datasetsRouter.get('/previewFile', cacheMaxAge24Hr, (req, res) => {
const { file } = req.query
const filePath = previewFileMap.get(file)
if (filePath) {
@@ -107,7 +110,37 @@ datasetsRouter.get('/previewFile', (req, res) => {
}
})
+const checkKgQuery = (req, res, next) => {
+ const { kgSchema } = req.query
+ if (kgSchema !== 'minds/core/dataset/v1.0.0') return res.status(400).send('Only kgSchema is required and the only accepted value is minds/core/dataset/v1.0.0')
+ else return next()
+}
+
+datasetsRouter.get('/kgInfo', checkKgQuery, cacheMaxAge24Hr, async (req, res) => {
+
+ const { kgId } = req.query
+ const { user } = req
+ const stream = await getDatasetFromId({ user, kgId, returnAsStream: true })
+ stream.pipe(res)
+})
+
+datasetsRouter.get('/downloadKgFiles', checkKgQuery, cacheMaxAge24Hr, async (req, res) => {
+ const { kgId } = req.query
+ const { user } = req
+ try {
+ const stream = await getDatasetFileAsZip({ user, kgId })
+ res.setHeader('Content-Type', 'application/zip')
+ stream.pipe(res)
+ } catch (e) {
+ console.log('datasets/index#downloadKgFiles', e)
+ res.status(400).send(e)
+ }
+})
+/**
+ * TODO
+ * deprecate jszip in favour of archiver
+ */
var JSZip = require("jszip");
diff --git a/deploy/datasets/query.js b/deploy/datasets/query.js
index b0df2342c..19a4cdac9 100644
--- a/deploy/datasets/query.js
+++ b/deploy/datasets/query.js
@@ -1,6 +1,8 @@
const fs = require('fs')
const request = require('request')
+const URL = require('url')
const path = require('path')
+const archiver = require('archiver')
const { commonSenseDsFilter } = require('./supplements/commonSense')
const { getPreviewFile, hasPreview } = require('./supplements/previewFile')
const { manualFilter: manualFilterDWM, manualMap: manualMapDWM } = require('./supplements/util/mapDwm')
@@ -9,7 +11,25 @@ const kgQueryUtil = require('./../auth/util')
let cachedData = null
let otherQueryResult = null
-const queryUrl = process.env.KG_DATASET_QUERY_URL || `https://kg.humanbrainproject.org/query/minds/core/dataset/v1.0.0/interactiveViewerKgQuery/instances?size=450&vocab=https%3A%2F%2Fschema.hbp.eu%2FmyQuery%2F`
+
+const KG_ROOT = process.env.KG_ROOT || `https://kg.humanbrainproject.org`
+const KG_PATH = process.env.KG_PATH || `/query/minds/core/dataset/v1.0.0/interactiveViewerKgQuery-v0_1`
+const KG_PARAM = {
+ size: process.env.KG_SEARCH_SIZE || '450',
+ vocab: process.env.KG_SEARCH_VOCAB || 'https://schema.hbp.eu/myQuery/'
+}
+
+const KG_QUERY_DATASETS_URL = new URL.URL(`${KG_ROOT}${KG_PATH}/instances`)
+for (let key in KG_PARAM) {
+ KG_QUERY_DATASETS_URL.searchParams.set(key, KG_PARAM[key])
+}
+
+const getKgQuerySingleDatasetUrl = ({ kgId }) => {
+ const _newUrl = new URL.URL(KG_QUERY_DATASETS_URL)
+ _newUrl.pathname = `${KG_PATH}/instances/${kgId}`
+ return _newUrl
+}
+
const timeout = process.env.TIMEOUT || 5000
const STORAGE_PATH = process.env.STORAGE_PATH || path.join(__dirname, 'data')
@@ -17,10 +37,10 @@ let getPublicAccessToken
const fetchDatasetFromKg = async ({ user } = {}) => {
- const { releasedOnly, option } = await getUserKGRequestInfo({ user })
+ const { releasedOnly, option } = await getUserKGRequestParam({ user })
return await new Promise((resolve, reject) => {
- request(`${queryUrl}${releasedOnly ? '&databaseScope=RELEASED' : ''}`, option, (err, resp, body) => {
+ request(`${KG_QUERY_DATASETS_URL}${releasedOnly ? '&databaseScope=RELEASED' : ''}`, option, (err, resp, body) => {
if (err)
return reject(err)
if (resp.statusCode >= 400)
@@ -238,7 +258,7 @@ function filterByqueryArg(cubeDots) {
async function getSpatialSearchOk({ user, boundingBoxInWaxhomV2VoxelSpace }) {
- const { releasedOnly, option } = await getUserKGRequestInfo({ user })
+ const { releasedOnly, option } = await getUserKGRequestParam({ user })
const spatialQuery = 'https://kg.humanbrainproject.org/query/minds/core/dataset/v1.0.0/spatialSimple/instances?size=10'
@@ -254,22 +274,30 @@ async function getSpatialSearchOk({ user, boundingBoxInWaxhomV2VoxelSpace }) {
})
}
-async function getUserKGRequestInfo({ user }) {
- const accessToken = user && user.tokenset && user.tokenset.access_token
+let publicAccessToken
+
+async function getUserKGRequestParam({ user }) {
+ /**
+ * n.b. ACCESS_TOKEN env var is usually only set during dev
+ */
+ const accessToken = (user && user.tokenset && user.tokenset.access_token) || process.env.ACCESS_TOKEN
const releasedOnly = !accessToken
- let publicAccessToken
- if (!accessToken && getPublicAccessToken) {
+ if (!accessToken && !publicAccessToken && getPublicAccessToken) {
publicAccessToken = await getPublicAccessToken()
}
- const option = accessToken || publicAccessToken || process.env.ACCESS_TOKEN
+ const option = accessToken || publicAccessToken
? {
auth: {
- 'bearer': accessToken || publicAccessToken || process.env.ACCESS_TOKEN
+ 'bearer': accessToken || publicAccessToken
}
}
: {}
- return {option, releasedOnly, token: accessToken || publicAccessToken || process.env.ACCESS_TOKEN}
+ return {
+ option,
+ releasedOnly,
+ token: accessToken || publicAccessToken
+ }
}
/**
@@ -293,4 +321,37 @@ const transformWaxholmV2NmToVoxel = (coord) => {
return coord.map((v, idx) => (v - transl[idx]) / voxelDim[idx] )
}
-
+const getDatasetFromId = async ({ user, kgId, returnAsStream = false }) => {
+ const { option, releasedOnly } = await getUserKGRequestParam({ user })
+ const _url = getKgQuerySingleDatasetUrl({ kgId })
+ if (releasedOnly) _url.searchParams.set('databaseScope', 'RELEASED')
+ if (returnAsStream) return request(_url, option)
+ else return new Promise((resolve, reject) => {
+ request(_url, option, (err, resp, body) => {
+ if (err) return reject(err)
+ if (resp.statusCode >= 400) return reject(resp.statusCode)
+ return resolve(JSON.parse(body))
+ })
+ })
+}
+
+const getDatasetFileAsZip = async ({ user, kgId } = {}) => {
+ if (!kgId) {
+ throw new Error('kgId must be defined')
+ }
+
+ const result = await getDatasetFromId({ user, kgId })
+ const { files } = result
+ const zip = archiver('zip')
+ for (let file of files) {
+ const { name, absolutePath } = file
+ zip.append(request(absolutePath), { name })
+ }
+
+ zip.finalize()
+
+ return zip
+}
+
+exports.getDatasetFromId = getDatasetFromId
+exports.getDatasetFileAsZip = getDatasetFileAsZip
\ No newline at end of file
diff --git a/deploy/package.json b/deploy/package.json
index 72d3c417e..a1d2f4121 100644
--- a/deploy/package.json
+++ b/deploy/package.json
@@ -13,6 +13,7 @@
"author": "",
"license": "ISC",
"dependencies": {
+ "archiver": "^3.0.0",
"body-parser": "^1.19.0",
"express": "^4.16.4",
"express-session": "^1.15.6",
diff --git a/src/atlasViewer/atlasViewer.template.html b/src/atlasViewer/atlasViewer.template.html
index 8c1054a71..3a44e12b0 100644
--- a/src/atlasViewer/atlasViewer.template.html
+++ b/src/atlasViewer/atlasViewer.template.html
@@ -269,6 +269,7 @@
[parcellationPublications] = "selectedParcellation['properties'] && selectedParcellation['properties']['publications']? selectedParcellation['properties']['publications']: null"
[parcellationNifti] = "selectedParcellation['properties'] && selectedParcellation['properties']['nifti']? selectedParcellation['properties']['nifti'] : null"
[templateExternalLink] ="selectedTemplate['properties'] && selectedTemplate['properties']['externalLink']? selectedTemplate['properties']['externalLink']: null"
- [parcellationExternalLink] ="selectedParcellation['properties'] && selectedParcellation['properties']['externalLink']? selectedParcellation['properties']['externalLink']: null">
+ [parcellationExternalLink] ="selectedParcellation['properties'] && selectedParcellation['properties']['externalLink']? selectedParcellation['properties']['externalLink']: null"
+ [kgId]="selectedParcellation.properties?.kgId">
</reference-toast-component>
</ng-template>
\ No newline at end of file
diff --git a/src/res/ext/MNI152.json b/src/res/ext/MNI152.json
index ba1e9e238..39a4f8d1f 100644
--- a/src/res/ext/MNI152.json
+++ b/src/res/ext/MNI152.json
@@ -14,7 +14,9 @@
"description": "not yet",
"publications": [],
"nifti": [{"file": "jubrain-max-pmap-v22c_space-mnicolin27.nii", "size": "4400000"}],
- "externalLink": "https://doi.org/10.25493/8EGG-ZAR"
+ "externalLink": "https://doi.org/10.25493/8EGG-ZAR",
+ "kgSchema": "minds/core/dataset/v1.0.0",
+ "kgId": "4ac9f0bc-560d-47e0-8916-7b24da9bb0ce"
},
"regions": [
{
diff --git a/src/services/zipFileDownload.service.ts b/src/services/zipFileDownload.service.ts
index 35edeff6f..620979e71 100644
--- a/src/services/zipFileDownload.service.ts
+++ b/src/services/zipFileDownload.service.ts
@@ -9,6 +9,19 @@ export class ZipFileDownloadService {
/**
* TODO make naming more generic
*/
+ downloadZipFromKg(kgId: string, filename: string = 'download'){
+ const _url = new URL(`${this.constantService.backendUrl}datasets/downloadKgFiles`)
+ const searchParam = _url.searchParams
+ searchParam.set('kgSchema', 'minds/core/dataset/v1.0.0')
+ searchParam.set('kgId', kgId)
+ return fetch(_url.toString())
+ .then(res => {
+ if (res.status >= 400) throw new Error(res.status.toString())
+ return res.blob()
+ })
+ .then(data => this.simpleDownload(data, filename))
+ }
+
downloadZip(publicationsText, fileName, niiFiles) {
const correctedName = fileName.replace(/[|&;$%@"<>()+,/]/g, "")
const body = {
diff --git a/src/ui/referenceToast/referenceToast.component.ts b/src/ui/referenceToast/referenceToast.component.ts
index 36cc042d0..bcb54410f 100644
--- a/src/ui/referenceToast/referenceToast.component.ts
+++ b/src/ui/referenceToast/referenceToast.component.ts
@@ -19,6 +19,8 @@ export class ReferenceToastComponent implements OnInit{
@Input() templateExternalLink? : any
@Input() parcellationExternalLink? : any
+ @Input() kgId?: string
+
downloadingProcess = false
niiFileSize = 0
@@ -32,6 +34,13 @@ export class ReferenceToastComponent implements OnInit{
}
}
+ downloadZipFromKg() {
+ this.downloadingProcess = true
+ this.zipFileDownloadService.downloadZipFromKg(this.kgId)
+ .then(() => this.downloadingProcess = false)
+ .catch(console.error)
+ }
+
downloadPublications() {
this.downloadingProcess = true
diff --git a/src/ui/referenceToast/referenceToast.template.html b/src/ui/referenceToast/referenceToast.template.html
index b35297ee1..6e953fb1a 100644
--- a/src/ui/referenceToast/referenceToast.template.html
+++ b/src/ui/referenceToast/referenceToast.template.html
@@ -27,7 +27,7 @@
<div class="align-self-end">
<a *ngIf="parcellationExternalLink" href="{{parcellationExternalLink}}" target="_blank"><button mat-raised-button color="primary" class="downloadPublications">Explore <i class="fas fa-external-link-alt"></i></button></a>
- <button mat-raised-button color="primary" class="downloadPublications" (click)="downloadPublications()" [disabled] = "downloadingProcess" >
+ <button mat-raised-button color="primary" class="downloadPublications" (click)="kgId ? downloadZipFromKg() : downloadPublications()" [disabled] = "downloadingProcess" >
Download
<span *ngIf="niiFileSize > 0">(.nii {{niiFileSize/1000000 | number:'.1-2'}} Mb) </span>
<i class="fas" [ngClass]="!downloadingProcess? 'fa-download' :'fa-spinner fa-pulse'"></i>
diff --git a/src/ui/signinBanner/signinBanner.template.html b/src/ui/signinBanner/signinBanner.template.html
index a0f0c16be..a46ccadc1 100644
--- a/src/ui/signinBanner/signinBanner.template.html
+++ b/src/ui/signinBanner/signinBanner.template.html
@@ -94,7 +94,9 @@
[parcellationDescription] = "selectedTemplate.parcellations[chosenParcellationIndex]['properties']['description']? selectedTemplate.parcellations[chosenParcellationIndex]['properties']['description'] : null"
[parcellationPublications] = "selectedTemplate.parcellations[chosenParcellationIndex]['properties']['publications']? selectedTemplate.parcellations[chosenParcellationIndex]['properties']['publications']: null"
[parcellationNifti] = "selectedTemplate.parcellations[chosenParcellationIndex]['properties']['nifti']? selectedTemplate.parcellations[chosenParcellationIndex]['properties']['nifti'] : null"
- [parcellationExternalLink] ="selectedTemplate.parcellations[chosenParcellationIndex]['properties']['externalLink']? selectedTemplate.parcellations[chosenParcellationIndex]['properties']['externalLink']: null">
+ [parcellationExternalLink] ="selectedTemplate.parcellations[chosenParcellationIndex]['properties']['externalLink']? selectedTemplate.parcellations[chosenParcellationIndex]['properties']['externalLink']: null"
+ [kgId]="selectedTemplate.parcellations[chosenParcellationIndex].properties?.kgId">
+
</reference-toast-component>
</div>
</div>
--
GitLab