feat: configurable image server location, configurable pmap + caching of kg results

deploy/app.js

@@ -84,8 +84,9 @@ app.use(require('./devBanner'))
  * only use compression for production
  * this allows locally built aot to be served without errors
  */
-
-const { compressionMiddleware } = require('nomiseco')
+const LOCAL_CDN_FLAG = !!process.env.PRECOMPUTED_SERVER
+const { compressionMiddleware, setAlwaysOff } = require('nomiseco')
+if (LOCAL_CDN_FLAG) setAlwaysOff(true)
 
 app.use(compressionMiddleware, express.static(PUBLIC_PATH))
 

deploy/datasets/query.js

@@ -8,7 +8,25 @@ const { getPreviewFile, hasPreview } = require('./supplements/previewFile')
 const { init: kgQueryUtilInit, getUserKGRequestParam } = require('./util')
 
 let cachedData = null
-let otherQueryResult = null
+
+const STORAGE_PATH = process.env.STORAGE_PATH || path.join(__dirname, 'data')
+const CACHE_DATASET_FILENAME = process.env.CACHE_DATASET_FILENAME || 'cachedKgDataset.json'
+
+fs.readFile(path.join(STORAGE_PATH, CACHE_DATASET_FILENAME), 'utf-8', (err, data) => {
+  /**
+   * the file may or may not be present on init
+   */
+  if (err) return
+
+  try {
+    cachedData = JSON.parse(data)
+  }catch (e) {
+    /**
+     * parsing saved cached json error
+     */
+    console.error(e)
+  }
+})
 
 const KG_ROOT = process.env.KG_ROOT || `https://kg.humanbrainproject.eu`
 const KG_PATH = process.env.KG_PATH || `/query/minds/core/dataset/v1.0.0/interactiveViewerKgQuery-v0_3`
@@ -48,6 +66,9 @@ const fetchDatasetFromKg = async ({ user } = {}) => {
 const cacheData = ({ results, ...rest }) => {
   cachedData = results
   otherQueryResult = rest
+  fs.writeFile(path.join(STORAGE_PATH, CACHE_DATASET_FILENAME), JSON.stringify(results), (err) => {
+    if (err) console.error('writing cached data fail')
+  })
   return cachedData
 }
 

deploy/datasets/supplements/previewFile.js

 const fs = require('fs')
 const path = require('path')
+const { reconfigureFlag, reconfigureUrl } = require('../../util/reconfigPrecomputedServer')
 
 const DISABLE_RECEPTOR_PREVIEW = process.env.DISABLE_RECEPTOR_PREVIEW
 const DISABLE_JUBRAIN_PMAP = process.env.DISABLE_JUBRAIN_PMAP
@@ -38,6 +39,19 @@ Promise.all([
     console.error('preview file error', e)
   })
 
+const processFile = ({ url, ...rest }) => {
+  const processedUrl = (url && !/^http/.test(url)
+    ? `${HOSTNAME}/${url}`
+    : reconfigureFlag
+      ? reconfigureUrl(url)
+      : url)
+    || null
+  return {
+    ...rest,
+    url: processedUrl
+  }
+}
+  
 exports.getPreviewFile = ({ datasetName, templateSelected }) => previewMap.get(datasetName)
   ? Promise.resolve(
       previewMap.get(datasetName)
@@ -46,16 +60,7 @@ exports.getPreviewFile = ({ datasetName, templateSelected }) => previewMap.get(d
           if (!templateSelected) return true
           return templateSpace === templateSelected
         })
-        .map(file => {
-          return {
-            ...file,
-            ...(file.url && !/^http/.test(file.url)
-              ? {
-                url: `${HOSTNAME}${HOST_PATHNAME}/${file.url}`
-              }
-              : {})
-          }
-        })
+        .map(processFile)
     )
   : Promise.reject(`Preview file cannot be found!`)
 

deploy/nehubaConfig/query.js

 const fs = require('fs')
 const path = require('path')
 const { BROTLI, GZIP } = require('nomiseco')
+const { reconfigureUrl } = require('../util/reconfigPrecomputedServer')
+const through2 = require('through2')
 
 const getFileAsPromise = filepath => new Promise((resolve, reject) => {
   fs.readFile(filepath, 'utf-8', (err, data) => {
     if (err) return reject(err)
-    resolve(data)
+
+    resolve(
+      PRECOMPUTED_SERVER
+        ? reconfigureUrl(data)
+        : data
+    )
   })
 })
 
@@ -26,6 +33,11 @@ exports.getTemplateNehubaConfig = ({configId, acceptedEncoding, returnAsStream})
     else return getFileAsPromise(`${filepath}.gz`)
   }
 
-  if (returnAsStream) return fs.createReadStream(filepath)
+  let data = ''
+  if (returnAsStream) return fs.createReadStream(filepath).pipe(
+    through2.obj(function(file, enc, cb){
+      cb(null, reconfigureUrl(file.toString()))
+    })
+  )
   else return getFileAsPromise(filepath)
-} 
\ No newline at end of file
+} 

deploy/package.json

@@ -23,7 +23,8 @@
     "nomiseco": "0.0.2",
     "openid-client": "^2.4.5",
     "passport": "^0.4.0",
-    "request": "^2.88.0"
+    "request": "^2.88.0",
+    "through2": "^3.0.1"
   },
   "devDependencies": {
     "chai": "^4.2.0",

deploy/util/reconfigPrecomputedServer.js

+/**
+ * n.b. trailing slash is required
+ * e.g. http://localhost:10080/
+ */
+const PRECOMPUTED_SERVER = process.env.PRECOMPUTED_SERVER
+
+const reconfigureFlag = !!PRECOMPUTED_SERVER
+
+exports.reconfigureFlag = reconfigureFlag
+
+exports.reconfigureUrl = (str) => {
+  if (!reconfigureFlag) return str
+  return str.replace(/precomputed:\/\/https?:\/\/.*?\//g, `precomputed://${PRECOMPUTED_SERVER}`)
+}
\ No newline at end of file