diff --git a/deploy/datasets/query.js b/deploy/datasets/query.js index 1b4ce0d4c5d13479e6f59a05df634fa741b403d8..500bc6867925210e3509905dbeed3595f76f26d3 100644 --- a/deploy/datasets/query.js +++ b/deploy/datasets/query.js @@ -7,29 +7,11 @@ const { getPreviewFile, hasPreview } = require('./supplements/previewFile') const { constants, init: kgQueryUtilInit, getUserKGRequestParam, filterDatasets, filterDatasetsByRegion } = require('./util') const ibc = require('./importIBS') const { returnAdditionalDatasets } = require('../regionalFeatures') +const { store } = require('../lruStore') -let cachedData = null - -const CACHE_DATASET_FILENAME = process.env.CACHE_DATASET_FILENAME || path.join(__dirname, 'cachedKgDataset.json') - -fs.readFile(CACHE_DATASET_FILENAME, 'utf-8', (err, data) => { - /** - * the file may or may not be present on init - */ - if (err) { - console.warn(`read cache failed`, err) - return - } - - try { - cachedData = JSON.parse(data) - }catch (e) { - /** - * parsing saved cached json error - */ - console.error(e) - } -}) +const IAV_DS_CACHE_KEY = 'IAV_DS_CACHE_KEY' +const IAV_DS_TIMESTAMP_KEY = 'IAV_DS_TIMESTAMP_KEY' +const IAV_DS_REFRESH_TIMESTAMP_KEY = 'IAV_DS_REFRESH_TIMESTAMP_KEY' const { KG_ROOT, KG_SEARCH_VOCAB } = constants @@ -67,50 +49,46 @@ const fetchDatasetFromKg = async ({ user } = {}) => { (err, resp, body) => { if (err) return reject(err) if (resp.statusCode >= 400) return reject(resp.statusCode) - try { - const json = JSON.parse(body) - return resolve(json) - }catch (e) { - console.warn(`parsing json obj error`, body) - reject(e) - } + resolve(body) }) }) } -const cacheData = ({ results, ...rest }) => { - cachedData = results - otherQueryResult = rest - fs.writeFile(CACHE_DATASET_FILENAME, JSON.stringify(results), (err) => { - if (err) console.error('writing cached data fail') - }) - return cachedData +const refreshCache = async () => { + store.set(IAV_DS_REFRESH_TIMESTAMP_KEY, new Date().toString()) + const text = await fetchDatasetFromKg() + await store.set(IAV_DS_CACHE_KEY, text) + await store.set(IAV_DS_REFRESH_TIMESTAMP_KEY, null) + await store.set(IAV_DS_TIMESTAMP_KEY, new Date().toString()) } -let fetchingPublicDataInProgress = false -let getPublicDsPr - const getPublicDs = async () => { console.log(`fetching public ds ...`) - /** - * every request to public ds will trigger a refresh pull from master KG (throttled pending on resolved request) - */ - if (!fetchingPublicDataInProgress) { - fetchingPublicDataInProgress = true - getPublicDsPr = fetchDatasetFromKg() - .then(_ => { - console.log(`public ds fetched!`) - fetchingPublicDataInProgress = false - getPublicDsPr = null - return _ - }) - .then(cacheData) + let cachedData = await store.get(IAV_DS_CACHE_KEY) + if (!cachedData) { + await refreshCache() + cachedData = await store.get(IAV_DS_CACHE_KEY) } - if (cachedData) return Promise.resolve(cachedData) - if (getPublicDsPr) return getPublicDsPr - throw `cached Data not yet resolved, neither is get public ds defined` + const timestamp = await store.get(IAV_DS_TIMESTAMP_KEY) + const refreshTimestamp = await store.get(IAV_DS_REFRESH_TIMESTAMP_KEY) + + if ( + new Date() - new Date(timestamp) > 1e3 * 60 * 30 + ) { + if ( + !refreshTimestamp || + new Date() - new Date(refreshTimestamp) > 1e3 * 60 * 5 + ) { + refreshCache() + } + } + if (cachedData) { + const { results } = JSON.parse(cachedData) + return Promise.resolve(results) + } + throw new Error(`cacheData not defined!`) } /** @@ -118,7 +96,9 @@ const getPublicDs = async () => { * getting individual ds is too slow */ const getDs = ({ user }) => (false && user - ? fetchDatasetFromKg({ user }).then(({ results }) => results) + ? fetchDatasetFromKg({ user }) + .then(text => JSON.parse(text)) + .then(({ results }) => results) : getPublicDs() ).then(async datasets => { diff --git a/deploy_env.md b/deploy_env.md index 085fc59d056b66f3e446dab49c6ddfa26a274385..a2526621d19ebda48b735b15533f176592589f9d 100644 --- a/deploy_env.md +++ b/deploy_env.md @@ -30,7 +30,6 @@ | --- | --- | --- | --- | | `REFRESH_TOKEN` | | `ACCESS_TOKEN` | **nb** as access tokens are usually short lived, this should only be set for development purposes -| `CACHE_DATASET_FILENAME` | | `deploy/dataset/cachedKgDataset.json` | | `KG_ROOT` | | `https://kg.humanbrainproject.eu/query` | | `KG_SEARCH_VOCAB` | | `https://schema.hbp.eu/myQuery/` | | `KG_DATASET_SEARCH_QUERY_NAME` | | `interactiveViewerKgQuery-v0_3` |