From c65adbb1b45ab35662efaf3edb9712f90a2f9b52 Mon Sep 17 00:00:00 2001 From: Rohan Juneja Date: Mon, 5 Feb 2024 18:26:25 -0800 Subject: [PATCH] use redis for smartapi specs --- package.json | 3 +- src/index.ts | 4 +-- src/load/all_specs_sync_loader.ts | 25 ++++++++++++---- src/load/sync_loader_factory.ts | 6 ++-- .../sync_builder_factory.ts | 6 ++-- .../sync_operations_builder.ts | 4 +-- .../sync_operations_builder_with_reasoner.ts | 29 ++++++++++++++----- 7 files changed, 52 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index 1fe7a93..66b5806 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,8 @@ "debug": "^4.3.4", "husky": "^8.0.3", "load-json-file": "^7.0.1", - "lodash": "^4.17.21" + "lodash": "^4.17.21", + "@biothings-explorer/utils": "workspace:../utils" }, "husky": { "hooks": { diff --git a/src/index.ts b/src/index.ts index b0a6198..8201edf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -57,8 +57,8 @@ export default class MetaKG { * Construct API Meta Knowledge Graph based on SmartAPI Specifications. * @param {string} tag - the SmartAPI tag to be filtered on */ - constructMetaKGSync(includeReasoner = false, options: BuilderOptions = {}): SmartAPIKGOperationObject[] { - this._ops = syncBuilderFactory(options, includeReasoner, this._file_path, this._predicates_path); + async constructMetaKGSync(includeReasoner = false, options: BuilderOptions = {}): Promise { + this._ops = await syncBuilderFactory(options, includeReasoner, this._file_path, this._predicates_path); return this.ops; } diff --git a/src/load/all_specs_sync_loader.ts b/src/load/all_specs_sync_loader.ts index 6e59ba5..f7b2cfe 100644 --- a/src/load/all_specs_sync_loader.ts +++ b/src/load/all_specs_sync_loader.ts @@ -4,6 +4,7 @@ import { SmartAPISpec } from "../parser/types"; import fs from "fs"; import Debug from "debug"; const debug = Debug("bte:smartapi-kg:AllSpecsSyncLoader"); +const { redisClient } = require("@biothings-explorer/utils"); export default class AllSpecsSyncLoader extends BaseLoader { private _file_path: string; @@ -11,10 +12,22 @@ export default class AllSpecsSyncLoader extends BaseLoader { super(); this._file_path = path; } - protected fetch(): SmartAPIQueryResult { - debug(`Fetching from file path: ${this._file_path}`); - const file = fs.readFileSync(this._file_path, "utf-8"); - const data = JSON.parse(file) as SmartAPIQueryResult | SmartAPISpec; + protected async fetch(): Promise { + debug(`Fetching from file path: ${this._file_path} ${redisClient.clientEnabled}`); + let data: SmartAPIQueryResult | SmartAPISpec; + + if (redisClient.clientEnabled) { + const redisData = await redisClient.client.getTimeout(`bte:smartapi:smartapi`).catch(console.log) + if (redisData) { + data = (JSON.parse(redisData))?.smartapi as SmartAPIQueryResult | SmartAPISpec; + } + } + + if (!data) { + const file = fs.readFileSync(this._file_path, "utf-8"); + data = JSON.parse(file) as SmartAPIQueryResult | SmartAPISpec; + } + let result; if (!("hits" in data)) { result = { @@ -31,8 +44,8 @@ export default class AllSpecsSyncLoader extends BaseLoader { return input.hits; } - load(): SmartAPISpec[] { - const specs = this.fetch(); + async load(): Promise { + const specs = await this.fetch(); return this.parse(specs); } } diff --git a/src/load/sync_loader_factory.ts b/src/load/sync_loader_factory.ts index 3bc726c..9bbbb3b 100644 --- a/src/load/sync_loader_factory.ts +++ b/src/load/sync_loader_factory.ts @@ -9,14 +9,14 @@ import { apiListObject } from "../types"; import Debug from "debug"; const debug = Debug("bte:smartapi-kg:SyncLoader"); -export const syncLoaderFactory = ( +export const syncLoaderFactory = async ( smartAPIID: string = undefined, teamName: string = undefined, tag: string = undefined, component: string = undefined, apiList: apiListObject = undefined, path: string, -): SmartAPISpec[] => { +): Promise => { let loader; if (!(typeof smartAPIID === "undefined")) { loader = new SingleSpecSyncLoader(smartAPIID, path, apiList); @@ -37,5 +37,5 @@ export const syncLoaderFactory = ( loader = new AllSpecsSyncLoader(path); debug("Using all specs sync loader now."); } - return loader.load(); + return await loader.load(); }; diff --git a/src/operations_builder/sync_builder_factory.ts b/src/operations_builder/sync_builder_factory.ts index b58cca3..37d4f2f 100644 --- a/src/operations_builder/sync_builder_factory.ts +++ b/src/operations_builder/sync_builder_factory.ts @@ -3,19 +3,19 @@ import SyncOperationsBuilderWithReasoner from "./sync_operations_builder_with_re import { BuilderOptions } from "../types"; import { SmartAPIKGOperationObject, SmartAPISpec } from "../parser/types"; -export function syncBuilderFactory( +export async function syncBuilderFactory( options: BuilderOptions, includeReasoner: boolean, smartapi_path: string, predicates_path: string, -): SmartAPIKGOperationObject[] { +): Promise { let builder; if (includeReasoner === true) { builder = new SyncOperationsBuilderWithReasoner(options, smartapi_path, predicates_path); } else { builder = new SyncOperationsBuilder(options, smartapi_path); } - const ops = builder.build(); + const ops = await builder.build(); const primaryKnowledgeAPIs = new Set(); options.apiList?.include.forEach(api => { diff --git a/src/operations_builder/sync_operations_builder.ts b/src/operations_builder/sync_operations_builder.ts index 3a5e165..8e07b01 100644 --- a/src/operations_builder/sync_operations_builder.ts +++ b/src/operations_builder/sync_operations_builder.ts @@ -10,8 +10,8 @@ export default class SyncOperationsBuilder extends BaseOperationsBuilder { this._file_path = path; } - build() { - const specs = syncLoaderFactory( + async build() { + const specs = await syncLoaderFactory( this._options.smartAPIID, this._options.teamName, this._options.tag, diff --git a/src/operations_builder/sync_operations_builder_with_reasoner.ts b/src/operations_builder/sync_operations_builder_with_reasoner.ts index 1e84329..8c6bddd 100644 --- a/src/operations_builder/sync_operations_builder_with_reasoner.ts +++ b/src/operations_builder/sync_operations_builder_with_reasoner.ts @@ -6,6 +6,7 @@ import { SmartAPIKGOperationObject } from "../parser/types"; import { PredicatesMetadata } from "../types"; import Debug from "debug"; const debug = Debug("bte:smartapi-kg:SyncOperationsBuilderWithReasoner"); +const { redisClient } = require("@biothings-explorer/utils"); import { SmartAPISpec } from "../parser/types"; declare global { @@ -136,14 +137,26 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui return ops; } - private fetch(): PredicatesMetadata[] { - const file = fs.readFileSync(this._predicates_file_path, "utf-8"); - const data = JSON.parse(file) as PredicatesMetadata[]; + private async fetch(): Promise { + let data: PredicatesMetadata[]; + + if (redisClient.clientEnabled) { + const redisData = await redisClient.client.getTimeout(`bte:smartapi:smartapi`); + if (redisData) { + data = (JSON.parse(redisData))?.predicates as PredicatesMetadata[]; + } + } + + if (!data) { + const file = fs.readFileSync(this._predicates_file_path, "utf-8"); + data = JSON.parse(file) as PredicatesMetadata[]; + } + return data; } - build() { - const specs = syncLoaderFactory( + async build() { + const specs = await syncLoaderFactory( this._options.smartAPIID, this._options.teamName, this._options.tag, @@ -152,15 +165,15 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui this._file_path, ); const nonTRAPIOps = this.loadOpsFromSpecs(specs); - const predicatesMetadata = this.fetch(); - global.missingAPIs = syncLoaderFactory( + const predicatesMetadata = await this.fetch(); + global.missingAPIs = (await syncLoaderFactory( undefined, undefined, undefined, undefined, undefined, this._file_path, - ).filter( + )).filter( spec => "info" in spec && "x-translator" in spec.info &&