use redis for smartapi specs

package.json

@@ -79,7 +79,8 @@
     "debug": "^4.3.4",
     "husky": "^8.0.3",
     "load-json-file": "^7.0.1",
-    "lodash": "^4.17.21"
+    "lodash": "^4.17.21",
+    "@biothings-explorer/utils": "workspace:../utils"
   },
   "husky": {
     "hooks": {

src/index.ts

@@ -57,8 +57,8 @@ export default class MetaKG {
    * Construct API Meta Knowledge Graph based on SmartAPI Specifications.
    * @param {string} tag - the SmartAPI tag to be filtered on
    */
-  constructMetaKGSync(includeReasoner = false, options: BuilderOptions = {}): SmartAPIKGOperationObject[] {
-    this._ops = syncBuilderFactory(options, includeReasoner, this._file_path, this._predicates_path);
+  async constructMetaKGSync(includeReasoner = false, options: BuilderOptions = {}): Promise<SmartAPIKGOperationObject[]> {
+    this._ops = await syncBuilderFactory(options, includeReasoner, this._file_path, this._predicates_path);
     return this.ops;
   }
 

src/load/all_specs_sync_loader.ts

@@ -4,17 +4,30 @@ import { SmartAPISpec } from "../parser/types";
 import fs from "fs";
 import Debug from "debug";
 const debug = Debug("bte:smartapi-kg:AllSpecsSyncLoader");
+const { redisClient } = require("@biothings-explorer/utils");
 
 export default class AllSpecsSyncLoader extends BaseLoader {
   private _file_path: string;
   constructor(path: string) {
     super();
     this._file_path = path;
   }
-  protected fetch(): SmartAPIQueryResult {
-    debug(`Fetching from file path: ${this._file_path}`);
-    const file = fs.readFileSync(this._file_path, "utf-8");
-    const data = JSON.parse(file) as SmartAPIQueryResult | SmartAPISpec;
+  protected async fetch(): Promise<SmartAPIQueryResult> {
+    debug(`Fetching from file path: ${this._file_path} ${redisClient.clientEnabled}`);
+    let data: SmartAPIQueryResult | SmartAPISpec;
+
+    if (redisClient.clientEnabled) {
+        const redisData = await redisClient.client.getTimeout(`bte:smartapi:smartapi`).catch(console.log)
+        if (redisData) {
+            data = (JSON.parse(redisData))?.smartapi as SmartAPIQueryResult | SmartAPISpec;
+        }
+    }
+
+    if (!data) {
+        const file = fs.readFileSync(this._file_path, "utf-8");
+        data = JSON.parse(file) as SmartAPIQueryResult | SmartAPISpec;
+    }
+
     let result;
     if (!("hits" in data)) {
       result = {
@@ -31,8 +44,8 @@ export default class AllSpecsSyncLoader extends BaseLoader {
     return input.hits;
   }
 
-  load(): SmartAPISpec[] {
-    const specs = this.fetch();
+  async load(): Promise<SmartAPISpec[]> {
+    const specs = await this.fetch();
     return this.parse(specs);
   }
 }

src/load/sync_loader_factory.ts

@@ -9,14 +9,14 @@ import { apiListObject } from "../types";
 import Debug from "debug";
 const debug = Debug("bte:smartapi-kg:SyncLoader");
 
-export const syncLoaderFactory = (
+export const syncLoaderFactory = async (
   smartAPIID: string = undefined,
   teamName: string = undefined,
   tag: string = undefined,
   component: string = undefined,
   apiList: apiListObject = undefined,
   path: string,
-): SmartAPISpec[] => {
+): Promise<SmartAPISpec[]> => {
   let loader;
   if (!(typeof smartAPIID === "undefined")) {
     loader = new SingleSpecSyncLoader(smartAPIID, path, apiList);
@@ -37,5 +37,5 @@ export const syncLoaderFactory = (
     loader = new AllSpecsSyncLoader(path);
     debug("Using all specs sync loader now.");
   }
-  return loader.load();
+  return await loader.load();
 };

src/operations_builder/sync_builder_factory.ts

@@ -3,19 +3,19 @@ import SyncOperationsBuilderWithReasoner from "./sync_operations_builder_with_re
 import { BuilderOptions } from "../types";
 import { SmartAPIKGOperationObject, SmartAPISpec } from "../parser/types";
 
-export function syncBuilderFactory(
+export async function syncBuilderFactory(
   options: BuilderOptions,
   includeReasoner: boolean,
   smartapi_path: string,
   predicates_path: string,
-): SmartAPIKGOperationObject[] {
+): Promise<SmartAPIKGOperationObject[]> {
   let builder;
   if (includeReasoner === true) {
     builder = new SyncOperationsBuilderWithReasoner(options, smartapi_path, predicates_path);
   } else {
     builder = new SyncOperationsBuilder(options, smartapi_path);
   }
-  const ops = builder.build();
+  const ops = await builder.build();
 
   const primaryKnowledgeAPIs = new Set();
   options.apiList?.include.forEach(api => {

src/operations_builder/sync_operations_builder.ts

@@ -10,8 +10,8 @@ export default class SyncOperationsBuilder extends BaseOperationsBuilder {
     this._file_path = path;
   }
 
-  build() {
-    const specs = syncLoaderFactory(
+  async build() {
+    const specs = await syncLoaderFactory(
       this._options.smartAPIID,
       this._options.teamName,
       this._options.tag,

src/operations_builder/sync_operations_builder_with_reasoner.ts

@@ -6,6 +6,7 @@ import { SmartAPIKGOperationObject } from "../parser/types";
 import { PredicatesMetadata } from "../types";
 import Debug from "debug";
 const debug = Debug("bte:smartapi-kg:SyncOperationsBuilderWithReasoner");
+const { redisClient } = require("@biothings-explorer/utils");
 import { SmartAPISpec } from "../parser/types";
 
 declare global {
@@ -136,14 +137,26 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui
     return ops;
   }
 
-  private fetch(): PredicatesMetadata[] {
-    const file = fs.readFileSync(this._predicates_file_path, "utf-8");
-    const data = JSON.parse(file) as PredicatesMetadata[];
+  private async fetch(): Promise<PredicatesMetadata[]> {
+    let data: PredicatesMetadata[];
+
+    if (redisClient.clientEnabled) {
+        const redisData = await redisClient.client.getTimeout(`bte:smartapi:smartapi`);
+        if (redisData) {
+            data = (JSON.parse(redisData))?.predicates as PredicatesMetadata[];
+        }
+    }
+
+    if (!data) {
+        const file = fs.readFileSync(this._predicates_file_path, "utf-8");
+        data = JSON.parse(file) as PredicatesMetadata[];
+    }
+
     return data;
   }
 
-  build() {
-    const specs = syncLoaderFactory(
+  async build() {
+    const specs = await syncLoaderFactory(
       this._options.smartAPIID,
       this._options.teamName,
       this._options.tag,
@@ -152,15 +165,15 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui
       this._file_path,
     );
     const nonTRAPIOps = this.loadOpsFromSpecs(specs);
-    const predicatesMetadata = this.fetch();
-    global.missingAPIs = syncLoaderFactory(
+    const predicatesMetadata = await this.fetch();
+    global.missingAPIs = (await syncLoaderFactory(
       undefined,
       undefined,
       undefined,
       undefined,
       undefined,
       this._file_path,
-    ).filter(
+    )).filter(
       spec =>
         "info" in spec &&
         "x-translator" in spec.info &&