update getgcdata function

odota · Nov 30, 2023 · 3d8a2cc · 3d8a2cc
1 parent f27ca8b
commit 3d8a2cc
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 77 deletions.
diff --git a/routes/spec.js b/routes/spec.js
@@ -12,7 +12,6 @@ const searchES = require("../store/searchES");
 const buildMatch = require("../store/buildMatch");
 const buildStatus = require("../store/buildStatus");
 const playerFields = require("./playerFields.json");
-// const getGcData = require("../util/getGcData");
 const utility = require("../util/utility");
 const db = require("../store/db");
 const redis = require("../store/redis");

diff --git a/sql/create_tables.sql b/sql/create_tables.sql
@@ -355,8 +355,6 @@ CREATE TABLE IF NOT EXISTS match_gcdata(
   match_id bigint PRIMARY KEY,
   cluster int,
   replay_salt int,
-  series_id int,
-  series_type int
 );
 
 CREATE TABLE IF NOT EXISTS items(

diff --git a/util/getGcData.js b/util/getGcData.js
@@ -3,79 +3,26 @@
  * Calls back with an object containing the GC data
  * */
 const moment = require("moment");
-const zlib = require("zlib");
 const utility = require("../util/utility");
 const config = require("../config");
 const queries = require("../store/queries");
 const db = require("../store/db");
 const redis = require("../store/redis");
+const { promisify }  = require('util');
 
 const secret = config.RETRIEVER_SECRET;
 const { getData, redisCount } = utility;
 const { insertMatch } = queries;
 
-function handleGcData(match, body, cb) {
-  // Persist parties and permanent buffs
-  const players = body.match.players.map((p, i) => ({
-    player_slot: p.player_slot,
-    party_id: p.party_id?.low,
-    permanent_buffs: p.permanent_buffs,
-    party_size: body.match.players.filter(
-      (matchPlayer) => matchPlayer.party_id?.low === p.party_id?.low
-    ).length,
-    net_worth: p.net_worth,
-  }));
-  const matchToInsert = {
-    match_id: match.match_id,
-    pgroup: match.pgroup,
-    players,
-    series_id: body.match.series_id,
-    series_type: body.match.series_type,
-  };
-  const gcdata = {
-    match_id: Number(match.match_id),
-    cluster: body.match.cluster,
-    replay_salt: body.match.replay_salt,
-    series_id: body.match.series_id,
-    series_type: body.match.series_type,
-  };
-  return insertMatch(
-    matchToInsert,
-    {
-      type: "gcdata",
-      skipParse: true,
-    },
-    (err) => {
-      if (err) {
-        return cb(err);
-      }
-      // Persist GC data to database
-      return queries.upsert(
-        db,
-        "match_gcdata",
-        gcdata,
-        {
-          match_id: match.match_id,
-        },
-        (err) => {
-          cb(err, gcdata);
-        }
-      );
-    }
-  );
-}
-
-redis.del("nonRetryable");
-
-function getGcDataFromRetriever(match, cb) {
+async function getGcDataFromRetriever(match, cb) {
   const retrieverArr = utility.getRetrieverArr(match.useGcDataArr);
   // make array of retriever urls and use a random one on each retry
   let urls = retrieverArr.map(
     (r) => `http://${r}?key=${secret}&match_id=${match.match_id}`
   );
   return getData(
     { url: urls, noRetry: match.noRetry, timeout: 5000 },
-    (err, body, metadata) => {
+    async (err, body, metadata) => {
       if (
         err ||
         !body ||
@@ -95,31 +42,63 @@ function getGcDataFromRetriever(match, cb) {
         "retrieverCounts",
         moment().startOf("hour").add(1, "hour").format("X")
       );
-
-      redis.setex(
-        `gcdata:${match.match_id}`,
-        60 * 60 * 24,
-        zlib.gzipSync(JSON.stringify(body))
-      );
       // TODO add discovered account_ids to database and fetch account data/rank medal
-      return handleGcData(match, body, cb);
+      try {
+      const players = body.match.players.map((p, i) => ({
+        player_slot: p.player_slot,
+        party_id: p.party_id?.low,
+        permanent_buffs: p.permanent_buffs,
+        party_size: body.match.players.filter(
+          (matchPlayer) => matchPlayer.party_id?.low === p.party_id?.low
+        ).length,
+        net_worth: p.net_worth,
+      }));
+      const matchToInsert = {
+        match_id: match.match_id,
+        pgroup: match.pgroup,
+        players,
+        series_id: body.match.series_id,
+        series_type: body.match.series_type,
+      };
+      const gcdata = {
+        match_id: Number(match.match_id),
+        cluster: body.match.cluster,
+        replay_salt: body.match.replay_salt,
+      };
+      // Put extra fields in matches/player_matches
+      await promisify(insertMatch)(
+        matchToInsert,
+        {
+          type: "gcdata",
+          skipParse: true,
+        });
+      // Persist GC data to database
+      await promisify(queries.upsert)(
+        db,
+        "match_gcdata",
+        gcdata,
+        {
+          match_id: match.match_id,
+        });
+      cb(null, gcdata);
+      } catch(e) {
+        cb(e);
+      }
     }
   );
 }
 
-module.exports = function getGcData(match, cb) {
+module.exports = async function getGcData(match, cb) {
   const matchId = match.match_id;
   if (!matchId || Number.isNaN(Number(matchId)) || Number(matchId) <= 0) {
     return cb(new Error("invalid match_id"));
   }
-  //return getGcDataFromRetriever(match, cb);
-  return redis.get(Buffer.from(`gcdata:${match.match_id}`), (err, body) => {
-    if (err) {
-      return cb(err);
-    }
-    if (body) {
-      return handleGcData(match, JSON.parse(zlib.gunzipSync(body)), cb);
-    }
-    return getGcDataFromRetriever(match, cb);
-  });
+  // Check if we have it in DB already
+  const saved = await db.raw(`select match_id, cluster, replay_salt from match_gcdata where match_id = ?`, [match.match_id]);
+  const gcdata = saved.rows[0];
+  if (gcdata) {
+    console.log('found cached gcdata for %s', matchId);
+    return cb(null, gcdata);
+  }
+  getGcDataFromRetriever(match, cb);
 };