From 3d8a2cc068ac277653a784f9f1ddb9d8342a2a62 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 30 Nov 2023 02:35:24 +0000 Subject: [PATCH] update getgcdata function --- routes/spec.js | 1 - sql/create_tables.sql | 2 - util/getGcData.js | 127 ++++++++++++++++++------------------------ 3 files changed, 53 insertions(+), 77 deletions(-) diff --git a/routes/spec.js b/routes/spec.js index a87adff40..33a757781 100644 --- a/routes/spec.js +++ b/routes/spec.js @@ -12,7 +12,6 @@ const searchES = require("../store/searchES"); const buildMatch = require("../store/buildMatch"); const buildStatus = require("../store/buildStatus"); const playerFields = require("./playerFields.json"); -// const getGcData = require("../util/getGcData"); const utility = require("../util/utility"); const db = require("../store/db"); const redis = require("../store/redis"); diff --git a/sql/create_tables.sql b/sql/create_tables.sql index bfd6e2867..0772562c9 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -355,8 +355,6 @@ CREATE TABLE IF NOT EXISTS match_gcdata( match_id bigint PRIMARY KEY, cluster int, replay_salt int, - series_id int, - series_type int ); CREATE TABLE IF NOT EXISTS items( diff --git a/util/getGcData.js b/util/getGcData.js index 7d858d17a..9c9a0b2a8 100644 --- a/util/getGcData.js +++ b/util/getGcData.js @@ -3,71 +3,18 @@ * Calls back with an object containing the GC data * */ const moment = require("moment"); -const zlib = require("zlib"); const utility = require("../util/utility"); const config = require("../config"); const queries = require("../store/queries"); const db = require("../store/db"); const redis = require("../store/redis"); +const { promisify } = require('util'); const secret = config.RETRIEVER_SECRET; const { getData, redisCount } = utility; const { insertMatch } = queries; -function handleGcData(match, body, cb) { - // Persist parties and permanent buffs - const players = body.match.players.map((p, i) => ({ - player_slot: p.player_slot, - party_id: p.party_id?.low, - permanent_buffs: p.permanent_buffs, - party_size: body.match.players.filter( - (matchPlayer) => matchPlayer.party_id?.low === p.party_id?.low - ).length, - net_worth: p.net_worth, - })); - const matchToInsert = { - match_id: match.match_id, - pgroup: match.pgroup, - players, - series_id: body.match.series_id, - series_type: body.match.series_type, - }; - const gcdata = { - match_id: Number(match.match_id), - cluster: body.match.cluster, - replay_salt: body.match.replay_salt, - series_id: body.match.series_id, - series_type: body.match.series_type, - }; - return insertMatch( - matchToInsert, - { - type: "gcdata", - skipParse: true, - }, - (err) => { - if (err) { - return cb(err); - } - // Persist GC data to database - return queries.upsert( - db, - "match_gcdata", - gcdata, - { - match_id: match.match_id, - }, - (err) => { - cb(err, gcdata); - } - ); - } - ); -} - -redis.del("nonRetryable"); - -function getGcDataFromRetriever(match, cb) { +async function getGcDataFromRetriever(match, cb) { const retrieverArr = utility.getRetrieverArr(match.useGcDataArr); // make array of retriever urls and use a random one on each retry let urls = retrieverArr.map( @@ -75,7 +22,7 @@ function getGcDataFromRetriever(match, cb) { ); return getData( { url: urls, noRetry: match.noRetry, timeout: 5000 }, - (err, body, metadata) => { + async (err, body, metadata) => { if ( err || !body || @@ -95,31 +42,63 @@ function getGcDataFromRetriever(match, cb) { "retrieverCounts", moment().startOf("hour").add(1, "hour").format("X") ); - - redis.setex( - `gcdata:${match.match_id}`, - 60 * 60 * 24, - zlib.gzipSync(JSON.stringify(body)) - ); // TODO add discovered account_ids to database and fetch account data/rank medal - return handleGcData(match, body, cb); + try { + const players = body.match.players.map((p, i) => ({ + player_slot: p.player_slot, + party_id: p.party_id?.low, + permanent_buffs: p.permanent_buffs, + party_size: body.match.players.filter( + (matchPlayer) => matchPlayer.party_id?.low === p.party_id?.low + ).length, + net_worth: p.net_worth, + })); + const matchToInsert = { + match_id: match.match_id, + pgroup: match.pgroup, + players, + series_id: body.match.series_id, + series_type: body.match.series_type, + }; + const gcdata = { + match_id: Number(match.match_id), + cluster: body.match.cluster, + replay_salt: body.match.replay_salt, + }; + // Put extra fields in matches/player_matches + await promisify(insertMatch)( + matchToInsert, + { + type: "gcdata", + skipParse: true, + }); + // Persist GC data to database + await promisify(queries.upsert)( + db, + "match_gcdata", + gcdata, + { + match_id: match.match_id, + }); + cb(null, gcdata); + } catch(e) { + cb(e); + } } ); } -module.exports = function getGcData(match, cb) { +module.exports = async function getGcData(match, cb) { const matchId = match.match_id; if (!matchId || Number.isNaN(Number(matchId)) || Number(matchId) <= 0) { return cb(new Error("invalid match_id")); } - //return getGcDataFromRetriever(match, cb); - return redis.get(Buffer.from(`gcdata:${match.match_id}`), (err, body) => { - if (err) { - return cb(err); - } - if (body) { - return handleGcData(match, JSON.parse(zlib.gunzipSync(body)), cb); - } - return getGcDataFromRetriever(match, cb); - }); + // Check if we have it in DB already + const saved = await db.raw(`select match_id, cluster, replay_salt from match_gcdata where match_id = ?`, [match.match_id]); + const gcdata = saved.rows[0]; + if (gcdata) { + console.log('found cached gcdata for %s', matchId); + return cb(null, gcdata); + } + getGcDataFromRetriever(match, cb); };