diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7b5b3dbb6..44297f752 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,5 +1,5 @@ name: docker-compose-actions-workflow -on: +on: push: pull_request: branches: diff --git a/dev/cassandraRandom.js b/dev/cassandraRandom.js index 78b076c1d..7d77ae7fe 100644 --- a/dev/cassandraRandom.js +++ b/dev/cassandraRandom.js @@ -1,38 +1,42 @@ -const cassandra = require('../store/cassandra'); -const db = require('../store/db'); +const cassandra = require("../store/cassandra"); +const db = require("../store/db"); function randomInteger(min, max) { - return Math.floor(Math.random() * (max - min + 1)) + min; - } + return Math.floor(Math.random() * (max - min + 1)) + min; +} let total = 0; let haveRows = 0; async function start() { - // Get the current max_match_id from postgres, subtract 200000000 - let max = (await db.raw(`select max(match_id) from public_matches`))?.rows?.[0]?.max; - let limit = max - 200000000; - while(true) { - // Test a random match ID - const rand = randomInteger(1, limit); + // Get the current max_match_id from postgres, subtract 200000000 + let max = (await db.raw(`select max(match_id) from public_matches`)) + ?.rows?.[0]?.max; + let limit = max - 200000000; + while (true) { + // Test a random match ID + const rand = randomInteger(1, limit); - let result = await cassandra.execute(`select match_id, player_slot, stuns from player_matches where match_id = ?`, [rand.toString()], { - prepare: true, - fetchSize: 10, - autoPage: true, - }); - total += 1; - // Check if there are rows - if (result.rows.length) { - haveRows += 1; - console.log(result.rows[0].match_id.toString(), 'has rows'); - } - if (total % 100 === 0) { - // Log number that have rows/don't have rows - console.log(haveRows, '/', total, 'have rows'); - } + let result = await cassandra.execute( + `select match_id, player_slot, stuns from player_matches where match_id = ?`, + [rand.toString()], + { + prepare: true, + fetchSize: 10, + autoPage: true, + } + ); + total += 1; + // Check if there are rows + if (result.rows.length) { + haveRows += 1; + console.log(result.rows[0].match_id.toString(), "has rows"); } - + if (total % 100 === 0) { + // Log number that have rows/don't have rows + console.log(haveRows, "/", total, "have rows"); + } + } } start(); diff --git a/processors/performanceOthers.js b/processors/performanceOthers.js index 9901da31b..7ff424d52 100644 --- a/processors/performanceOthers.js +++ b/processors/performanceOthers.js @@ -6,7 +6,7 @@ function greevilsGreed(e, container, meta) { const goldBase = 3; let goldStack = e.greevils_greed_stack * 3; - goldStack = Math.min(goldStack, 18); + goldStack = Math.min(goldStack, 18); alchPlayer.performance_others = { greevils_greed_gold: 0, diff --git a/routes/keyManagement.js b/routes/keyManagement.js index 75595f051..ad698b661 100644 --- a/routes/keyManagement.js +++ b/routes/keyManagement.js @@ -50,7 +50,7 @@ async function getOpenInvoices(customerId) { const invoices = await stripe.invoices.list({ customer: customerId, limit: 100, - status: 'open' + status: "open", }); return invoices.data; @@ -81,13 +81,13 @@ keys async.parallel( { customer: (cb) => { - if(!keyRecord) { + if (!keyRecord) { return cb(); } - const { api_key, customer_id, subscription_id} = keyRecord; + const { api_key, customer_id, subscription_id } = keyRecord; const toReturn = { - api_key + api_key, }; stripe.customers @@ -107,23 +107,22 @@ keys .catch((err) => cb(err)); }, openInvoices: (cb) => { - if(allKeyRecords.length === 0) { + if (allKeyRecords.length === 0) { return cb(); } customer_id = allKeyRecords[0].customer_id; - getOpenInvoices(customer_id).then(invoices => { - const processed = invoices.map(i => ({ + getOpenInvoices(customer_id).then((invoices) => { + const processed = invoices.map((i) => ({ id: i.id, amountDue: i.amount_due, paymentLink: i.hosted_invoice_url, - created: i.created + created: i.created, })); - return cb(null,processed); + return cb(null, processed); }); - }, usage: (cb) => { db.raw( @@ -218,14 +217,19 @@ keys return res.sendStatus(200); } // returning customer - else if(allKeyRecords.length > 0) { + else if (allKeyRecords.length > 0) { customer_id = allKeyRecords[0].customer_id; const invoices = await getOpenInvoices(customer_id); if (invoices.length > 0) { - console.log("Open invoices exist for", req.user.account_id, "customer", customer_id); - return res.status(402).json({error: "Open invoice"}); + console.log( + "Open invoices exist for", + req.user.account_id, + "customer", + customer_id + ); + return res.status(402).json({ error: "Open invoice" }); } try { @@ -239,7 +243,7 @@ keys } } // New customer -> create customer first - else { + else { try { const customer = await stripe.customers.create({ source: token.id, @@ -249,7 +253,7 @@ keys }, }); customer_id = customer.id; - } catch (err) { + } catch (err) { // probably insufficient funds return res.status(402).json(err); } diff --git a/store/queries.js b/store/queries.js index 9a38fce97..b58f12ca7 100644 --- a/store/queries.js +++ b/store/queries.js @@ -1443,15 +1443,20 @@ function insertMatch(match, options, cb) { } function decideScenarios(cb) { - if (options.type === 'parsed' && - match.match_id % 100 < config.SCENARIOS_SAMPLE_PERCENT) { + if ( + options.type === "parsed" && + match.match_id % 100 < config.SCENARIOS_SAMPLE_PERCENT + ) { return redis.rpush("scenariosQueue", match.match_id, cb); } return cb(); } function decideBenchmarks(cb) { - if (options.origin === "scanner" && match.match_id % 100 < config.BENCHMARKS_SAMPLE_PERCENT) { + if ( + options.origin === "scanner" && + match.match_id % 100 < config.BENCHMARKS_SAMPLE_PERCENT + ) { return redis.rpush("parsedBenchmarksQueue", match.match_id, cb); } return cb(); @@ -1573,7 +1578,8 @@ function insertMatch(match, options, cb) { }, }, { - priority: (match.leagueid || hasTrackedPlayer) ? -1 : options.priority, + priority: + match.leagueid || hasTrackedPlayer ? -1 : options.priority, attempts: options.attempts || 15, }, cb diff --git a/svc/apiadmin.js b/svc/apiadmin.js index b500a8c40..7d08ab93e 100644 --- a/svc/apiadmin.js +++ b/svc/apiadmin.js @@ -149,14 +149,19 @@ async function updateStripeUsage(cb) { // but we'd have to make changes to web.js and metrics await stripe.subscriptionItems.createUsageRecord(sub.items.data[0].id, { quantity: Math.ceil(usageCount / config.API_BILLING_UNIT), - action: 'set', + action: "set", timestamp: sub.current_period_end - 1, }); - console.log("updateStripeUsage updated", sub.id, usageCount, Math.ceil(usageCount / config.API_BILLING_UNIT)); + console.log( + "updateStripeUsage updated", + sub.id, + usageCount, + Math.ceil(usageCount / config.API_BILLING_UNIT) + ); } else { // console.log(`updateStripeUsage No usage for ${sub.id}`); } - } + } console.log(`updateStripeUsage processed ${num} records`); cb(); } catch (err) { @@ -167,7 +172,6 @@ async function updateStripeUsage(cb) { invokeInterval(function updateAPIKeysInRedis(cb) { queries.getAPIKeys(db, (err, rows) => { - if (err) { cb(err); } else if (rows.length > 0) { @@ -190,5 +194,7 @@ invokeInterval(function updateAPIKeysInRedis(cb) { }); }, 5 * 60 * 1000); // Update every 5 min -invokeInterval(function runStoreUsageCounts(cb){storeUsageCounts(0, cb);}, 10 * 60 * 1000); // Every 10 minutes +invokeInterval(function runStoreUsageCounts(cb) { + storeUsageCounts(0, cb); +}, 10 * 60 * 1000); // Every 10 minutes invokeInterval(updateStripeUsage, 5 * 60 * 1000); // Every 5 minutes diff --git a/svc/cassandraDelete.js b/svc/cassandraDelete.js index 9325db358..4358755ac 100644 --- a/svc/cassandraDelete.js +++ b/svc/cassandraDelete.js @@ -1,51 +1,79 @@ -const cassandra = require('../store/cassandra'); -const db = require('../store/db'); -const crypto = require('crypto'); +const cassandra = require("../store/cassandra"); +const db = require("../store/db"); +const crypto = require("crypto"); function genRandomNumber(byteCount, radix) { - return BigInt('0x' + crypto.randomBytes(byteCount).toString('hex')).toString(radix) + return BigInt("0x" + crypto.randomBytes(byteCount).toString("hex")).toString( + radix + ); } const PARSED_DATA_DELETE_ID = 0; async function start() { - // Get the current max_match_id from postgres, subtract 200000000 - let max = (await db.raw(`select max(match_id) from public_matches`))?.rows?.[0]?.max; - let limit = max - 200000000; - while(true) { - // delete older unparsed match/player_match rows - // We can backfill these from Steam API on demand - try { - // Convert to signed bigint - const randomBigint = BigInt.asIntN(64, genRandomNumber(8, 10)); - let result = await cassandra.execute(`select match_id, version, token(match_id) from matches where token(match_id) >= ? limit 500 ALLOW FILTERING;`, [randomBigint.toString()], { - prepare: true, - fetchSize: 500, - autoPage: true, - }); - - // Put the ones that don't have parsed data or are too old into an array - let ids = result.rows.filter(result => (result.version == null || result.match_id < PARSED_DATA_DELETE_ID) && result.match_id < limit).map(result => result.match_id); - console.log(ids.length, 'out of', result.rows.length, 'to delete, ex:', ids[0]?.toString()); + // Get the current max_match_id from postgres, subtract 200000000 + let max = (await db.raw(`select max(match_id) from public_matches`)) + ?.rows?.[0]?.max; + let limit = max - 200000000; + while (true) { + // delete older unparsed match/player_match rows + // We can backfill these from Steam API on demand + try { + // Convert to signed bigint + const randomBigint = BigInt.asIntN(64, genRandomNumber(8, 10)); + let result = await cassandra.execute( + `select match_id, version, token(match_id) from matches where token(match_id) >= ? limit 500 ALLOW FILTERING;`, + [randomBigint.toString()], + { + prepare: true, + fetchSize: 500, + autoPage: true, + } + ); - // Delete matches - await Promise.all(ids.map(id => cassandra.execute(`DELETE from matches where match_id = ?`, [id], { - prepare: true, - }) - )); - // Delete player_matches - await Promise.all(ids.map(id => cassandra.execute(`DELETE from player_matches where match_id = ?`, [id], { - prepare: true, - }) - )); + // Put the ones that don't have parsed data or are too old into an array + let ids = result.rows + .filter( + (result) => + (result.version == null || + result.match_id < PARSED_DATA_DELETE_ID) && + result.match_id < limit + ) + .map((result) => result.match_id); + console.log( + ids.length, + "out of", + result.rows.length, + "to delete, ex:", + ids[0]?.toString() + ); - const parsedIds = result.rows.filter(result => result.version != null).map(result => result.match_id); - await Promise.all(parsedIds.map(id => db.raw(`INSERT INTO parsed_matches(match_id) VALUES(?) ON CONFLICT DO NOTHING`, [Number(id)]))); - } catch(e) { - console.log(e); - } + // Delete matches + await Promise.all( + ids.map((id) => + cassandra.execute(`DELETE from matches where match_id = ?`, [id], { + prepare: true, + }) + ) + ); + // Delete player_matches + await Promise.all( + ids.map((id) => + cassandra.execute( + `DELETE from player_matches where match_id = ?`, + [id], + { + prepare: true, + } + ) + ) + ); + const parsedIds = result.rows.filter(result => result.version != null).map(result => result.match_id); + await Promise.all(parsedIds.map(id => db.raw(`INSERT INTO parsed_matches(match_id) VALUES(?) ON CONFLICT DO NOTHING`, [Number(id)]))); + } catch (e) { + console.log(e); } - + } } start(); diff --git a/svc/retriever.js b/svc/retriever.js index 4c56fe159..4b561fa19 100644 --- a/svc/retriever.js +++ b/svc/retriever.js @@ -129,7 +129,7 @@ Steam.servers = [ ]; function selfDestruct() { - console.log('shutting down'); + console.log("shutting down"); process.exit(0); } @@ -255,7 +255,10 @@ function init() { }); client.on("connected", () => { const logOnDetails = chooseLoginInfo(); - console.log("[STEAM] Trying to log on with %s", JSON.stringify(logOnDetails)); + console.log( + "[STEAM] Trying to log on with %s", + JSON.stringify(logOnDetails) + ); client.steamUser.logOn(logOnDetails); }); client.on("logOnResponse", (logOnResp) => { @@ -393,7 +396,7 @@ function init() { client.connect(); } */ - /* + /* client.on('loggedOff', () => { console.log('relogging'); setTimeout(()=> { @@ -507,7 +510,7 @@ app.use((req, res, cb) => { profileRequests, getUptime(), matchRequestDelay + matchRequestDelayIncr, - req.query, + req.query ); const shouldRestart = // (matchSuccesses / matchRequests < 0.1 && matchRequests > 100 && getUptime() > minUpTimeSeconds) || diff --git a/test/test.js b/test/test.js index ab6bc4a68..f7ee6f373 100644 --- a/test/test.js +++ b/test/test.js @@ -398,7 +398,7 @@ describe("api", () => { (verb, cb) => { if ( path.indexOf("/explorer") === 0 || - path.indexOf("/request") === 0 || + path.indexOf("/request") === 0 ) { return cb(err); } diff --git a/util/getGcData.js b/util/getGcData.js index d0e3313b2..8e4580783 100644 --- a/util/getGcData.js +++ b/util/getGcData.js @@ -8,7 +8,7 @@ const config = require("../config"); const queries = require("../store/queries"); const db = require("../store/db"); const redis = require("../store/redis"); -const { promisify } = require('util'); +const { promisify } = require("util"); const secret = config.RETRIEVER_SECRET; const { getData, redisCount } = utility; @@ -44,44 +44,38 @@ async function getGcDataFromRetriever(match, cb) { ); // TODO add discovered account_ids to database and fetch account data/rank medal try { - const players = body.match.players.map((p, i) => ({ - player_slot: p.player_slot, - party_id: p.party_id?.low, - permanent_buffs: p.permanent_buffs, - party_size: body.match.players.filter( - (matchPlayer) => matchPlayer.party_id?.low === p.party_id?.low - ).length, - net_worth: p.net_worth, - })); - const matchToInsert = { - match_id: match.match_id, - pgroup: match.pgroup, - players, - series_id: body.match.series_id, - series_type: body.match.series_type, - }; - const gcdata = { - match_id: Number(match.match_id), - cluster: body.match.cluster, - replay_salt: body.match.replay_salt, - }; - // Put extra fields in matches/player_matches - await promisify(insertMatch)( - matchToInsert, - { + const players = body.match.players.map((p, i) => ({ + player_slot: p.player_slot, + party_id: p.party_id?.low, + permanent_buffs: p.permanent_buffs, + party_size: body.match.players.filter( + (matchPlayer) => matchPlayer.party_id?.low === p.party_id?.low + ).length, + net_worth: p.net_worth, + })); + const matchToInsert = { + match_id: match.match_id, + pgroup: match.pgroup, + players, + series_id: body.match.series_id, + series_type: body.match.series_type, + }; + const gcdata = { + match_id: Number(match.match_id), + cluster: body.match.cluster, + replay_salt: body.match.replay_salt, + }; + // Put extra fields in matches/player_matches + await promisify(insertMatch)(matchToInsert, { type: "gcdata", skipParse: true, }); - // Persist GC data to database - await promisify(queries.upsert)( - db, - "match_gcdata", - gcdata, - { + // Persist GC data to database + await promisify(queries.upsert)(db, "match_gcdata", gcdata, { match_id: match.match_id, }); - cb(null, gcdata); - } catch(e) { + cb(null, gcdata); + } catch (e) { cb(e); } } @@ -94,10 +88,13 @@ module.exports = async function getGcData(match, cb) { return cb(new Error("invalid match_id")); } // Check if we have it in DB already - const saved = await db.raw(`select match_id, cluster, replay_salt from match_gcdata where match_id = ?`, [match.match_id]); + const saved = await db.raw( + `select match_id, cluster, replay_salt from match_gcdata where match_id = ?`, + [match.match_id] + ); const gcdata = saved.rows[0]; if (gcdata) { - console.log('found cached gcdata for %s', matchId); + console.log("found cached gcdata for %s", matchId); redisCount(redis, "cached_gcdata"); return cb(null, gcdata); }