-
Notifications
You must be signed in to change notification settings - Fork 303
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
7811124
commit 819a449
Showing
10 changed files
with
179 additions
and
131 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| name: docker-compose-actions-workflow | ||
| on: | ||
| on: | ||
| push: | ||
| pull_request: | ||
| branches: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,38 +1,42 @@ | ||
| const cassandra = require('../store/cassandra'); | ||
| const db = require('../store/db'); | ||
| const cassandra = require("../store/cassandra"); | ||
| const db = require("../store/db"); | ||
|
|
||
| function randomInteger(min, max) { | ||
| return Math.floor(Math.random() * (max - min + 1)) + min; | ||
| } | ||
| return Math.floor(Math.random() * (max - min + 1)) + min; | ||
| } | ||
|
|
||
| let total = 0; | ||
| let haveRows = 0; | ||
|
|
||
| async function start() { | ||
| // Get the current max_match_id from postgres, subtract 200000000 | ||
| let max = (await db.raw(`select max(match_id) from public_matches`))?.rows?.[0]?.max; | ||
| let limit = max - 200000000; | ||
| while(true) { | ||
| // Test a random match ID | ||
| const rand = randomInteger(1, limit); | ||
| // Get the current max_match_id from postgres, subtract 200000000 | ||
| let max = (await db.raw(`select max(match_id) from public_matches`)) | ||
| ?.rows?.[0]?.max; | ||
| let limit = max - 200000000; | ||
| while (true) { | ||
| // Test a random match ID | ||
| const rand = randomInteger(1, limit); | ||
|
|
||
| let result = await cassandra.execute(`select match_id, player_slot, stuns from player_matches where match_id = ?`, [rand.toString()], { | ||
| prepare: true, | ||
| fetchSize: 10, | ||
| autoPage: true, | ||
| }); | ||
| total += 1; | ||
| // Check if there are rows | ||
| if (result.rows.length) { | ||
| haveRows += 1; | ||
| console.log(result.rows[0].match_id.toString(), 'has rows'); | ||
| } | ||
| if (total % 100 === 0) { | ||
| // Log number that have rows/don't have rows | ||
| console.log(haveRows, '/', total, 'have rows'); | ||
| } | ||
| let result = await cassandra.execute( | ||
| `select match_id, player_slot, stuns from player_matches where match_id = ?`, | ||
| [rand.toString()], | ||
| { | ||
| prepare: true, | ||
| fetchSize: 10, | ||
| autoPage: true, | ||
| } | ||
| ); | ||
| total += 1; | ||
| // Check if there are rows | ||
| if (result.rows.length) { | ||
| haveRows += 1; | ||
| console.log(result.rows[0].match_id.toString(), "has rows"); | ||
| } | ||
|
|
||
| if (total % 100 === 0) { | ||
| // Log number that have rows/don't have rows | ||
| console.log(haveRows, "/", total, "have rows"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| start(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,51 +1,79 @@ | ||
| const cassandra = require('../store/cassandra'); | ||
| const db = require('../store/db'); | ||
| const crypto = require('crypto'); | ||
| const cassandra = require("../store/cassandra"); | ||
| const db = require("../store/db"); | ||
| const crypto = require("crypto"); | ||
|
|
||
| function genRandomNumber(byteCount, radix) { | ||
| return BigInt('0x' + crypto.randomBytes(byteCount).toString('hex')).toString(radix) | ||
| return BigInt("0x" + crypto.randomBytes(byteCount).toString("hex")).toString( | ||
| radix | ||
| ); | ||
| } | ||
|
|
||
| const PARSED_DATA_DELETE_ID = 0; | ||
|
|
||
| async function start() { | ||
| // Get the current max_match_id from postgres, subtract 200000000 | ||
| let max = (await db.raw(`select max(match_id) from public_matches`))?.rows?.[0]?.max; | ||
| let limit = max - 200000000; | ||
| while(true) { | ||
| // delete older unparsed match/player_match rows | ||
| // We can backfill these from Steam API on demand | ||
| try { | ||
| // Convert to signed bigint | ||
| const randomBigint = BigInt.asIntN(64, genRandomNumber(8, 10)); | ||
| let result = await cassandra.execute(`select match_id, version, token(match_id) from matches where token(match_id) >= ? limit 500 ALLOW FILTERING;`, [randomBigint.toString()], { | ||
| prepare: true, | ||
| fetchSize: 500, | ||
| autoPage: true, | ||
| }); | ||
|
|
||
| // Put the ones that don't have parsed data or are too old into an array | ||
| let ids = result.rows.filter(result => (result.version == null || result.match_id < PARSED_DATA_DELETE_ID) && result.match_id < limit).map(result => result.match_id); | ||
| console.log(ids.length, 'out of', result.rows.length, 'to delete, ex:', ids[0]?.toString()); | ||
| // Get the current max_match_id from postgres, subtract 200000000 | ||
| let max = (await db.raw(`select max(match_id) from public_matches`)) | ||
| ?.rows?.[0]?.max; | ||
| let limit = max - 200000000; | ||
| while (true) { | ||
| // delete older unparsed match/player_match rows | ||
| // We can backfill these from Steam API on demand | ||
| try { | ||
| // Convert to signed bigint | ||
| const randomBigint = BigInt.asIntN(64, genRandomNumber(8, 10)); | ||
| let result = await cassandra.execute( | ||
| `select match_id, version, token(match_id) from matches where token(match_id) >= ? limit 500 ALLOW FILTERING;`, | ||
| [randomBigint.toString()], | ||
| { | ||
| prepare: true, | ||
| fetchSize: 500, | ||
| autoPage: true, | ||
| } | ||
| ); | ||
|
|
||
| // Delete matches | ||
| await Promise.all(ids.map(id => cassandra.execute(`DELETE from matches where match_id = ?`, [id], { | ||
| prepare: true, | ||
| }) | ||
| )); | ||
| // Delete player_matches | ||
| await Promise.all(ids.map(id => cassandra.execute(`DELETE from player_matches where match_id = ?`, [id], { | ||
| prepare: true, | ||
| }) | ||
| )); | ||
| // Put the ones that don't have parsed data or are too old into an array | ||
| let ids = result.rows | ||
| .filter( | ||
| (result) => | ||
| (result.version == null || | ||
| result.match_id < PARSED_DATA_DELETE_ID) && | ||
| result.match_id < limit | ||
| ) | ||
| .map((result) => result.match_id); | ||
| console.log( | ||
| ids.length, | ||
| "out of", | ||
| result.rows.length, | ||
| "to delete, ex:", | ||
| ids[0]?.toString() | ||
| ); | ||
|
|
||
| const parsedIds = result.rows.filter(result => result.version != null).map(result => result.match_id); | ||
| await Promise.all(parsedIds.map(id => db.raw(`INSERT INTO parsed_matches(match_id) VALUES(?) ON CONFLICT DO NOTHING`, [Number(id)]))); | ||
| } catch(e) { | ||
| console.log(e); | ||
| } | ||
| // Delete matches | ||
| await Promise.all( | ||
| ids.map((id) => | ||
| cassandra.execute(`DELETE from matches where match_id = ?`, [id], { | ||
| prepare: true, | ||
| }) | ||
| ) | ||
| ); | ||
| // Delete player_matches | ||
| await Promise.all( | ||
| ids.map((id) => | ||
| cassandra.execute( | ||
| `DELETE from player_matches where match_id = ?`, | ||
| [id], | ||
| { | ||
| prepare: true, | ||
| } | ||
| ) | ||
| ) | ||
| ); | ||
| const parsedIds = result.rows.filter(result => result.version != null).map(result => result.match_id); | ||
| await Promise.all(parsedIds.map(id => db.raw(`INSERT INTO parsed_matches(match_id) VALUES(?) ON CONFLICT DO NOTHING`, [Number(id)]))); | ||
| } catch (e) { | ||
| console.log(e); | ||
| } | ||
|
|
||
| } | ||
| } | ||
|
|
||
| start(); |
Oops, something went wrong.