fix inserting pro data

odota · Dec 1, 2023 · b5be678 · b5be678
1 parent be2dce9
commit b5be678
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 30 deletions.
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
@@ -451,6 +451,14 @@ CREATE TABLE IF NOT EXISTS subscriber (
   status varchar(100)
 );
 
+CREATE TABLE IF NOT EXISTS match_blobs (
+  PRIMARY KEY (match_id),
+  match_id bigint,
+  basic json,
+  gcdata json,
+  replay json
+);
+
 DO $$
 BEGIN
     IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'readonly') THEN

diff --git a/store/queries.js b/store/queries.js
@@ -1095,12 +1095,12 @@ function insertMatch(match, options, cb) {
   async function upsertMatchPostgres(cb) {
     // Check if leagueid is premium/professional
     const result =
-      match.leagueid &&
+      match.leagueid ?
       (await db.raw(
         `select leagueid from leagues where leagueid = ? and (tier = 'premium' OR tier = 'professional')`,
         [match.leagueid]
-      ));
-    const pass = result?.rows?.length > 0 && utility.isProMatch(match);
+      )) : null;
+    const pass = result?.rows?.length > 0;
     if (!pass) {
       // Skip this if not a pro match
       return cb();

diff --git a/svc/cassandraDelete.js b/svc/cassandraDelete.js
@@ -11,8 +11,6 @@ function genRandomNumber(byteCount, radix) {
   );
 }
 
-const PARSED_DATA_DELETE_ID = 0;
-
 async function start() {
   // Get the current max_match_id from postgres, subtract 200000000
   const max = (await db.raw('select max(match_id) from public_matches'))
@@ -29,39 +27,32 @@ async function start() {
         [randomBigint.toString()],
         {
           prepare: true,
-          fetchSize: 500,
+          fetchSize: 100,
           autoPage: true,
         }
       );
 
       // Put the ones that don't have parsed data or are too old into an array
-      const ids = result.rows
+      const unparsedIds = result.rows
         .filter(
           (result) =>
-            (result.version == null ||
-              result.match_id < PARSED_DATA_DELETE_ID) &&
+            result.version == null &&
             result.match_id < limit
         )
         .map((result) => result.match_id);
-      console.log(
-        ids.length,
-        'out of',
+      const parsedIds = result.rows
+        .filter((result) => result.version != null && result.match_id < limit)
+        .map((result) => result.match_id);
+      console.log('%s unparsed to delete, %s parsed to archive, %s total, del ID: %s',
+        unparsedIds.length,
+        parsedIds.length,
         result.rows.length,
-        'to delete, ex:',
-        ids[0]?.toString()
-      );
-
-      // Delete matches
-      await Promise.all(
-        ids.map((id) =>
-          cassandra.execute('DELETE from matches where match_id = ?', [id], {
-            prepare: true,
-          })
-        )
+        unparsedIds[0]?.toString()
       );
+      // NOTE: Due to lack of transactions there might be some orphaned player_matches without match
       // Delete player_matches
       await Promise.all(
-        ids.map((id) =>
+        unparsedIds.map((id) =>
           cassandra.execute(
             'DELETE from player_matches where match_id = ?',
             [id],
@@ -71,9 +62,14 @@ async function start() {
           )
         )
       );
-      const parsedIds = result.rows
-        .filter((result) => result.version != null)
-        .map((result) => result.match_id);
+      // Delete matches
+      await Promise.all(
+        unparsedIds.map((id) =>
+          cassandra.execute('DELETE from matches where match_id = ?', [id], {
+            prepare: true,
+          })
+        )
+      );
       config.MATCH_ARCHIVE_S3_ENDPOINT &&
         (await Promise.all(parsedIds.map((id) => doArchive(id))));
 
@@ -102,16 +98,16 @@ async function doArchive(matchId) {
   const result = await archivePut(matchId.toString(), blob);
   if (result) {
     // TODO Delete from Cassandra after archival
-    // await cassandra.execute("DELETE from matches where match_id = ?", [matchId], {
-    //   prepare: true,
-    // });
     // await cassandra.execute(
     //   "DELETE from player_matches where match_id = ?",
     //   [matchId],
     //   {
     //     prepare: true,
     //   }
     // );
+    // await cassandra.execute("DELETE from matches where match_id = ?", [matchId], {
+    //   prepare: true,
+    // });
   }
   return;
 }