From b5be67811ba9ca1d4328e8bf603def1d853faee0 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardzchung@gmail.com>
Date: Fri, 1 Dec 2023 05:42:44 +0000
Subject: [PATCH] fix inserting pro data

---
 sql/create_tables.sql  |  8 +++++++
 store/queries.js       |  6 ++---
 svc/cassandraDelete.js | 50 +++++++++++++++++++-----------------------
 3 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 1eea2d956..549be04d2 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -451,6 +451,14 @@ CREATE TABLE IF NOT EXISTS subscriber (
   status varchar(100)
 );
 
+CREATE TABLE IF NOT EXISTS match_blobs (
+  PRIMARY KEY (match_id),
+  match_id bigint,
+  basic json,
+  gcdata json,
+  replay json
+);
+
 DO $$
 BEGIN
     IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'readonly') THEN
diff --git a/store/queries.js b/store/queries.js
index 1d6822bd4..0e9f9227a 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -1095,12 +1095,12 @@ function insertMatch(match, options, cb) {
   async function upsertMatchPostgres(cb) {
     // Check if leagueid is premium/professional
     const result =
-      match.leagueid &&
+      match.leagueid ?
       (await db.raw(
         `select leagueid from leagues where leagueid = ? and (tier = 'premium' OR tier = 'professional')`,
         [match.leagueid]
-      ));
-    const pass = result?.rows?.length > 0 && utility.isProMatch(match);
+      )) : null;
+    const pass = result?.rows?.length > 0;
     if (!pass) {
       // Skip this if not a pro match
       return cb();
diff --git a/svc/cassandraDelete.js b/svc/cassandraDelete.js
index ea72f8bb6..16e0c46a9 100644
--- a/svc/cassandraDelete.js
+++ b/svc/cassandraDelete.js
@@ -11,8 +11,6 @@ function genRandomNumber(byteCount, radix) {
   );
 }
 
-const PARSED_DATA_DELETE_ID = 0;
-
 async function start() {
   // Get the current max_match_id from postgres, subtract 200000000
   const max = (await db.raw('select max(match_id) from public_matches'))
@@ -29,39 +27,32 @@ async function start() {
         [randomBigint.toString()],
         {
           prepare: true,
-          fetchSize: 500,
+          fetchSize: 100,
           autoPage: true,
         }
       );
 
       // Put the ones that don't have parsed data or are too old into an array
-      const ids = result.rows
+      const unparsedIds = result.rows
         .filter(
           (result) =>
-            (result.version == null ||
-              result.match_id < PARSED_DATA_DELETE_ID) &&
+            result.version == null &&
             result.match_id < limit
         )
         .map((result) => result.match_id);
-      console.log(
-        ids.length,
-        'out of',
+      const parsedIds = result.rows
+        .filter((result) => result.version != null && result.match_id < limit)
+        .map((result) => result.match_id);
+      console.log('%s unparsed to delete, %s parsed to archive, %s total, del ID: %s',
+        unparsedIds.length,
+        parsedIds.length,
         result.rows.length,
-        'to delete, ex:',
-        ids[0]?.toString()
-      );
-
-      // Delete matches
-      await Promise.all(
-        ids.map((id) =>
-          cassandra.execute('DELETE from matches where match_id = ?', [id], {
-            prepare: true,
-          })
-        )
+        unparsedIds[0]?.toString()
       );
+      // NOTE: Due to lack of transactions there might be some orphaned player_matches without match
       // Delete player_matches
       await Promise.all(
-        ids.map((id) =>
+        unparsedIds.map((id) =>
           cassandra.execute(
             'DELETE from player_matches where match_id = ?',
             [id],
@@ -71,9 +62,14 @@ async function start() {
           )
         )
       );
-      const parsedIds = result.rows
-        .filter((result) => result.version != null)
-        .map((result) => result.match_id);
+      // Delete matches
+      await Promise.all(
+        unparsedIds.map((id) =>
+          cassandra.execute('DELETE from matches where match_id = ?', [id], {
+            prepare: true,
+          })
+        )
+      );
       config.MATCH_ARCHIVE_S3_ENDPOINT &&
         (await Promise.all(parsedIds.map((id) => doArchive(id))));
 
@@ -102,9 +98,6 @@ async function doArchive(matchId) {
   const result = await archivePut(matchId.toString(), blob);
   if (result) {
     // TODO Delete from Cassandra after archival
-    // await cassandra.execute("DELETE from matches where match_id = ?", [matchId], {
-    //   prepare: true,
-    // });
     // await cassandra.execute(
     //   "DELETE from player_matches where match_id = ?",
     //   [matchId],
@@ -112,6 +105,9 @@ async function doArchive(matchId) {
     //     prepare: true,
     //   }
     // );
+    // await cassandra.execute("DELETE from matches where match_id = ?", [matchId], {
+    //   prepare: true,
+    // });
   }
   return;
 }