[#] fetching sns texts record

sns_process/index.js

@@ -405,6 +405,65 @@ async function fetchTwitterHandlesAndUpdate() {
     }
 }
 
+async function fetchDomainTextsAndUpdate() {
+    try {
+        console.log("Connected to PostgreSQL");
+
+        const batchSize = 100;
+        let allCount = 0;
+        let batchCount = 0;
+        let offset = 0;
+        let hasMoreRows = true;
+
+        while (hasMoreRows) {
+            // Fetch 100 distinct owners from the database
+            const query = `
+                SELECT name
+                FROM sns_profile 
+                WHERE name IS NOT NULL AND owner != '11111111111111111111111111111111'
+                LIMIT $1 OFFSET $2`;
+
+            const batch = await db.any(query, [batchSize, offset]);
+            let hasTextsCount = 0;
+            if (batch.length > 0) {
+                for (const row of batch) {
+                    const name = row.name;
+                    const domainName = name.endsWith('.sol') ? name.slice(0, -4) : name;
+                    const domain_texts = await retryGetTexts(domainName);
+                    if (Object.keys(domain_texts).length > 0) {
+                        hasTextsCount += 1;
+                        allCount++;
+                        const texts_json = JSON.stringify(domain_texts);
+                        console.log(`domain ${domainName} texts:`, texts_json);
+                        await db.none(
+                            `UPDATE sns_profile 
+                             SET texts = $1
+                             WHERE name = $2`,
+                            [texts_json, name]
+                        );
+                    }
+                }
+
+                // Update last processed ID to keep track of pagination
+                offset += batchSize;
+                batchCount++;
+                console.log(`offset ${offset}`)
+                console.log(`Batch(${batchCount}) ${batch.length}rows has ${hasTextsCount} upserted.`);
+            }
+
+            // Stop the loop if there are no more rows to fetch
+            if (batch.length < batchSize) {
+                hasMoreRows = false;
+            }
+        }
+        console.log(`All batches completed. Total upserted records: ${allCount}`);
+    } catch (error) {
+        console.error('Error:', error);
+    } finally {
+        console.log("Disconnected from PostgreSQL");
+    }
+}
+
 async function processAndUpsertBatchForloop(batch) {
     try {
         for (const domain_pubkey of batch) {
@@ -712,6 +771,64 @@ const getTwitterHandle = limiter.wrap(async (owner) => {
         }
     }
 });
+
+async function retryGetTexts(domainName, retries = 3) {
+    for (let attempt = 0; attempt < retries; attempt++) {
+        try {
+            return await getTexts(domainName);
+        } catch (error) {
+            console.error(`Attempt ${attempt + 1} failed for getTexts ${domainName}:`, error);
+        }
+
+        // Wait for 3 seconds before retrying
+        await new Promise(resolve => setTimeout(resolve, 3000));
+    }
+
+    // If all retries fail, return null
+    return {};
+}
+
+
+const getTexts = limiter.wrap(async (domainName) => {
+    try {
+        const record_keys = [
+            Record.IPNS,
+            Record.IPFS,
+            Record.ARWV,
+            Record.SOL,
+            Record.BTC,
+            Record.LTC,
+            Record.DOGE,
+            Record.BSC,
+            Record.Email,
+            Record.Url,
+            Record.Discord,
+            Record.Reddit,
+            Record.Twitter,
+            Record.Telegram,
+            Record.Pic,
+            Record.SHDW,
+            Record.POINT,
+            Record.Injective,
+            Record.Backpack
+        ];
+
+        const texts = {}; // json object
+        const records = await getRecords(SOLANA_MAIN_CLIENT, domainName, record_keys, true);
+        for (let i = 0; i < records.length; i++) {
+            if (records[i] !== undefined) {
+                if (record_keys[i]) {
+                    texts[record_keys[i].toLowerCase()] = records[i];
+                }
+            }
+        }
+        // console.log(`domain ${domainName} texts:`, texts);
+        return texts;
+    } catch (error) {
+        console.error(`Error fetching texts error:`, error);
+        throw error;
+    }
+});
 
 const getPublicKeyFromSolDomain = limiter.wrap(async (domain) => {
     const { pubkey } = getDomainKeySync(domain);
@@ -726,7 +843,8 @@ const run = async () => {
     // await fetchDomainsAndUpsert();
     // await fetchDomainsByOwnersAndUpsert();
     // await fetchPrimaryDomainsAndUpdate();
-    await fetchTwitterHandlesAndUpdate();
+    // await fetchTwitterHandlesAndUpdate();
+    await fetchDomainTextsAndUpdate();
 };
 
 // Execute the run function
@@ -759,4 +877,8 @@ run().catch(console.error);
 // const pubkey = new PublicKey("GaK7gLTevzAUHMpXkBupRTCeeFxWCq5wVkeHnnNDtivv");
 // const pubkey = new PublicKey("HKKp49qGWXd639QsuH7JiLijfVW5UtCVY4s1n2HANwEA");
 
-// await getTwitterHandle("HKKp49qGWXd639QsuH7JiLijfVW5UtCVY4s1n2HANwEA");
+// await getTwitterHandle("HKKp49qGWXd639QsuH7JiLijfVW5UtCVY4s1n2HANwEA");
+
+// await retryGetTexts("bonfida")
+// await getTexts("vaxa")
+// await getTexts("komacash")