diff --git a/sns_process/index.js b/sns_process/index.js index eccb5f1..5691428 100644 --- a/sns_process/index.js +++ b/sns_process/index.js @@ -349,6 +349,66 @@ async function fetchPrimaryDomainsAndUpdate() { } } +async function fetchTwitterLookupAndUpdate() { + try { + console.log("Connected to PostgreSQL"); + + const batchSize = 100; + let allCount = 0; + let batchCount = 0; + let offset = 0; + let hasMoreRows = true; + + while (hasMoreRows) { + // Fetch 100 distinct owners from the database + const query = ` + SELECT DISTINCT ON (identity) identity + FROM nextid_proof + WHERE identity IS NOT NULL AND platform = 'twitter' + LIMIT $1 OFFSET $2`; + + const batch = await db.any(query, [batchSize, offset]); + let hasTwitterReverseLookupCount = 0; + if (batch.length > 0) { + for (const row of batch) { + const handle = row.identity; + const solanaAddress = await retryGetTwitterReverseLookup(handle); + if (solanaAddress) { + hasTwitterReverseLookupCount += 1; + allCount++; + const formattedNow = dayjs().format('YYYY-MM-DD HH:mm:ss'); + console.log([handle, solanaAddress, formattedNow]); + await db.none( + `INSERT INTO solana_twitter (twitter_handle, address, update_time) + VALUES ($1, $2, $3) + ON CONFLICT (twitter_handle, address) DO UPDATE + SET update_time = EXCLUDED.update_time; + `, + [handle, solanaAddress, formattedNow] + ); + } + } + + // Update last processed ID to keep track of pagination + offset += batchSize; + batchCount++; + console.log(`offset ${offset}`) + console.log(`Batch(${batchCount}) ${batch.length}rows has ${hasTwitterReverseLookupCount} upserted.`); + } + + // Stop the loop if there are no more rows to fetch + if (batch.length < batchSize) { + hasMoreRows = false; + } + } + console.log(`All batches completed. Total upserted records: ${allCount}`); + } catch (error) { + console.error('Error:', error); + } finally { + console.log("Disconnected from PostgreSQL"); + } +} + async function fetchTwitterHandlesAndUpdate() { try { console.log("Connected to PostgreSQL"); @@ -382,6 +442,15 @@ async function fetchTwitterHandlesAndUpdate() { WHERE owner = $2`, [twitterHandle, owner] ); + const formattedNow = dayjs().format('YYYY-MM-DD HH:mm:ss'); + await db.none( + `INSERT INTO solana_twitter (twitter_handle, address, update_time) + VALUES ($1, $2, $3) + ON CONFLICT (twitter_handle, address) DO UPDATE + SET update_time = EXCLUDED.update_time; + `, + [twitterHandle, owner, formattedNow] + ); } } @@ -772,20 +841,73 @@ const getTwitterHandle = limiter.wrap(async (owner) => { } }); +async function retryGetTwitterReverseLookup(handle, retries = 3) { + for (let attempt = 0; attempt < retries; attempt++) { + try { + return await getTwitterReverseLookup(handle); // Attempt to fetch texts + } catch (error) { + // Check for a specific error type (InvalidRecordData) + if (error.type === 'AccountDoesNotExist') { + // console.error(`Attempt ${attempt + 1} failed for handle ${handle}: Record data is malformed.`); + return null; // Return empty object if the data is malformed + } else { + console.error(`Attempt ${attempt + 1} failed for getTwitterReverseLookup ${handle}:`, error); + } + } + + // Wait for 3 seconds before retrying + await new Promise(resolve => setTimeout(resolve, 3000)); + } + + // If all retries fail, return an empty object or handle differently + return null; // Return empty object as a fallback +} + +const getTwitterReverseLookup = limiter.wrap(async (handle) => { + try { + const registry = await getTwitterRegistry(SOLANA_MAIN_CLIENT, handle); + const owner = registry.owner.toBase58(); + + if (owner) { + console.log(`Found Twitter handle: ${handle} reverse lookup for ${owner}`); + return owner; + } else { + console.log(`No address found for the given handle ${handle}.`); + return null + } + } catch (error) { + if (error instanceof Error && error.type === 'AccountDoesNotExist') { + // Log a friendly error message if the Twitter handle does not exist + console.log(`No reverse address found for this twitter handle(${handle}). Proceeding without it.`); + return null + } else { + // Re-throw any other error types + console.error("An unexpected error occurred:", error); + throw error + } + } +}); + async function retryGetTexts(domainName, retries = 3) { for (let attempt = 0; attempt < retries; attempt++) { try { - return await getTexts(domainName); + return await getTexts(domainName); // Attempt to fetch texts } catch (error) { - console.error(`Attempt ${attempt + 1} failed for getTexts ${domainName}:`, error); + // Check for a specific error type (InvalidRecordData) + if (error.type === 'InvalidRecordData') { + console.error(`Attempt ${attempt + 1} failed for domain ${domainName}: Record data is malformed.`); + return {}; // Return empty object if the data is malformed + } else { + console.error(`Attempt ${attempt + 1} failed for getTexts ${domainName}:`, error); + } } // Wait for 3 seconds before retrying await new Promise(resolve => setTimeout(resolve, 3000)); } - // If all retries fail, return null - return {}; + // If all retries fail, return an empty object or handle differently + return {}; // Return empty object as a fallback } @@ -825,8 +947,15 @@ const getTexts = limiter.wrap(async (domainName) => { // console.log(`domain ${domainName} texts:`, texts); return texts; } catch (error) { - console.error(`Error fetching texts error:`, error); - throw error; + if (error instanceof Error && error.type === 'InvalidRecordData') { + // Log a friendly error message if the Twitter handle does not exist + // console.log("No record data found for this domain"); + return {} + } else { + // Re-throw any other error types + console.error(`Error fetching texts error:`, error); + throw error + } } }); @@ -844,7 +973,8 @@ const run = async () => { // await fetchDomainsByOwnersAndUpsert(); // await fetchPrimaryDomainsAndUpdate(); // await fetchTwitterHandlesAndUpdate(); - await fetchDomainTextsAndUpdate(); + // await fetchDomainTextsAndUpdate(); + await fetchTwitterLookupAndUpdate(); }; // Execute the run function @@ -881,4 +1011,6 @@ run().catch(console.error); // await retryGetTexts("bonfida") // await getTexts("vaxa") -// await getTexts("komacash") \ No newline at end of file +// await getTexts("komacash") + +// await getTwitterReverseLookup("suji_yan"); \ No newline at end of file diff --git a/src/script/create_sns_profile.sql b/src/script/create_sns_profile.sql index 5c35a81..a57de34 100644 --- a/src/script/create_sns_profile.sql +++ b/src/script/create_sns_profile.sql @@ -28,3 +28,16 @@ CREATE INDEX sns_profile_nft_owner ON sns_profile (nft_owner); CREATE INDEX sns_profile_owner_index ON sns_profile (owner); CREATE INDEX sns_profile_resolved_index ON sns_profile (resolved_address); CREATE INDEX sns_profile_reverse_index ON sns_profile (reverse_address); + +CREATE TABLE solana_twitter ( + twitter_handle VARCHAR(255), + address VARCHAR(66), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT unique_solana_twitter UNIQUE (twitter_handle, address) +); + + +INSERT INTO solana_twitter (twitter_handle, address) +SELECT DISTINCT twitter_handle, owner +FROM sns_profile +WHERE twitter_handle != '';