Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sns process #4

Merged
merged 2 commits into from
Nov 7, 2024
Merged

Sns process #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
affaca3
[!] fix InvalidRecordData
ZhongFuze Nov 6, 2024
7a8aeb7
[!] twitter reverse lookup
ZhongFuze Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 140 additions & 8 deletions sns_process/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,66 @@ async function fetchPrimaryDomainsAndUpdate() {
}
}

async function fetchTwitterLookupAndUpdate() {
try {
console.log("Connected to PostgreSQL");

const batchSize = 100;
let allCount = 0;
let batchCount = 0;
let offset = 0;
let hasMoreRows = true;

while (hasMoreRows) {
// Fetch 100 distinct owners from the database
const query = `
SELECT DISTINCT ON (identity) identity
FROM nextid_proof
WHERE identity IS NOT NULL AND platform = 'twitter'
LIMIT $1 OFFSET $2`;

const batch = await db.any(query, [batchSize, offset]);
let hasTwitterReverseLookupCount = 0;
if (batch.length > 0) {
for (const row of batch) {
const handle = row.identity;
const solanaAddress = await retryGetTwitterReverseLookup(handle);
if (solanaAddress) {
hasTwitterReverseLookupCount += 1;
allCount++;
const formattedNow = dayjs().format('YYYY-MM-DD HH:mm:ss');
console.log([handle, solanaAddress, formattedNow]);
await db.none(
`INSERT INTO solana_twitter (twitter_handle, address, update_time)
VALUES ($1, $2, $3)
ON CONFLICT (twitter_handle, address) DO UPDATE
SET update_time = EXCLUDED.update_time;
`,
[handle, solanaAddress, formattedNow]
);
}
}

// Update last processed ID to keep track of pagination
offset += batchSize;
batchCount++;
console.log(`offset ${offset}`)
console.log(`Batch(${batchCount}) ${batch.length}rows has ${hasTwitterReverseLookupCount} upserted.`);
}

// Stop the loop if there are no more rows to fetch
if (batch.length < batchSize) {
hasMoreRows = false;
}
}
console.log(`All batches completed. Total upserted records: ${allCount}`);
} catch (error) {
console.error('Error:', error);
} finally {
console.log("Disconnected from PostgreSQL");
}
}

async function fetchTwitterHandlesAndUpdate() {
try {
console.log("Connected to PostgreSQL");
Expand Down Expand Up @@ -382,6 +442,15 @@ async function fetchTwitterHandlesAndUpdate() {
WHERE owner = $2`,
[twitterHandle, owner]
);
const formattedNow = dayjs().format('YYYY-MM-DD HH:mm:ss');
await db.none(
`INSERT INTO solana_twitter (twitter_handle, address, update_time)
VALUES ($1, $2, $3)
ON CONFLICT (twitter_handle, address) DO UPDATE
SET update_time = EXCLUDED.update_time;
`,
[twitterHandle, owner, formattedNow]
);
}
}

Expand Down Expand Up @@ -772,20 +841,73 @@ const getTwitterHandle = limiter.wrap(async (owner) => {
}
});

async function retryGetTwitterReverseLookup(handle, retries = 3) {
for (let attempt = 0; attempt < retries; attempt++) {
try {
return await getTwitterReverseLookup(handle); // Attempt to fetch texts
} catch (error) {
// Check for a specific error type (InvalidRecordData)
if (error.type === 'AccountDoesNotExist') {
// console.error(`Attempt ${attempt + 1} failed for handle ${handle}: Record data is malformed.`);
return null; // Return empty object if the data is malformed
} else {
console.error(`Attempt ${attempt + 1} failed for getTwitterReverseLookup ${handle}:`, error);
}
}

// Wait for 3 seconds before retrying
await new Promise(resolve => setTimeout(resolve, 3000));
}

// If all retries fail, return an empty object or handle differently
return null; // Return empty object as a fallback
}

const getTwitterReverseLookup = limiter.wrap(async (handle) => {
try {
const registry = await getTwitterRegistry(SOLANA_MAIN_CLIENT, handle);
const owner = registry.owner.toBase58();

if (owner) {
console.log(`Found Twitter handle: ${handle} reverse lookup for ${owner}`);
return owner;
} else {
console.log(`No address found for the given handle ${handle}.`);
return null
}
} catch (error) {
if (error instanceof Error && error.type === 'AccountDoesNotExist') {
// Log a friendly error message if the Twitter handle does not exist
console.log(`No reverse address found for this twitter handle(${handle}). Proceeding without it.`);
return null
} else {
// Re-throw any other error types
console.error("An unexpected error occurred:", error);
throw error
}
}
});

async function retryGetTexts(domainName, retries = 3) {
for (let attempt = 0; attempt < retries; attempt++) {
try {
return await getTexts(domainName);
return await getTexts(domainName); // Attempt to fetch texts
} catch (error) {
console.error(`Attempt ${attempt + 1} failed for getTexts ${domainName}:`, error);
// Check for a specific error type (InvalidRecordData)
if (error.type === 'InvalidRecordData') {
console.error(`Attempt ${attempt + 1} failed for domain ${domainName}: Record data is malformed.`);
return {}; // Return empty object if the data is malformed
} else {
console.error(`Attempt ${attempt + 1} failed for getTexts ${domainName}:`, error);
}
}

// Wait for 3 seconds before retrying
await new Promise(resolve => setTimeout(resolve, 3000));
}

// If all retries fail, return null
return {};
// If all retries fail, return an empty object or handle differently
return {}; // Return empty object as a fallback
}


Expand Down Expand Up @@ -825,8 +947,15 @@ const getTexts = limiter.wrap(async (domainName) => {
// console.log(`domain ${domainName} texts:`, texts);
return texts;
} catch (error) {
console.error(`Error fetching texts error:`, error);
throw error;
if (error instanceof Error && error.type === 'InvalidRecordData') {
// Log a friendly error message if the Twitter handle does not exist
// console.log("No record data found for this domain");
return {}
} else {
// Re-throw any other error types
console.error(`Error fetching texts error:`, error);
throw error
}
}
});

Expand All @@ -844,7 +973,8 @@ const run = async () => {
// await fetchDomainsByOwnersAndUpsert();
// await fetchPrimaryDomainsAndUpdate();
// await fetchTwitterHandlesAndUpdate();
await fetchDomainTextsAndUpdate();
// await fetchDomainTextsAndUpdate();
await fetchTwitterLookupAndUpdate();
};

// Execute the run function
Expand Down Expand Up @@ -881,4 +1011,6 @@ run().catch(console.error);

// await retryGetTexts("bonfida")
// await getTexts("vaxa")
// await getTexts("komacash")
// await getTexts("komacash")

// await getTwitterReverseLookup("suji_yan");
13 changes: 13 additions & 0 deletions src/script/create_sns_profile.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,16 @@ CREATE INDEX sns_profile_nft_owner ON sns_profile (nft_owner);
CREATE INDEX sns_profile_owner_index ON sns_profile (owner);
CREATE INDEX sns_profile_resolved_index ON sns_profile (resolved_address);
CREATE INDEX sns_profile_reverse_index ON sns_profile (reverse_address);

CREATE TABLE solana_twitter (
twitter_handle VARCHAR(255),
address VARCHAR(66),
update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT unique_solana_twitter UNIQUE (twitter_handle, address)
);


INSERT INTO solana_twitter (twitter_handle, address)
SELECT DISTINCT twitter_handle, owner
FROM sns_profile
WHERE twitter_handle != '';
Loading