From 7b2629d7a4587f7dc7ea853bcb5d38104eb45436 Mon Sep 17 00:00:00 2001 From: Mariano Goldman Date: Thu, 22 Aug 2024 11:28:13 -0300 Subject: [PATCH] feat: adapt existing commands to use new lambda function ab-admin --- src/commands/queue-ab-conversion-about.ts | 12 ++++--- src/commands/queue-ab-conversion-snapshot.ts | 33 +++++++++++++------- src/commands/queue-ab-conversion.ts | 16 ++++++---- src/helpers/asset-bundles.ts | 31 ++++++++++-------- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/src/commands/queue-ab-conversion-about.ts b/src/commands/queue-ab-conversion-about.ts index d824cdf..3dcf253 100644 --- a/src/commands/queue-ab-conversion-about.ts +++ b/src/commands/queue-ab-conversion-about.ts @@ -1,7 +1,7 @@ import { AuthLinkType } from '@dcl/schemas' import arg from 'arg' import { assert } from '../helpers/assert' -import { multiPlatformFlag, queueConversions } from '../helpers/asset-bundles' +import { defaultAbAdmin, Platform, queueConversions } from '../helpers/asset-bundles' import { fetch } from 'undici' import { CliError } from '../bin' import { parseEntityUrn } from '../helpers/parseEntityUrn' @@ -10,6 +10,7 @@ export default async () => { const args = arg({ '--about-url': String, '--ab-server': String, + '--platform': [String], '--token': String, '--force': Boolean, '--prioritize': Boolean @@ -17,7 +18,8 @@ export default async () => { const aboutUrl = args['--about-url']! const token = args['--token']! - const abServer = args['--ab-server'] || multiPlatformFlag + const abServer = args['--ab-server'] || defaultAbAdmin + const platforms = (args['--platform'] as Platform[]) || Object.values(Platform) const force = args['--force'] || false const shouldPrioritize = !!args['--prioritize'] @@ -25,6 +27,7 @@ export default async () => { assert(!!token, '--token is missing') console.log(`> Parameters:`) + console.log(` Platform(s): ${platforms.join(',')}`) console.log(` Asset bundle server: ${abServer}`) console.log(` Force rebuild: ${force}`) @@ -53,9 +56,10 @@ export default async () => { force }, token, - shouldPrioritize + shouldPrioritize, + platforms ) - console.log(` Result: ${JSON.stringify(result)}`) + console.log(` Result:`, JSON.stringify(result, undefined, 2)) } console.log(`Finished!`) diff --git a/src/commands/queue-ab-conversion-snapshot.ts b/src/commands/queue-ab-conversion-snapshot.ts index 9f272ee..06d06d0 100644 --- a/src/commands/queue-ab-conversion-snapshot.ts +++ b/src/commands/queue-ab-conversion-snapshot.ts @@ -3,7 +3,7 @@ import arg from 'arg' import { fetch } from 'undici' import { CliError } from '../bin' import { assert } from '../helpers/assert' -import { multiPlatformFlag, queueConversions } from '../helpers/asset-bundles' +import { defaultAbAdmin, Platform, queueConversions } from '../helpers/asset-bundles' import { StringDecoder } from 'string_decoder' import { exit } from 'process' @@ -18,13 +18,15 @@ export default async () => { '--start-date': String, '--grep': String, '--ab-server': String, + '--platform': [String], '--token': String, '--prioritize': Boolean }) const snapshot = args['--snapshot'] || 'wearable' const token = args['--token']! - const abServer = args['--ab-server'] || multiPlatformFlag + const abServer = args['--ab-server'] || defaultAbAdmin + const platforms = (args['--platform'] as Platform[]) || Object.values(Platform) const shouldPrioritize = !!args['--prioritize'] assert(!!snapshot, '--snapshot is missing') @@ -32,6 +34,7 @@ export default async () => { console.log(`> Parameters:`) const contentUrl = (args['--content-server'] || 'https://peer.decentraland.org/content').replace(/\/$/, '') + console.log(` Platform(s): ${platforms.join(',')}`) console.log(` Entity type: ${snapshot}`) console.log(` Content server: ${contentUrl}`) console.log(` Asset bundle server: ${abServer}`) @@ -39,9 +42,9 @@ export default async () => { if (snapshot === 'worlds') { const specificWorld: string | undefined = args['--world-name'] if (specificWorld) { - await processWorld(abServer, token, specificWorld, shouldPrioritize) + await processWorld(abServer, token, specificWorld, shouldPrioritize, platforms) } else { - await processWorlds(abServer, token, shouldPrioritize) + await processWorlds(abServer, token, shouldPrioritize, platforms) } console.log(`Finished!`) @@ -97,7 +100,7 @@ export default async () => { } if (startDate <= entity.entityTimestamp && entity.entityType === snapshot) { - await tryRetryQueueConversion(abServer, entity.entityId, contentUrl, token, shouldPrioritize) + await tryRetryQueueConversion(abServer, entity.entityId, contentUrl, token, shouldPrioritize, platforms) console.log(` (${i + 1}/${snapshotsCount}) [${percent}%]`, entity.entityId, entity.pointers[0]) } @@ -110,7 +113,7 @@ export default async () => { console.log(`Finished!`) } -const processWorlds = async (abServer: string, token: string, prioritize: boolean) => { +const processWorlds = async (abServer: string, token: string, prioritize: boolean, platforms: Platform[]) => { console.log('Processing worlds.') const worldsIndexUrl = 'https://worlds-content-server.decentraland.org/index' const worldsContentUrl = 'https://worlds-content-server.decentraland.org/' @@ -133,12 +136,18 @@ const processWorlds = async (abServer: string, token: string, prioritize: boolea console.log(`> [${percent}%]`, name, scene.id) - await tryRetryQueueConversion(abServer, scene.id, worldsContentUrl, token, prioritize) + await tryRetryQueueConversion(abServer, scene.id, worldsContentUrl, token, prioritize, platforms) } } } -const processWorld = async (abServer: string, token: string, worldName: string, prioritize: boolean) => { +const processWorld = async ( + abServer: string, + token: string, + worldName: string, + prioritize: boolean, + platforms: Platform[] +) => { console.log(`Processing world: ${worldName}.`) const worldsIndexUrl = 'https://worlds-content-server.decentraland.org/index' const worldsContentUrl = 'https://worlds-content-server.decentraland.org/' @@ -162,7 +171,7 @@ const processWorld = async (abServer: string, token: string, worldName: string, const percent = (100 * ((j + 1) / scenes.length)).toFixed(2) console.log(`> [${percent}%]`, world.name, scene.id) - await tryRetryQueueConversion(abServer, scene.id, worldsContentUrl, token, prioritize) + await tryRetryQueueConversion(abServer, scene.id, worldsContentUrl, token, prioritize, platforms) } } } @@ -174,6 +183,7 @@ const tryRetryQueueConversion = async ( contentUrl: string, token: string, prioritize: boolean, + platforms: Platform[], retryCount: number = 0 ) => { if (retryCount > 3) { @@ -197,12 +207,13 @@ const tryRetryQueueConversion = async ( contentServerUrls: [contentUrl] }, token, - prioritize + prioritize, + platforms ) } catch (error) { console.log(`> Unexpected error, retrying in 5 seconds...`) await new Promise((f) => setTimeout(f, 5000)) - await tryRetryQueueConversion(abServer, entityId, contentUrl, token, prioritize, retryCount + 1) + await tryRetryQueueConversion(abServer, entityId, contentUrl, token, prioritize, platforms, retryCount + 1) } } diff --git a/src/commands/queue-ab-conversion.ts b/src/commands/queue-ab-conversion.ts index f71fbb0..bf76c06 100644 --- a/src/commands/queue-ab-conversion.ts +++ b/src/commands/queue-ab-conversion.ts @@ -1,7 +1,7 @@ import { AuthLinkType, IPFSv1, IPFSv2 } from '@dcl/schemas' import arg from 'arg' import { assert } from '../helpers/assert' -import { multiPlatformFlag, queueConversions } from '../helpers/asset-bundles' +import { defaultAbAdmin, Platform, queueConversions } from '../helpers/asset-bundles' import { getActiveEntities } from '../helpers/downloads' export default async () => { @@ -10,6 +10,7 @@ export default async () => { '--pointer': [String], '--content-server': String, '--ab-server': String, + '--platform': [String], '--token': String, '--prioritize': Boolean }) @@ -17,7 +18,9 @@ export default async () => { const pointers = args['--pointer'] || [] const cids = args['--cid'] || [] const token = args['--token']! - const abServer = args['--ab-server'] || multiPlatformFlag + const abServer = args['--ab-server'] || defaultAbAdmin + const platforms = (args['--platform'] as Platform[]) || Object.values(Platform) + const contentUrl = (args['--content-server'] || 'https://peer.decentraland.org/content').replace(/\/$/, '') const shouldPrioritize = !!args['--prioritize'] assert(!!token, '--token is missing') @@ -25,8 +28,8 @@ export default async () => { console.log(`> Parameters:`) pointers.length && console.log(` Pointers: ${pointers.join(',')}`) - cids.length && console.log(` CIDs: ${cids.join(',')}`) - const contentUrl = (args['--content-server'] || 'https://peer.decentraland.org/content').replace(/\/$/, '') + cids.length && console.log(` CIDs: ${cids.join(', ')}`) + console.log(` Platform(s): ${platforms.join(',')}`) console.log(` Content server: ${contentUrl}`) console.log(` Asset bundle server: ${abServer}`) @@ -65,9 +68,10 @@ export default async () => { contentServerUrls: [contentUrl] }, token, - shouldPrioritize + shouldPrioritize, + platforms ) - console.log(` Result: ${JSON.stringify(result)}`) + console.log(` Result:`, JSON.stringify(result, undefined, 2)) } console.log(`Finished!`) diff --git a/src/helpers/asset-bundles.ts b/src/helpers/asset-bundles.ts index bcc43e8..06cdc69 100644 --- a/src/helpers/asset-bundles.ts +++ b/src/helpers/asset-bundles.ts @@ -2,7 +2,13 @@ import { DeploymentToSqs } from '@dcl/schemas/dist/misc/deployments-to-sqs' import { fetch } from 'undici' import { CliError } from '../bin' -export const multiPlatformFlag = 'MultiPlatform' +export const defaultAbAdmin = 'https://ab-admin.decentraland.org' + +export enum Platform { + WEBGL = 'webgl', + WINDOWS = 'windows', + MAC = 'mac' +} const abServers = [ 'https://asset-bundle-converter.decentraland.org', @@ -13,35 +19,34 @@ export async function queueConversions( customABConverterServer: string, entity: DeploymentToSqs, token: string, - prioritize: boolean + prioritize: boolean, + platforms: Platform[] ): Promise> { const ids: Array<{ id: string }> = [] - if (customABConverterServer === multiPlatformFlag) { - for (const assetConverterServer of abServers) { - ids.push(await queueConversion(assetConverterServer, entity, token, prioritize)) - } - } else { - ids.push(await queueConversion(customABConverterServer, entity, token, prioritize)) - } + ids.push(await queueConversion(customABConverterServer, entity, token, prioritize, platforms)) + return ids } -export async function queueConversion( +async function queueConversion( assetConverterServer: string, body: DeploymentToSqs, token: string, - prioritize: boolean + prioritize: boolean, + platforms: Platform[] ): Promise<{ id: string }> { - const url = `${assetConverterServer}/queue-task` + const url = new URL(`${assetConverterServer}/enqueue-task`) + platforms.forEach((platform) => url.searchParams.append('platform', platform)) if (prioritize) { ;(body as any).prioritize = true } + console.log(`> Enqueueing task to ${url}`, JSON.stringify([body])) const res = await fetch(url, { method: 'post', - body: JSON.stringify(body), + body: JSON.stringify([body]), headers: { 'content-type': 'application/json', Authorization: token