Merge branch 'main' of https://github.com/MystenLabs/walrus-sites int…

…o vl/SEINT-275-caching

portal/server/allowlist_checker.ts

@@ -0,0 +1,30 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+import { createClient, EdgeConfigClient } from '@vercel/edge-config';
+import { config } from 'configuration_loader';
+
+let edgeConfigAllowlistClient: EdgeConfigClient | undefined;
+if (config.enableAllowlist){
+    edgeConfigAllowlistClient = createClient(config.edgeConfigAllowlist);
+}
+/**
+* Check if a given subdomain is allowed to be served by the walrus site.
+* @param subdomain The walrus site subdomain to inspect
+* @returns true if the subdomain is allowed (has premium), false otherwise
+*/
+export async function isAllowed(subdomain: string): Promise<boolean> {
+    if (!config.enableAllowlist){
+        return false
+    }
+
+    if (!edgeConfigAllowlistClient){
+        throw new Error('Edge config allowlist client not initialized!')
+    }
+
+    const allowed: boolean = await edgeConfigAllowlistClient.has(
+       subdomain,
+    );
+
+    return allowed
+}

portal/server/app/route.ts

@@ -4,29 +4,19 @@
 import { getDomain, getSubdomainAndPath } from "@lib/domain_parsing";
 import { redirectToAggregatorUrlResponse, redirectToPortalURLResponse } from "@lib/redirects";
 import { getBlobIdLink, getObjectIdLink } from "@lib/links";
-import { UrlFetcher } from "@lib/url_fetcher";
-import { ResourceFetcher } from "@lib/resource";
-import { RPCSelector } from "@lib/rpc_selector";
 
+import { isAllowed } from "allowlist_checker";
 import { siteNotFound } from "@lib/http/http_error_responses";
 import integrateLoggerWithSentry from "sentry_logger";
 import blocklistChecker from "custom_blocklist_checker";
-import { SuiNSResolver } from "@lib/suins";
-import { WalrusSitesRouter } from "@lib/routing";
 import { config } from "configuration_loader";
+import { standardUrlFetcher, premiumUrlFetcher } from "url_fetcher_factory";
 
 if (config.enableSentry) {
     // Only integrate Sentry on production.
     integrateLoggerWithSentry();
 }
 
-const rpcSelector = new RPCSelector(config.rpcUrlList);
-const urlFetcher = new UrlFetcher(
-    new ResourceFetcher(rpcSelector),
-    new SuiNSResolver(rpcSelector),
-    new WalrusSitesRouter(rpcSelector)
-);
-
 export async function GET(req: Request) {
     const originalUrl = req.headers.get("x-original-url");
     if (!originalUrl) {
@@ -55,6 +45,7 @@ export async function GET(req: Request) {
             return siteNotFound();
         }
 
+        const urlFetcher = await isAllowed(parsedUrl.subdomain ?? '') ? premiumUrlFetcher : standardUrlFetcher;
         if (requestDomain == portalDomain && parsedUrl.subdomain) {
             return await urlFetcher.resolveDomainAndFetchUrl(parsedUrl, null, blocklistChecker);
         }
@@ -63,6 +54,8 @@ export async function GET(req: Request) {
     const atBaseUrl = portalDomain == url.host.split(":")[0];
     if (atBaseUrl) {
         console.log("Serving the landing page from walrus...");
+        // Always use the premium page fetcher for the landing page (when available).
+        const urlFetcher = config.enableAllowlist ? premiumUrlFetcher : standardUrlFetcher;
         const response = await urlFetcher.resolveDomainAndFetchUrl(
             {
                 subdomain: config.landingPageOidB36,

portal/server/configuration_loader.ts

@@ -15,7 +15,9 @@ function toBoolean(value: string): Boolean {
 */
 export type Configuration = {
     edgeConfig?: string;
+    edgeConfigAllowlist?: string;
     enableBlocklist: Boolean;
+    enableAllowlist: Boolean;
     landingPageOidB36: string;
     portalDomainNameLength?: number;
     premiumRpcUrlList: string[];
@@ -36,7 +38,9 @@ class ConfigurationLoader {
     get config(): Configuration {
         return {
             enableBlocklist: this.loadEnableBlocklist(),
+            enableAllowlist: this.loadEnableAllowlist(),
             edgeConfig: this.loadEdgeConfig(),
+            edgeConfigAllowlist: this.loadEdgeConfigAllowlist(),
             landingPageOidB36: this.loadLandingPageOidB36(),
             portalDomainNameLength: this.loadPortalDomainNameLength(),
             premiumRpcUrlList: this.loadPremiumRpcUrlList(),
@@ -51,6 +55,10 @@ class ConfigurationLoader {
         return this.loadEnableBlocklist() ? process.env.EDGE_CONFIG : undefined
     }
 
+    private loadEdgeConfigAllowlist(): string | undefined {
+        return this.loadEnableAllowlist() ? process.env.EDGE_CONFIG_ALLOWLIST : undefined
+    }
+
     private loadEnableBlocklist(): Boolean {
         if (!process.env.ENABLE_BLOCKLIST) {
             throw new Error('Missing ENABLE_BLOCKLIST environment variable.')
@@ -62,6 +70,17 @@ class ConfigurationLoader {
         return toBoolean(enable)
     }
 
+    private loadEnableAllowlist(): Boolean {
+        if (!process.env.ENABLE_ALLOWLIST) {
+            throw new Error('Missing ENABLE_ALLOWLIST environment variable.')
+        }
+        const enable = process.env.ENABLE_ALLOWLIST.toLowerCase()
+        if (!isStringBoolean(enable)) {
+            throw new Error('ENABLE_ALLOWLIST must be "true" or "false".')
+        }
+        return toBoolean(enable)
+    }
+
     private loadLandingPageOidB36(): string {
         const pageOidB36 = process.env.LANDING_PAGE_OID_B36
         if (!pageOidB36) {

portal/server/middleware.ts

@@ -7,6 +7,10 @@ import type { NextRequest } from 'next/server'
 export function middleware(request: NextRequest) {
     const urlOriginal = extractUrlFrom(request)
     const alreadyAtRoot = request.nextUrl.pathname === '/'
+    // Bypass middleware for walrus-sites-sw.js
+    if (request.nextUrl.pathname.endsWith('walrus-sites-sw.js')) {
+        return NextResponse.next()
+    }
     if (alreadyAtRoot) {
         const response = NextResponse.next()
         response.headers.set('x-original-url', urlOriginal)

portal/server/public/walrus-sites-sw.js

@@ -0,0 +1,15 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+if ("serviceWorker" in navigator) {
+  navigator.serviceWorker
+    .getRegistrations()
+    .then(function (registrations) {
+      registrations.forEach(function (registration) {
+        registration.unregister();
+      });
+    })
+    .catch(function (error) {
+        console.error("Error unregistering service workers:", error);
+    });
+}

portal/server/url_fetcher_factory.ts

@@ -0,0 +1,39 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+import { UrlFetcher } from "@lib/url_fetcher";
+import { ResourceFetcher } from "@lib/resource";
+import { RPCSelector } from "@lib/rpc_selector";
+import { SuiNSResolver } from "@lib/suins";
+import { WalrusSitesRouter } from "@lib/routing";
+import { config } from "configuration_loader";
+
+/**
+* A factory class for creating page fetchers.
+* Page fetchers can be either premium or standard.
+* Premium fetchers use premium RPC nodes that can serve content faster and more reliably,
+* while standard fetchers use standard RPC nodes.
+*/
+class UrlFetcherFactory {
+    private static readonly premiumRpcSelector = new RPCSelector(config.premiumRpcUrlList);
+    private static readonly standardRpcSelector = new RPCSelector(config.rpcUrlList);
+
+    public static premiumUrlFetcher(): UrlFetcher {
+        return new UrlFetcher(
+            new ResourceFetcher(this.standardRpcSelector),
+            new SuiNSResolver(this.standardRpcSelector),
+            new WalrusSitesRouter(this.standardRpcSelector)
+        );
+    }
+
+    public static standardUrlFetcher(): UrlFetcher {
+        return new UrlFetcher(
+            new ResourceFetcher(this.premiumRpcSelector),
+            new SuiNSResolver(this.premiumRpcSelector),
+            new WalrusSitesRouter(this.premiumRpcSelector)
+        );
+    }
+}
+
+export const standardUrlFetcher = UrlFetcherFactory.standardUrlFetcher();
+export const premiumUrlFetcher = UrlFetcherFactory.premiumUrlFetcher();