have fallback IPFS

gitcoinco · Sep 5, 2024 · 540dc22 · 540dc22
1 parent b6eb77b
commit 540dc22
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 28 deletions.
diff --git a/indexer-compose.yml b/indexer-compose.yml
@@ -20,7 +20,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}
@@ -62,7 +62,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}

diff --git a/src/config.ts b/src/config.ts
@@ -1487,7 +1487,7 @@ const CHAINS: Chain[] = [
       .default("https://evm-rpc.sei-apis.com")
       .parse(process.env.SEI_MAINNET_RPC_URL),
     pricesFromTimestamp: Date.UTC(2024, 0, 1, 0, 0, 0),
-    maxGetLogsRange: 10000,
+    maxGetLogsRange: 1000,
     tokens: [
       {
         code: "SEI",
@@ -1818,7 +1818,7 @@ export type Config = {
   httpServerWaitForSync: boolean;
   httpServerEnabled: boolean;
   indexerEnabled: boolean;
-  ipfsGateway: string;
+  ipfsGateways: string[];
   coingeckoApiKey: string | null;
   coingeckoApiUrl: string;
   chains: Chain[];
@@ -1968,10 +1968,11 @@ export function getConfig(): Config {
 
   const runOnce = z.boolean().default(false).parse(args["run-once"]);
 
-  const ipfsGateway = z
+  const ipfsGateways = z
     .string()
-    .default("https://ipfs.io")
-    .parse(process.env.IPFS_GATEWAY);
+    .array()
+    .default(["https://ipfs.io"])
+    .parse(JSON.parse(process.env.IPFS_GATEWAYS!));
 
   const sentryDsn = z
     .union([z.string(), z.null()])
@@ -2028,7 +2029,7 @@ export function getConfig(): Config {
     cacheDir,
     logLevel,
     runOnce,
-    ipfsGateway,
+    ipfsGateways,
     passportScorerId,
     apiHttpPort,
     pinoPretty,

diff --git a/src/index.ts b/src/index.ts
@@ -465,29 +465,61 @@ async function catchupAndWatchChain(
         return undefined;
       }
 
-      const url = `${config.ipfsGateway}/ipfs/${cid}`;
+      // Fetch from a single IPFS gateway
+      const fetchFromGateway = async (url: string): Promise<T | undefined> => {
+        try {
+          const res = await fetch(url, {
+            timeout: 2000,
+            onRetry(cause) {
+              chainLogger.debug({
+                msg: "Retrying IPFS request",
+                url: url,
+                err: cause,
+              });
+            },
+            retry: { retries: 3, minTimeout: 2000, maxTimeout: 60 * 10000 },
+            // IPFS data is immutable, we can rely entirely on the cache when present
+            cache: "force-cache",
+            cachePath:
+              config.cacheDir !== null
+                ? path.join(config.cacheDir, "ipfs")
+                : undefined,
+          });
 
-      // chainLogger.trace(`Fetching ${url}`);
+          if (res.ok) {
+            return (await res.json()) as T; // Return the fetched data
+          } else {
+            chainLogger.warn(
+              `Failed to fetch from ${url}, status: ${res.status} ${res.statusText}`
+            );
+          }
+        } catch (err) {
+          chainLogger.error(`Error fetching from gateway ${url}: ${String(err)}`);
+        }
+      };
 
-      const res = await fetch(url, {
-        timeout: 2000,
-        onRetry(cause) {
-          chainLogger.debug({
-            msg: "Retrying IPFS request",
-            url: url,
-            err: cause,
-          });
-        },
-        retry: { retries: 3, minTimeout: 2000, maxTimeout: 60 * 10000 },
-        // IPFS data is immutable, we can rely entirely on the cache when present
-        cache: "force-cache",
-        cachePath:
-          config.cacheDir !== null
-            ? path.join(config.cacheDir, "ipfs")
-            : undefined,
-      });
+      // Iterate through each gateway and attempt to fetch data
+      for (const gateway of config.ipfsGateways) {
+        const url = `${gateway}/ipfs/${cid}`;
+        chainLogger.info(`Trying IPFS gateway: ${gateway} for CID: ${cid}`);
 
-      return (await res.json()) as T;
+        const result = await fetchFromGateway(url);
+        if (result !== undefined) {
+          chainLogger.info(
+            `Fetch successful from gateway: ${gateway} for CID: ${cid}`
+          );
+          return result; // Return the result if fetched successfully
+        } else {
+          chainLogger.warn(
+            `IPFS fetch failed for gateway ${gateway} for CID ${cid}`
+          );
+        }
+      }
+
+      chainLogger.error(
+        `Failed to fetch IPFS data for CID ${cid} from all gateways.`
+      );
+      return undefined; // Return undefined if all gateways fail
     };
 
     chainLogger.info("DEBUG: catching up with blockchain events");