have fallback IPFS

gitcoinco · Sep 5, 2024 · 07c5f20 · 07c5f20
1 parent b6eb77b
commit 07c5f20
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 28 deletions.
diff --git a/indexer-compose.yml b/indexer-compose.yml
@@ -20,7 +20,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}
@@ -62,7 +62,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}

diff --git a/src/config.ts b/src/config.ts
@@ -1487,7 +1487,7 @@ const CHAINS: Chain[] = [
       .default("https://evm-rpc.sei-apis.com")
       .parse(process.env.SEI_MAINNET_RPC_URL),
     pricesFromTimestamp: Date.UTC(2024, 0, 1, 0, 0, 0),
-    maxGetLogsRange: 10000,
+    maxGetLogsRange: 999,
     tokens: [
       {
         code: "SEI",
@@ -1818,7 +1818,7 @@ export type Config = {
   httpServerWaitForSync: boolean;
   httpServerEnabled: boolean;
   indexerEnabled: boolean;
-  ipfsGateway: string;
+  ipfsGateways: string[];
   coingeckoApiKey: string | null;
   coingeckoApiUrl: string;
   chains: Chain[];
@@ -1968,10 +1968,11 @@ export function getConfig(): Config {
 
   const runOnce = z.boolean().default(false).parse(args["run-once"]);
 
-  const ipfsGateway = z
+  const ipfsGateways = z
     .string()
-    .default("https://ipfs.io")
-    .parse(process.env.IPFS_GATEWAY);
+    .array()
+    .default(["https://ipfs.io"])
+    .parse(process.env.IPFS_GATEWAYS);
 
   const sentryDsn = z
     .union([z.string(), z.null()])
@@ -2028,7 +2029,7 @@ export function getConfig(): Config {
     cacheDir,
     logLevel,
     runOnce,
-    ipfsGateway,
+    ipfsGateways,
     passportScorerId,
     apiHttpPort,
     pinoPretty,

diff --git a/src/index.ts b/src/index.ts
@@ -465,29 +465,53 @@ async function catchupAndWatchChain(
         return undefined;
       }
 
-      const url = `${config.ipfsGateway}/ipfs/${cid}`;
+      // Fetch from a single IPFS gateway
+      const fetchFromGateway = async (url: string): Promise<T | undefined> => {
+        for (let attempt = 0; attempt < 3; attempt++) {
+          try {
+            const res = await fetch(url, {
+              timeout: 2000,
+              onRetry(cause) {
+                chainLogger.debug({
+                  msg: "Retrying IPFS request",
+                  url: url,
+                  err: cause,
+                });
+              },
+              retry: { retries: 3, minTimeout: 2000, maxTimeout: 60 * 10000 },
+              // IPFS data is immutable, we can rely entirely on the cache when present
+              cache: "force-cache",
+              cachePath:
+                config.cacheDir !== null
+                  ? path.join(config.cacheDir, "ipfs")
+                  : undefined,
+            });
 
-      // chainLogger.trace(`Fetching ${url}`);
+            if (res.ok) {
+              return (await res.json()) as T; // Return the fetched data
+            } else {
+              chainLogger.warn(`Failed to fetch from ${url}, status: ${res.status}`);
+            }
 
-      const res = await fetch(url, {
-        timeout: 2000,
-        onRetry(cause) {
-          chainLogger.debug({
-            msg: "Retrying IPFS request",
-            url: url,
-            err: cause,
-          });
-        },
-        retry: { retries: 3, minTimeout: 2000, maxTimeout: 60 * 10000 },
-        // IPFS data is immutable, we can rely entirely on the cache when present
-        cache: "force-cache",
-        cachePath:
-          config.cacheDir !== null
-            ? path.join(config.cacheDir, "ipfs")
-            : undefined,
-      });
+          } catch (err) {
+            chainLogger.warn(`Attempt ${attempt} failed for ${url}: ${err}`);
+          }
+        }
+      };
+
+      // Iterate through each gateway and attempt to fetch data
+      for (const gateway of config.ipfsGateways) {
+        const url = `${gateway}/ipfs/${cid}`;
+        chainLogger.info(`Trying IPFS gateway: ${url}`);
+
+        const result = await fetchFromGateway(url);
+        if (result !== undefined) {
+          return result; // Return the result if fetched successfully
+        }
+      }
 
-      return (await res.json()) as T;
+      chainLogger.error(`Failed to fetch IPFS data for CID ${cid} from all gateways.`);
+      return undefined; // Return undefined if all gateways fail
     };
 
     chainLogger.info("DEBUG: catching up with blockchain events");