have fallback IPFS

docs/reindexing.md

@@ -12,9 +12,15 @@ When deploying changes to the indexer, it's important to clarify the results you
     - The indexer will create a new schema in Postgres named `chain_data_${version}`. If this schema does not exist, it will be created, all necessary tables will be set up, and indexing will start from scratch.
     - If the schema already exists, the indexer will resume indexing from the last indexed block unless the `--drop-db` flag is specified via the CLI. This will drop the existing database and start fresh.
 
-### Using `--drop-db` in Development
+### Dropping Schemas in Development
 
-- During development, you can use the `--drop-db` flag to ensure the indexer always deletes the existing schema and migrates from scratch. This can be useful for testing schema changes and event handler modifications without retaining old data.
+- During development, you can use the `--drop-db` flag to ensure the indexer always deletes all existing schema and migrates from scratch. This can be useful for testing schema changes and event handler modifications without retaining old data.
+
+- During development, you can use the `--drop-chain-db` flag to ensure the indexer always deletes chain schema and migrates from scratch. 
+
+- During development, you can use the `--drop-ipfs-db` flag to ensure the indexer always deletes ipfs schema and migrates from scratch. 
+
+- During development, you can use the `--drop-price-db` flag to ensure the indexer always deletes price schema and migrates from scratch. 
 
 ### Important Notes
 

indexer-compose.yml

@@ -20,7 +20,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}
@@ -62,7 +62,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}

src/config.ts

@@ -21,6 +21,8 @@ type CoingeckoSupportedChainId =
   | 1088;
 
 const CHAIN_DATA_VERSION = "81";
+const IPFS_DATA_VERSION = "1";
+const PRICE_DATA_VERSION = "1";
 
 export type Token = {
   code: string;
@@ -1487,7 +1489,7 @@ const CHAINS: Chain[] = [
       .default("https://evm-rpc.sei-apis.com")
       .parse(process.env.SEI_MAINNET_RPC_URL),
     pricesFromTimestamp: Date.UTC(2024, 0, 1, 0, 0, 0),
-    maxGetLogsRange: 10000,
+    maxGetLogsRange: 1000,
     tokens: [
       {
         code: "SEI",
@@ -1818,7 +1820,7 @@ export type Config = {
   httpServerWaitForSync: boolean;
   httpServerEnabled: boolean;
   indexerEnabled: boolean;
-  ipfsGateway: string;
+  ipfsGateways: string[];
   coingeckoApiKey: string | null;
   coingeckoApiUrl: string;
   chains: Chain[];
@@ -1829,11 +1831,18 @@ export type Config = {
   readOnlyDatabaseUrl: string;
   dataVersion: string;
   databaseSchemaName: string;
+  ipfsDataVersion: string;
+  ipfsDatabaseSchemaName: string;
+  priceDataVersion: string;
+  priceDatabaseSchemaName: string;
   hostname: string;
   pinoPretty: boolean;
   deploymentEnvironment: "local" | "development" | "staging" | "production";
   enableResourceMonitor: boolean;
   dropDb: boolean;
+  dropChainDb: boolean;
+  dropIpfsDb: boolean;
+  dropPriceDb: boolean;
   removeCache: boolean;
   estimatesLinearQfWorkerPoolSize: number | null;
 };
@@ -1901,6 +1910,15 @@ export function getConfig(): Config {
       "drop-db": {
         type: "boolean",
       },
+      "drop-chain-db": {
+        type: "boolean",
+      },
+      "drop-ipfs-db": {
+        type: "boolean",
+      },
+      "drop-price-db": {
+        type: "boolean",
+      },
       "rm-cache": {
         type: "boolean",
       },
@@ -1968,10 +1986,11 @@ export function getConfig(): Config {
 
   const runOnce = z.boolean().default(false).parse(args["run-once"]);
 
-  const ipfsGateway = z
+  const ipfsGateways = z
     .string()
-    .default("https://ipfs.io")
-    .parse(process.env.IPFS_GATEWAY);
+    .array()
+    .default(["https://ipfs.io"])
+    .parse(JSON.parse(process.env.IPFS_GATEWAYS!));
 
   const sentryDsn = z
     .union([z.string(), z.null()])
@@ -1988,7 +2007,16 @@ export function getConfig(): Config {
   const dataVersion = CHAIN_DATA_VERSION;
   const databaseSchemaName = `chain_data_${dataVersion}`;
 
+  const ipfsDataVersion = IPFS_DATA_VERSION;
+  const ipfsDatabaseSchemaName = `ipfs_data_${ipfsDataVersion}`;
+
+  const priceDataVersion = PRICE_DATA_VERSION;
+  const priceDatabaseSchemaName = `price_data_${priceDataVersion}`;
+
   const dropDb = z.boolean().default(false).parse(args["drop-db"]);
+  const dropChainDb = z.boolean().default(false).parse(args["drop-chain-db"]);
+  const dropIpfsDb = z.boolean().default(false).parse(args["drop-ipfs-db"]);
+  const dropPriceDb = z.boolean().default(false).parse(args["drop-price-db"]);
 
   const removeCache = z.boolean().default(false).parse(args["rm-cache"]);
 
@@ -2028,7 +2056,7 @@ export function getConfig(): Config {
     cacheDir,
     logLevel,
     runOnce,
-    ipfsGateway,
+    ipfsGateways,
     passportScorerId,
     apiHttpPort,
     pinoPretty,
@@ -2037,9 +2065,16 @@ export function getConfig(): Config {
     databaseUrl,
     readOnlyDatabaseUrl,
     dropDb,
+    dropChainDb,
+    dropIpfsDb,
+    dropPriceDb,
     removeCache,
     dataVersion,
     databaseSchemaName,
+    ipfsDataVersion,
+    ipfsDatabaseSchemaName,
+    priceDataVersion,
+    priceDatabaseSchemaName,
     httpServerWaitForSync,
     httpServerEnabled,
     indexerEnabled,

src/database/changeset.ts

@@ -16,6 +16,7 @@ import {
   NewPrice,
   NewLegacyProject,
   NewApplicationPayout,
+  NewIpfsData,
 } from "./schema.js";
 
 export type DataChange =
@@ -140,4 +141,8 @@ export type DataChange =
   | {
       type: "InsertApplicationPayout";
       payout: NewApplicationPayout;
+    }
+  | {
+      type: "InsertIpfsData";
+      ipfs: NewIpfsData;
     };