Merge branch 'main' into datadog

.github/workflows/deploy-do-indexer.yml

@@ -0,0 +1,35 @@
+name: Deploy DO Indexer
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install dependencies
+        run: npm ci
+      - name: Test build
+        run: npm run build
+  deploy-indexer:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Deploy to production
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.DO_INDEXER_IP }}
+          username: ${{ secrets.DO_USER }}
+          key: ${{ secrets.DO_SSH_KEY }}
+          script: |
+            export N_PREFIX="$HOME/n"; [[ :$PATH: == *":$N_PREFIX/bin:"* ]] || PATH+=":$N_PREFIX/bin"
+            cd grants-stack-indexer
+            git fetch origin main
+            git reset --hard origin/main
+            npm install && npm run build
+            pm2 reload indexer

.github/workflows/deploy-do-web.yml

@@ -0,0 +1,24 @@
+name: Deploy DO Web
+
+on: workflow_dispatch
+
+env:
+  FILENAME: Dockerfile.web
+  IMAGE_NAME: gitcoinco/indexer-web
+  IMAGE_TAG: ${{ github.sha }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build the Docker image
+        run: docker build -f "$FILENAME" -t "$IMAGE_NAME:$IMAGE_TAG" . # build the Docker image using envs defined above
+
+      # login to dockerhub then push the image to the dockerhub repo
+      - name: Push Docker image
+        run: |-
+          echo ${{secrets.DOCKERHUB_PASS}} | docker login -u ${{secrets.DOCKERHUB_USER}} --password-stdin
+          docker push "$IMAGE_NAME:$IMAGE_TAG"

Dockerfile.index

@@ -0,0 +1,24 @@
+FROM node:20-alpine
+
+WORKDIR /app
+
+# Bundle app source
+COPY src src
+COPY tsconfig.json ./
+COPY vite.config.ts ./
+COPY .eslintrc.cjs ./
+COPY .prettierrc.json ./
+COPY package*.json ./
+
+RUN npm ci
+RUN npm run lint
+
+RUN npm run build
+
+RUN npm run test
+
+
+EXPOSE 8080
+
+
+CMD [ "npm", "start", "--", "--indexer", "--http" ]

Dockerfile.web

@@ -0,0 +1,24 @@
+FROM node:20-alpine
+
+WORKDIR /app
+
+# Bundle app source
+COPY src src
+COPY tsconfig.json ./
+COPY vite.config.ts ./
+COPY .eslintrc.cjs ./
+COPY .prettierrc.json ./
+COPY package*.json ./
+
+RUN npm ci
+RUN npm run lint
+
+RUN npm run build
+
+RUN npm run test
+
+
+EXPOSE 8080
+
+
+CMD [ "npm", "start", "--", "--http", "--http-wait-for-sync=false" ]

READ_DOCKER_KB.md

@@ -0,0 +1,70 @@
+We followed this tutorial: https://www.digitalocean.com/community/tutorials/deploying-an-express-application-on-a-kubernetes-cluster
+
+- To build the docker images:
+
+  ```
+  docker build -f Dockerfile.index -t indexer/index .
+  docker build -f Dockerfile.web -t indexer/web .
+  ```
+
+- To push the images to docker hub:
+
+  ```
+  docker tag indexer/index:latest gitcoinco/indexer-index:latest
+  docker push gitcoinco/indexer-index:latest
+
+  docker tag indexer/web:latest gitcoinco/indexer-web:latest
+  docker push gitcoinco/indexer-web:latest
+  ```
+
+### SOLUTION FOR ENVIRONMENT VARIABLES (kubernetes doesn't support .env file):
+
+- Created a config map:
+
+  ```
+  kubectl create configmap indexer-web-config --from-env-file=.env
+  ```
+
+- Reference the config map in kb-deployment.yml with:
+
+  ```
+  envFrom:
+    - configMapRef:
+        name: indexer-web-config
+  ```
+
+- NOT DONE - For sensitive data we could have creaded a secret with:
+
+  ```
+  kubectl create secret generic indexer-web-secrets --from-env-file=.env
+  ```
+
+- NOT DONE - And referenced the secret in kb-deployment.yml with:
+
+  ```
+  envFrom:
+    - secretRef:
+        name: indexer-web-secrets
+  ```
+
+### Logs
+
+Get the current instances of server:
+
+```
+kubectl get pods
+```
+
+Possible response:
+
+```
+NAME                                      READY   STATUS    RESTARTS   AGE
+indexer-web-554574455d-lg98l   1/1        Running              0          41m
+indexer-web-554574455d-t4tgj   1/1        Running               0          41m
+```
+
+Then to see the logs of the first instance:
+
+```
+kubectl logs indexer-web-554574455d-lg98l
+```

ecosystem.config.cjs

@@ -0,0 +1,9 @@
+module.exports = {
+  apps: [
+    {
+      name: "indexer",
+      script: "npm",
+      args: "start -- --indexer --http",
+    },
+  ],
+};

indexer-compose.yml

@@ -0,0 +1,98 @@
+version: "3.8"
+
+services:
+  web:
+    build:
+      context: .
+      dockerfile: Dockerfile.web
+    ports:
+      - "8081:8080" # Map the port your web server uses
+    depends_on:
+      - db
+    environment:
+      INDEXED_CHAINS: ${INDEXED_CHAINS}
+      PASSPORT_SCORER_ID: ${PASSPORT_SCORER_ID}
+      STORAGE_DIR: ${STORAGE_DIR}
+      DEPLOYMENT_ENVIRONMENT: ${DEPLOYMENT_ENVIRONMENT}
+      PORT: ${PORT}
+      LOG_LEVEL: ${LOG_LEVEL}
+      BUILD_TAG: ${BUILD_TAG}
+      ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
+      ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
+      PINO_PRETTY: ${PINO_PRETTY}
+      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      COINGECKO_API_KEY: ${COINGECKO_API_KEY}
+      GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
+      SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}
+      POLYGON_MUMBAI_RPC_URL: ${POLYGON_MUMBAI_RPC_URL}
+      AVALANCHE_RPC_URL: ${AVALANCHE_RPC_URL}
+      OPTIMISM_RPC_URL: ${OPTIMISM_RPC_URL}
+      SENTRY_DSN: ${SENTRY_DSN}
+      PGN_TESTNET_RPC_URL: ${PGN_TESTNET_RPC_URL}
+      ARBITRUM_GOERLI_RPC_URL: ${ARBITRUM_GOERLI_RPC_URL}
+      FANTOM_RPC_URL: ${FANTOM_RPC_URL}
+      BASE_RPC_URL: ${BASE_RPC_URL}
+      PGN_RPC_URL: ${PGN_RPC_URL}
+      GOERLI_RPC_URL: ${GOERLI_RPC_URL}
+      AVALANCHE_FUJI_RPC_URL: ${AVALANCHE_FUJI_RPC_URL}
+      ARBITRUM_RPC_URL: ${ARBITRUM_RPC_URL}
+      SEI_MAINNET_RPC_URL: ${SEI_MAINNET_RPC_URL}
+      MAINNET_RPC_URL: ${MAINNET_RPC_URL}
+      POLYGON_RPC_URL: ${POLYGON_RPC_URL}
+      METIS_ANDROMEDA_RPC_URL: ${METIS_ANDROMEDA_RPC_URL}
+      SCROLL_SEPOLIA_RPC_URL: ${SCROLL_SEPOLIA_RPC_URL}
+      DATABASE_URL: "postgresql://postgres:postgres@db:5432/grants_stack_indexer"
+
+  index:
+    build:
+      context: .
+      dockerfile: Dockerfile.index
+    ports:
+      - "8080:8080" # Map the port your web server uses
+    depends_on:
+      - db
+    environment:
+      INDEXED_CHAINS: ${INDEXED_CHAINS}
+      PASSPORT_SCORER_ID: ${PASSPORT_SCORER_ID}
+      STORAGE_DIR: ${STORAGE_DIR}
+      DEPLOYMENT_ENVIRONMENT: ${DEPLOYMENT_ENVIRONMENT}
+      PORT: ${PORT}
+      LOG_LEVEL: ${LOG_LEVEL}
+      BUILD_TAG: ${BUILD_TAG}
+      ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
+      ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
+      PINO_PRETTY: ${PINO_PRETTY}
+      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      COINGECKO_API_KEY: ${COINGECKO_API_KEY}
+      GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
+      SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}
+      POLYGON_MUMBAI_RPC_URL: ${POLYGON_MUMBAI_RPC_URL}
+      AVALANCHE_RPC_URL: ${AVALANCHE_RPC_URL}
+      OPTIMISM_RPC_URL: ${OPTIMISM_RPC_URL}
+      SENTRY_DSN: ${SENTRY_DSN}
+      PGN_TESTNET_RPC_URL: ${PGN_TESTNET_RPC_URL}
+      ARBITRUM_GOERLI_RPC_URL: ${ARBITRUM_GOERLI_RPC_URL}
+      FANTOM_RPC_URL: ${FANTOM_RPC_URL}
+      BASE_RPC_URL: ${BASE_RPC_URL}
+      PGN_RPC_URL: ${PGN_RPC_URL}
+      GOERLI_RPC_URL: ${GOERLI_RPC_URL}
+      AVALANCHE_FUJI_RPC_URL: ${AVALANCHE_FUJI_RPC_URL}
+      ARBITRUM_RPC_URL: ${ARBITRUM_RPC_URL}
+      SEI_MAINNET_RPC_URL: ${SEI_MAINNET_RPC_URL}
+      MAINNET_RPC_URL: ${MAINNET_RPC_URL}
+      POLYGON_RPC_URL: ${POLYGON_RPC_URL}
+      METIS_ANDROMEDA_RPC_URL: ${METIS_ANDROMEDA_RPC_URL}
+      SCROLL_SEPOLIA_RPC_URL: ${SCROLL_SEPOLIA_RPC_URL}
+      DATABASE_URL: "postgresql://postgres:postgres@db:5432/grants_stack_indexer"
+
+  db:
+    image: postgres:15 # Use the version of PostgreSQL you need
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: grants_stack_indexer
+    volumes:
+      - db_data:/var/lib/postgresql/data
+
+volumes:
+  db_data:

kb-deployment.yaml

@@ -0,0 +1,22 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: indexer-web
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: indexer-web
+  template:
+    metadata:
+      labels:
+        app: indexer-web
+    spec:
+      containers:
+        - name: indexer-web
+          image: "gitcoinco/indexer-web:latest"
+          ports:
+            - containerPort: 80
+          envFrom:
+            - configMapRef:
+                name: indexer-web-config

kb-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: load-balancer
+  labels:
+    app: indexer-web
+spec:
+  type: LoadBalancer
+  ports:
+    - name: http
+      port: 80
+      targetPort: 8080
+    - name: https
+      port: 443
+      targetPort: 8080
+  selector:
+    app: indexer-web

src/config.ts

@@ -20,7 +20,7 @@ type CoingeckoSupportedChainId =
   | 42220
   | 1088;
 
-const CHAIN_DATA_VERSION = "75";
+const CHAIN_DATA_VERSION = "79";
 
 export type Token = {
   code: string;
@@ -94,6 +94,15 @@ const CHAINS: Chain[] = [
           address: "0x0000000000000000000000000000000000000000",
         },
       },
+      {
+        code: "eBTC",
+        address: "0x661c70333aa1850ccdbae82776bb436a0fcfeefb",
+        decimals: 18,
+        priceSource: {
+          chainId: 1,
+          address: "0x661c70333aa1850ccdbae82776bb436a0fcfeefb",
+        },
+      },
     ],
     subscriptions: [
       {
@@ -1342,6 +1351,7 @@ const CHAINS: Chain[] = [
       .default("https://rpc.scroll.io")
       .parse(process.env.SCROLL_RPC_URL),
     pricesFromTimestamp: Date.UTC(2024, 0, 1, 0, 0, 0),
+    maxGetLogsRange: 9000,
     tokens: [
       {
         code: "ETH",
@@ -1748,7 +1758,7 @@ const CHAINS: Chain[] = [
       },
       {
         contractName: "AlloV2/Allo/V1",
-        address: "0xB087535DB0df98fC4327136e897A5985E5Cfbd66",
+        address: "0x1133eA7Af70876e64665ecD07C0A0476d09465a1",
         fromBlock: 17860000,
       },
     ],
@@ -1817,6 +1827,7 @@ export type Config = {
   sentryDsn: string | null;
   databaseUrl: string;
   readOnlyDatabaseUrl: string;
+  dataVersion: string;
   databaseSchemaName: string;
   hostname: string;
   pinoPretty: boolean;
@@ -1970,7 +1981,8 @@ export function getConfig(): Config {
     .default(databaseUrl)
     .parse(process.env.READ_ONLY_DATABASE_URL);
 
-  const databaseSchemaName = `chain_data_${CHAIN_DATA_VERSION}`;
+  const dataVersion = CHAIN_DATA_VERSION;
+  const databaseSchemaName = `chain_data_${dataVersion}`;
 
   const dropDb = z.boolean().default(false).parse(args["drop-db"]);
 
@@ -2019,6 +2031,7 @@ export function getConfig(): Config {
     databaseUrl,
     readOnlyDatabaseUrl,
     dropDb,
+    dataVersion,
     databaseSchemaName,
     httpServerWaitForSync,
     httpServerEnabled,