Initial Commit

.github/workflows/docker-publish.yml

@@ -0,0 +1,103 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  release:
+    types: ['published']
+  push:
+    branches: [ "main", "dockerify"]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: powerloom/submission-sequencer-batcher
+
+
+jobs:
+  build:
+
+    runs-on:
+      group: larger-runners
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      #- name: Install cosign
+      #  if: github.event_name != 'pull_request'
+      #  uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0
+      #  with:
+      #    cosign-release: 'v1.11.0'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      # - name: Sign the published Docker image
+      #   if: ${{ github.event_name != 'pull_request' }}
+      #   env:
+      #     COSIGN_EXPERIMENTAL: "true"
+      #   # This step uses the identity token to provision an ephemeral certificate
+      #   # against the sigstore community Fulcio instance.
+      #   run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}

Dockerfile

@@ -0,0 +1,29 @@
+# Use the official Golang image as the build environment
+FROM golang:1.20 as builder
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Copy go.mod and go.sum files to the working directory
+COPY go.mod go.sum ./
+
+# Download the dependencies
+RUN go mod download
+
+# Copy the rest of the application code to the working directory
+COPY . .
+
+# Build the Go application
+RUN CGO_ENABLED=0 GOOS=linux go build -o /sequencer ./cmd/main.go
+
+# Use a minimal base image
+FROM scratch
+
+# Copy the binary from the builder stage
+COPY --from=builder /sequencer /sequencer
+
+# Expose port 9988 for api server
+EXPOSE 9988
+
+# Command to run the application
+CMD ["/sequencer"]

README.md

@@ -0,0 +1,49 @@
+# Sequencer Deployment
+Scripts to deploy PowerLoom Sequencer
+
+## Requirements
+
+1. Latest version of `docker` (`>= 20.10.21`) and `docker-compose` (`>= v2.13.0`)
+2. At least 4 core CPU, 8GB RAM and 50GB SSD - make sure to choose the correct spec when deploying to Github Codespaces.
+3. IPFS node
+    - While we have __included__ a node in our autobuild docker setup, IPFS daemon can hog __*a lot*__ of resources - it is not recommended to run this on a personal computer unless you have a strong internet connection and dedicated CPU+RAM.
+    - 3rd party IPFS services that provide default IFPS interface like Infura are now supported.
+
+
+
+## Running the Sequencer Node
+
+Clone the repository against the testnet branch.
+
+`git clone https://github.com/PowerLoom/proto-snapshot-collector.git --single-branch powerloom_sequencer --branch lite_node_test && cd powerloom_sequencer`
+
+
+### Deployment steps
+
+1. Copy `env.example` to `.env`.
+    - Ensure the following required variables are filled:
+        - `SIGNER_ACCOUNT_ADDRESS`: The address of the signer account. This is your whitelisted address on the protocol. **Using a burner account is highly recommended**
+        - `SIGNER_ACCOUNT_PRIVATE_KEY`: The private key corresponding to the signer account address.
+        - `PROST_CHAIN_ID`: The chain ID for the PROST RPC service.
+        - `RENDEZVOUS_POINT`: The identifier for locating all relayer peers which are the only way to access the sequencer and submit snapshots.
+        - `PROTOCOL_STATE_CONTRACT`: The contract address for the protocol state.
+        - `PROST_RPC_URL`: The URL for the PROST RPC service.
+        - `IPFS_URL`: The URL for the IPFS (InterPlanetary File System) service in HTTP(s) (e.g. `https://ipfs.infura.io:5001`) multiaddr format (e.g. `/dns4/ipfs.infura.io/tcp/5001/https`)
+
+    - Optionally, you may also set the following variables:
+        - `BATCH_SIZE`: The number of submissions to be included per batch.
+        - `BLOCK_TIME`: The block time of the chain.
+        - `REDIS_HOST` & `REDIS_PORT`: The redis server connection url (if you wish to use a separate one).
+
+2. Build the image
+
+   `./build-docker.sh`
+
+3. Run the following command (ideally in a `screen`) and follow instructions
+
+   `./run.sh`
+
+## Troubleshooting
+### To be added
+### Stopping and Resetting
+1. To shutdown services, just press `Ctrl+C` (and again to force).

build-docker.sh

@@ -0,0 +1 @@
+docker build -t proto-snapshot-collector . --no-cache

build.sh

@@ -0,0 +1,4 @@
+#bash scripts/generate.sh
+cd cmd
+go build .
+#docker-compose -f docker-compose.yaml up

cmd/main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"collector/config"
+	"collector/pkgs/helpers/clients"
+	"collector/pkgs/helpers/ipfs"
+	"collector/pkgs/helpers/prost"
+	"collector/pkgs/helpers/redis"
+	"collector/pkgs/helpers/utils"
+	"collector/pkgs/service"
+	"sync"
+	"time"
+)
+
+func main() {
+	utils.InitLogger()
+	config.LoadConfig()
+
+	clients.InitializeReportingClient(config.SettingsObj.SlackReportingUrl, 5*time.Second)
+	clients.InitializeRewardsBackendClient(config.SettingsObj.RewardsBackendUrl, 5*time.Second)
+
+	var wg sync.WaitGroup
+
+	prost.ConfigureClient()
+	prost.ConfigureContractInstance()
+	redis.RedisClient = redis.NewRedisClient()
+	ipfs.ConnectIPFSNode()
+
+	prost.PopulateStateVars()
+	prost.InitializeTxManager()
+
+	wg.Add(1)
+	go service.StartApiServer()
+	go prost.StartFetchingBlocks()
+	wg.Wait()
+}

config/settings.go

@@ -0,0 +1,136 @@
+package config
+
+import (
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/common"
+	log "github.com/sirupsen/logrus"
+	"os"
+	"strconv"
+)
+
+var SettingsObj *Settings
+
+type Settings struct {
+	ClientUrl                 string
+	ContractAddress           string
+	RedisHost                 string
+	RedisPort                 string
+	IPFSUrl                   string
+	DataMarketAddress         string
+	DataMarketContractAddress common.Address
+	SignerAccountAddresses    []string
+	PrivateKeys               []string
+	AuthReadToken             string
+	AuthWriteToken            string
+	BatchSize                 int
+	ChainID                   int64
+	BlockTime                 int
+	SlackReportingUrl         string
+	RewardsBackendUrl         string
+}
+
+func LoadConfig() {
+	missingEnvVars := []string{}
+
+	requiredEnvVars := []string{
+		"PROST_RPC_URL",
+		"PROTOCOL_STATE_CONTRACT",
+		"REDIS_HOST",
+		"REDIS_PORT",
+		"IPFS_URL",
+		"DATA_MARKET_CONTRACT",
+		"AUTH_READ_TOKEN",
+		"AUTH_WRITE_TOKEN",
+		"SIGNER_ACCOUNT_ADDRESSES",
+		"SIGNER_ACCOUNT_PRIVATE_KEYS",
+		"BATCH_SIZE",
+		"PROST_CHAIN_ID",
+		"BLOCK_TIME",
+		"SLACK_REPORTING_URL",
+		"REWARDS_BACKEND_URL",
+	}
+
+	for envVar := range requiredEnvVars {
+		if getEnv(requiredEnvVars[envVar], "") == "" {
+			missingEnvVars = append(missingEnvVars, requiredEnvVars[envVar])
+		}
+	}
+
+	if len(missingEnvVars) > 0 {
+		log.Fatalf("Missing required environment variables: %v", missingEnvVars)
+	}
+
+	config := Settings{
+		ClientUrl:                 getEnv("PROST_RPC_URL", ""),
+		ContractAddress:           getEnv("PROTOCOL_STATE_CONTRACT", ""),
+		RedisHost:                 getEnv("REDIS_HOST", ""),
+		RedisPort:                 getEnv("REDIS_PORT", ""),
+		IPFSUrl:                   getEnv("IPFS_URL", ""),
+		DataMarketAddress:         getEnv("DATA_MARKET_CONTRACT", ""),
+		AuthReadToken:             getEnv("AUTH_READ_TOKEN", ""),
+		AuthWriteToken:            getEnv("AUTH_WRITE_TOKEN", ""),
+		SlackReportingUrl:         getEnv("SLACK_REPORTING_URL", ""),
+		RewardsBackendUrl:         getEnv("REWARDS_BACKEND_URL", ""),
+		DataMarketContractAddress: common.HexToAddress(getEnv("DATA_MARKET_ADDRESS", "")),
+	}
+
+	signerAddressesList := []string{}
+	signerAddressesListParseErr := json.Unmarshal(
+		[]byte(getEnv("SIGNER_ACCOUNT_ADDRESSES", "[]")),
+		&signerAddressesList,
+	)
+	if signerAddressesListParseErr != nil {
+		log.Fatalf(
+			"Failed to parse SIGNER_ACCOUNT_ADDRESSES environment variable: %v",
+			signerAddressesListParseErr,
+		)
+	}
+	config.SignerAccountAddresses = signerAddressesList
+
+	signerPrivateKeysList := []string{}
+	signerPrivateKeysListParseErr := json.Unmarshal(
+		[]byte(getEnv("SIGNER_ACCOUNT_PRIVATE_KEYS", "[]")),
+		&signerPrivateKeysList,
+	)
+	if signerPrivateKeysListParseErr != nil {
+		log.Fatalf(
+			"Failed to parse SIGNER_ACCOUNT_PRIVATE_KEYS environment variable: %v",
+			signerPrivateKeysListParseErr,
+		)
+	}
+	config.PrivateKeys = signerPrivateKeysList
+
+	chainId, chainIdParseErr := strconv.ParseInt(getEnv("PROST_CHAIN_ID", ""), 10, 64)
+	if chainIdParseErr != nil {
+		log.Fatalf("Failed to parse PROST_CHAIN_ID environment variable: %v", chainIdParseErr)
+	}
+	config.ChainID = chainId
+
+	batchSize, batchSizeParseErr := strconv.Atoi(getEnv("BATCH_SIZE", ""))
+	if batchSizeParseErr != nil {
+		log.Fatalf("Failed to parse BATCH_SIZE environment variable: %v", batchSizeParseErr)
+	}
+	config.BatchSize = batchSize
+
+	blockTime, blockTimeParseErr := strconv.Atoi(getEnv("BLOCK_TIME", ""))
+	if blockTimeParseErr != nil {
+		log.Fatalf("Failed to parse BLOCK_TIME environment variable: %v", blockTimeParseErr)
+	}
+	config.BlockTime = blockTime
+
+	SettingsObj = &config
+}
+
+func getEnv(key, defaultValue string) string {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+	return value
+}
+
+func checkOptionalEnvVar(value, key string) {
+	if value == "" {
+		log.Warnf("Optional environment variable %s is not set", key)
+	}
+}

env.example

@@ -0,0 +1,14 @@
+SOURCE_RPC_URL=
+SIGNER_ACCOUNT_ADDRESSES=
+SIGNER_ACCOUNT_PRIVATE_KEYS=
+PROST_RPC_URL=
+PROTOCOL_STATE_CONTRACT=
+DATA_MARKET_CONTRACT=
+AUTH_READ_TOKEN=
+PROST_CHAIN_ID=
+RELAYER_RENDEZVOUS_POINT=
+IPFS_URL=
+BATCH_SIZE=
+BLOCK_TIME=
+FULL_NODES=
+SLACK_REPORTING_URL=