-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit c67f39c
Showing
42 changed files
with
16,500 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
name: Docker | ||
|
||
# This workflow uses actions that are not certified by GitHub. | ||
# They are provided by a third-party and are governed by | ||
# separate terms of service, privacy policy, and support | ||
# documentation. | ||
|
||
on: | ||
release: | ||
types: ['published'] | ||
push: | ||
branches: [ "main", "dockerify"] | ||
# Publish semver tags as releases. | ||
tags: [ 'v*.*.*' ] | ||
pull_request: | ||
branches: [ "main" ] | ||
|
||
env: | ||
# Use docker.io for Docker Hub if empty | ||
REGISTRY: ghcr.io | ||
# github.repository as <account>/<repo> | ||
IMAGE_NAME: powerloom/submission-sequencer-batcher | ||
|
||
|
||
jobs: | ||
build: | ||
|
||
runs-on: | ||
group: larger-runners | ||
permissions: | ||
contents: read | ||
packages: write | ||
# This is used to complete the identity challenge | ||
# with sigstore/fulcio when running outside of PRs. | ||
id-token: write | ||
|
||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v3 | ||
with: | ||
submodules: recursive | ||
|
||
# Install the cosign tool except on PR | ||
# https://github.com/sigstore/cosign-installer | ||
#- name: Install cosign | ||
# if: github.event_name != 'pull_request' | ||
# uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 | ||
# with: | ||
# cosign-release: 'v1.11.0' | ||
|
||
- name: Set up QEMU | ||
uses: docker/setup-qemu-action@v2 | ||
|
||
|
||
# Workaround: https://github.com/docker/build-push-action/issues/461 | ||
- name: Setup Docker buildx | ||
uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf | ||
|
||
# Login against a Docker registry except on PR | ||
# https://github.com/docker/login-action | ||
- name: Log into registry ${{ env.REGISTRY }} | ||
if: github.event_name != 'pull_request' | ||
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c | ||
with: | ||
registry: ${{ env.REGISTRY }} | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
# Extract metadata (tags, labels) for Docker | ||
# https://github.com/docker/metadata-action | ||
- name: Extract Docker metadata | ||
id: meta | ||
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 | ||
with: | ||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | ||
|
||
# Build and push Docker image with Buildx (don't push on PR) | ||
# https://github.com/docker/build-push-action | ||
- name: Build and push Docker image | ||
id: build-and-push | ||
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a | ||
with: | ||
context: . | ||
platforms: linux/amd64,linux/arm64 | ||
push: ${{ github.event_name != 'pull_request' }} | ||
tags: ${{ steps.meta.outputs.tags }} | ||
labels: ${{ steps.meta.outputs.labels }} | ||
cache-from: type=gha | ||
cache-to: type=gha,mode=max | ||
|
||
|
||
# Sign the resulting Docker image digest except on PRs. | ||
# This will only write to the public Rekor transparency log when the Docker | ||
# repository is public to avoid leaking data. If you would like to publish | ||
# transparency data even for private images, pass --force to cosign below. | ||
# https://github.com/sigstore/cosign | ||
# - name: Sign the published Docker image | ||
# if: ${{ github.event_name != 'pull_request' }} | ||
# env: | ||
# COSIGN_EXPERIMENTAL: "true" | ||
# # This step uses the identity token to provision an ephemeral certificate | ||
# # against the sigstore community Fulcio instance. | ||
# run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
# Use the official Golang image as the build environment | ||
FROM golang:1.20 as builder | ||
|
||
# Set the working directory inside the container | ||
WORKDIR /app | ||
|
||
# Copy go.mod and go.sum files to the working directory | ||
COPY go.mod go.sum ./ | ||
|
||
# Download the dependencies | ||
RUN go mod download | ||
|
||
# Copy the rest of the application code to the working directory | ||
COPY . . | ||
|
||
# Build the Go application | ||
RUN CGO_ENABLED=0 GOOS=linux go build -o /sequencer ./cmd/main.go | ||
|
||
# Use a minimal base image | ||
FROM scratch | ||
|
||
# Copy the binary from the builder stage | ||
COPY --from=builder /sequencer /sequencer | ||
|
||
# Expose port 9988 for api server | ||
EXPOSE 9988 | ||
|
||
# Command to run the application | ||
CMD ["/sequencer"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
# Sequencer Deployment | ||
Scripts to deploy PowerLoom Sequencer | ||
|
||
## Requirements | ||
|
||
1. Latest version of `docker` (`>= 20.10.21`) and `docker-compose` (`>= v2.13.0`) | ||
2. At least 4 core CPU, 8GB RAM and 50GB SSD - make sure to choose the correct spec when deploying to Github Codespaces. | ||
3. IPFS node | ||
- While we have __included__ a node in our autobuild docker setup, IPFS daemon can hog __*a lot*__ of resources - it is not recommended to run this on a personal computer unless you have a strong internet connection and dedicated CPU+RAM. | ||
- 3rd party IPFS services that provide default IFPS interface like Infura are now supported. | ||
|
||
|
||
|
||
## Running the Sequencer Node | ||
|
||
Clone the repository against the testnet branch. | ||
|
||
`git clone https://github.com/PowerLoom/proto-snapshot-collector.git --single-branch powerloom_sequencer --branch lite_node_test && cd powerloom_sequencer` | ||
|
||
|
||
### Deployment steps | ||
|
||
1. Copy `env.example` to `.env`. | ||
- Ensure the following required variables are filled: | ||
- `SIGNER_ACCOUNT_ADDRESS`: The address of the signer account. This is your whitelisted address on the protocol. **Using a burner account is highly recommended** | ||
- `SIGNER_ACCOUNT_PRIVATE_KEY`: The private key corresponding to the signer account address. | ||
- `PROST_CHAIN_ID`: The chain ID for the PROST RPC service. | ||
- `RENDEZVOUS_POINT`: The identifier for locating all relayer peers which are the only way to access the sequencer and submit snapshots. | ||
- `PROTOCOL_STATE_CONTRACT`: The contract address for the protocol state. | ||
- `PROST_RPC_URL`: The URL for the PROST RPC service. | ||
- `IPFS_URL`: The URL for the IPFS (InterPlanetary File System) service in HTTP(s) (e.g. `https://ipfs.infura.io:5001`) multiaddr format (e.g. `/dns4/ipfs.infura.io/tcp/5001/https`) | ||
|
||
- Optionally, you may also set the following variables: | ||
- `BATCH_SIZE`: The number of submissions to be included per batch. | ||
- `BLOCK_TIME`: The block time of the chain. | ||
- `REDIS_HOST` & `REDIS_PORT`: The redis server connection url (if you wish to use a separate one). | ||
|
||
2. Build the image | ||
|
||
`./build-docker.sh` | ||
|
||
3. Run the following command (ideally in a `screen`) and follow instructions | ||
|
||
`./run.sh` | ||
|
||
## Troubleshooting | ||
### To be added | ||
### Stopping and Resetting | ||
1. To shutdown services, just press `Ctrl+C` (and again to force). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
docker build -t proto-snapshot-collector . --no-cache |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
#bash scripts/generate.sh | ||
cd cmd | ||
go build . | ||
#docker-compose -f docker-compose.yaml up |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
package main | ||
|
||
import ( | ||
"collector/config" | ||
"collector/pkgs/helpers/clients" | ||
"collector/pkgs/helpers/ipfs" | ||
"collector/pkgs/helpers/prost" | ||
"collector/pkgs/helpers/redis" | ||
"collector/pkgs/helpers/utils" | ||
"collector/pkgs/service" | ||
"sync" | ||
"time" | ||
) | ||
|
||
func main() { | ||
utils.InitLogger() | ||
config.LoadConfig() | ||
|
||
clients.InitializeReportingClient(config.SettingsObj.SlackReportingUrl, 5*time.Second) | ||
clients.InitializeRewardsBackendClient(config.SettingsObj.RewardsBackendUrl, 5*time.Second) | ||
|
||
var wg sync.WaitGroup | ||
|
||
prost.ConfigureClient() | ||
prost.ConfigureContractInstance() | ||
redis.RedisClient = redis.NewRedisClient() | ||
ipfs.ConnectIPFSNode() | ||
|
||
prost.PopulateStateVars() | ||
prost.InitializeTxManager() | ||
|
||
wg.Add(1) | ||
go service.StartApiServer() | ||
go prost.StartFetchingBlocks() | ||
wg.Wait() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,136 @@ | ||
package config | ||
|
||
import ( | ||
"encoding/json" | ||
"github.com/ethereum/go-ethereum/common" | ||
log "github.com/sirupsen/logrus" | ||
"os" | ||
"strconv" | ||
) | ||
|
||
var SettingsObj *Settings | ||
|
||
type Settings struct { | ||
ClientUrl string | ||
ContractAddress string | ||
RedisHost string | ||
RedisPort string | ||
IPFSUrl string | ||
DataMarketAddress string | ||
DataMarketContractAddress common.Address | ||
SignerAccountAddresses []string | ||
PrivateKeys []string | ||
AuthReadToken string | ||
AuthWriteToken string | ||
BatchSize int | ||
ChainID int64 | ||
BlockTime int | ||
SlackReportingUrl string | ||
RewardsBackendUrl string | ||
} | ||
|
||
func LoadConfig() { | ||
missingEnvVars := []string{} | ||
|
||
requiredEnvVars := []string{ | ||
"PROST_RPC_URL", | ||
"PROTOCOL_STATE_CONTRACT", | ||
"REDIS_HOST", | ||
"REDIS_PORT", | ||
"IPFS_URL", | ||
"DATA_MARKET_CONTRACT", | ||
"AUTH_READ_TOKEN", | ||
"AUTH_WRITE_TOKEN", | ||
"SIGNER_ACCOUNT_ADDRESSES", | ||
"SIGNER_ACCOUNT_PRIVATE_KEYS", | ||
"BATCH_SIZE", | ||
"PROST_CHAIN_ID", | ||
"BLOCK_TIME", | ||
"SLACK_REPORTING_URL", | ||
"REWARDS_BACKEND_URL", | ||
} | ||
|
||
for envVar := range requiredEnvVars { | ||
if getEnv(requiredEnvVars[envVar], "") == "" { | ||
missingEnvVars = append(missingEnvVars, requiredEnvVars[envVar]) | ||
} | ||
} | ||
|
||
if len(missingEnvVars) > 0 { | ||
log.Fatalf("Missing required environment variables: %v", missingEnvVars) | ||
} | ||
|
||
config := Settings{ | ||
ClientUrl: getEnv("PROST_RPC_URL", ""), | ||
ContractAddress: getEnv("PROTOCOL_STATE_CONTRACT", ""), | ||
RedisHost: getEnv("REDIS_HOST", ""), | ||
RedisPort: getEnv("REDIS_PORT", ""), | ||
IPFSUrl: getEnv("IPFS_URL", ""), | ||
DataMarketAddress: getEnv("DATA_MARKET_CONTRACT", ""), | ||
AuthReadToken: getEnv("AUTH_READ_TOKEN", ""), | ||
AuthWriteToken: getEnv("AUTH_WRITE_TOKEN", ""), | ||
SlackReportingUrl: getEnv("SLACK_REPORTING_URL", ""), | ||
RewardsBackendUrl: getEnv("REWARDS_BACKEND_URL", ""), | ||
DataMarketContractAddress: common.HexToAddress(getEnv("DATA_MARKET_ADDRESS", "")), | ||
} | ||
|
||
signerAddressesList := []string{} | ||
signerAddressesListParseErr := json.Unmarshal( | ||
[]byte(getEnv("SIGNER_ACCOUNT_ADDRESSES", "[]")), | ||
&signerAddressesList, | ||
) | ||
if signerAddressesListParseErr != nil { | ||
log.Fatalf( | ||
"Failed to parse SIGNER_ACCOUNT_ADDRESSES environment variable: %v", | ||
signerAddressesListParseErr, | ||
) | ||
} | ||
config.SignerAccountAddresses = signerAddressesList | ||
|
||
signerPrivateKeysList := []string{} | ||
signerPrivateKeysListParseErr := json.Unmarshal( | ||
[]byte(getEnv("SIGNER_ACCOUNT_PRIVATE_KEYS", "[]")), | ||
&signerPrivateKeysList, | ||
) | ||
if signerPrivateKeysListParseErr != nil { | ||
log.Fatalf( | ||
"Failed to parse SIGNER_ACCOUNT_PRIVATE_KEYS environment variable: %v", | ||
signerPrivateKeysListParseErr, | ||
) | ||
} | ||
config.PrivateKeys = signerPrivateKeysList | ||
|
||
chainId, chainIdParseErr := strconv.ParseInt(getEnv("PROST_CHAIN_ID", ""), 10, 64) | ||
if chainIdParseErr != nil { | ||
log.Fatalf("Failed to parse PROST_CHAIN_ID environment variable: %v", chainIdParseErr) | ||
} | ||
config.ChainID = chainId | ||
|
||
batchSize, batchSizeParseErr := strconv.Atoi(getEnv("BATCH_SIZE", "")) | ||
if batchSizeParseErr != nil { | ||
log.Fatalf("Failed to parse BATCH_SIZE environment variable: %v", batchSizeParseErr) | ||
} | ||
config.BatchSize = batchSize | ||
|
||
blockTime, blockTimeParseErr := strconv.Atoi(getEnv("BLOCK_TIME", "")) | ||
if blockTimeParseErr != nil { | ||
log.Fatalf("Failed to parse BLOCK_TIME environment variable: %v", blockTimeParseErr) | ||
} | ||
config.BlockTime = blockTime | ||
|
||
SettingsObj = &config | ||
} | ||
|
||
func getEnv(key, defaultValue string) string { | ||
value := os.Getenv(key) | ||
if value == "" { | ||
return defaultValue | ||
} | ||
return value | ||
} | ||
|
||
func checkOptionalEnvVar(value, key string) { | ||
if value == "" { | ||
log.Warnf("Optional environment variable %s is not set", key) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
SOURCE_RPC_URL= | ||
SIGNER_ACCOUNT_ADDRESSES= | ||
SIGNER_ACCOUNT_PRIVATE_KEYS= | ||
PROST_RPC_URL= | ||
PROTOCOL_STATE_CONTRACT= | ||
DATA_MARKET_CONTRACT= | ||
AUTH_READ_TOKEN= | ||
PROST_CHAIN_ID= | ||
RELAYER_RENDEZVOUS_POINT= | ||
IPFS_URL= | ||
BATCH_SIZE= | ||
BLOCK_TIME= | ||
FULL_NODES= | ||
SLACK_REPORTING_URL= |
Oops, something went wrong.