From b115afff7878884ed236e2a7664deb71212ee2e6 Mon Sep 17 00:00:00 2001 From: 0xKurt Date: Mon, 23 Sep 2024 16:51:56 +0200 Subject: [PATCH] clone schema and clean chain prompt input changes script updates clear subscriptions add readme Update src/config.ts --- READ_REINDEX_SINGLE_CHAIN.md | 48 ++++++++++++ cloneSchemaAndCleanChain.sh | 138 +++++++++++++++++++++++++++++++++++ src/config.ts | 2 +- 3 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 READ_REINDEX_SINGLE_CHAIN.md create mode 100755 cloneSchemaAndCleanChain.sh diff --git a/READ_REINDEX_SINGLE_CHAIN.md b/READ_REINDEX_SINGLE_CHAIN.md new file mode 100644 index 00000000..731d19a1 --- /dev/null +++ b/READ_REINDEX_SINGLE_CHAIN.md @@ -0,0 +1,48 @@ +# Overview + +This guide provides instructions on how to clone a source PostgreSQL schema to a target schema, delete specific data for a given chain ID, and subsequently restart the indexer to reindex the removed chain. + +## Prerequisites + +1. **Node.js and npm installed** (to run the script and manage packages). +2. **PostgreSQL installed and configured**. + +## Configuration + +Before running the script, ensure you have the following configurations in place: + +**Database URL** in the `.env` file: + +``` +DATABASE_URL=postgres://user:password@localhost:5432/your_database?sslmode=no-verify +``` + +## Running the Script + +1. **Make the script executable**: + ```bash + chmod +x cloneSchemaAndCleanChain.sh + ``` + +2. **Run the script**: + ```bash + ./cloneSchemaAndCleanChain.sh + ``` + + You will be prompted to enter: + - Source schema number + - Target schema number + - Chain ID to delete data for + + Example: + ```bash + Enter source schema number: 1 + Enter target schema number: 2 + Enter chain ID to delete data for: 1329 + ``` + +## Reindexing the Chain + +- Open `config.ts` and modify the configuration for the chain data to point to the new target schema. + +- After the script has successfully cloned the schema and deleted the data for the specified chain ID, you can start the indexer as usual. The indexer will detect the missing data for the chain ID and begin reindexing it. \ No newline at end of file diff --git a/cloneSchemaAndCleanChain.sh b/cloneSchemaAndCleanChain.sh new file mode 100755 index 00000000..4414e6ee --- /dev/null +++ b/cloneSchemaAndCleanChain.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# ======================================== CONFIG ================================================= + +# Define source and target schemas and chain ID to delete data for +read -p "Enter source schema number: " SOURCE_SCHEMA_NUM +read -p "Enter target schema number: " TARGET_SCHEMA_NUM +read -p "Enter chain ID to delete data for: " CHAIN_ID + +SOURCE_SCHEMA="chain_data_${SOURCE_SCHEMA_NUM}" +TARGET_SCHEMA="chain_data_${TARGET_SCHEMA_NUM}" + +# Read the DATABASE_URL from the .env +# Example: DATABASE_URL=postgres://user:password@localhost:5432/grants_stack_indexer?sslmode=no-verify +source .env +DATABASE_URL=${DATABASE_URL:-} + +if [ -z "$DATABASE_URL" ]; then + echo "Error: DATABASE_URL is not set in the environment." + exit 1 +fi + +# Path to SSL certificates +SSL_DIR="./ssl" # Suggest to place certificates in ./ssl directory +SSL_ROOT_CERT="$SSL_DIR/ca-certificate.crt" +SSL_CERT="$SSL_DIR/client-cert.pem" +SSL_KEY="$SSL_DIR/client-key.pem" + +# ================================================================================================= + +# Extract components from DATABASE_URL using sed +USERNAME=$(echo $DATABASE_URL | sed -n 's#.*//\([^:]*\):.*#\1#p') +PASSWORD=$(echo $DATABASE_URL | sed -n 's#.*//[^:]*:\([^@]*\)@.*#\1#p') +HOST=$(echo $DATABASE_URL | sed -n 's#.*@\(.*\):[0-9]*/.*#\1#p') +PORT=$(echo $DATABASE_URL | sed -n 's#.*:\([0-9]*\)/.*#\1#p') +DB_NAME=$(echo $DATABASE_URL | sed -n 's#.*/\([^?]*\).*#\1#p') +SSL_MODE=$(echo $DATABASE_URL | sed -n 's#.*sslmode=\([^&]*\).*#\1#p') + +# Set PGPASSWORD environment variable for non-interactive password authentication +export PGPASSWORD=$PASSWORD + +# Confirm action +echo "You are about to clone schema '$SOURCE_SCHEMA' to '$TARGET_SCHEMA' and delete data for chain ID: $CHAIN_ID." +read -p "Do you want to continue? (y/n): " CONFIRM +if [ "$CONFIRM" != "y" ]; then + echo "Operation cancelled." + exit 1 +fi + +# Function to handle errors +handle_error() { + echo "Error occurred. Exiting." + unset PGPASSWORD + exit 1 +} + +# Trap errors and call handle_error function +trap 'handle_error' ERR + +# Check SSL certificates if SSL mode is required +if [ "$SSL_MODE" == "require" ]; then + echo "SSL mode is enabled. Checking for SSL certificates in $SSL_DIR..." + if [ ! -f "$SSL_ROOT_CERT" ]; then + echo "Missing $SSL_ROOT_CERT. Please place the CA certificate in the $SSL_DIR directory." + exit 1 + fi + if [ ! -f "$SSL_CERT" ]; then + echo "Missing $SSL_CERT. Please place the client certificate in the $SSL_DIR directory." + exit 1 + fi + if [ ! -f "$SSL_KEY" ]; then + echo "Missing $SSL_KEY. Please place the client key in the $SSL_DIR directory." + exit 1 + fi +fi + +# Connection string with SSL options if required +if [ "$SSL_MODE" == "require" ]; then + CONNECTION_STRING="sslmode=$SSL_MODE sslrootcert=$SSL_ROOT_CERT sslcert=$SSL_CERT sslkey=$SSL_KEY host=$HOST port=$PORT dbname=$DB_NAME user=$USERNAME password=$PASSWORD" +else + CONNECTION_STRING="host=$HOST port=$PORT dbname=$DB_NAME user=$USERNAME password=$PASSWORD" +fi + +# Check if target schema exists +SCHEMA_EXISTS=$(psql "$CONNECTION_STRING" -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name = '$TARGET_SCHEMA';") + +if [ "$SCHEMA_EXISTS" == "1" ]; then + echo "Error: Target schema '$TARGET_SCHEMA' already exists. Exiting." + unset PGPASSWORD + exit 1 +fi + +echo "Creating new target schema..." +psql "$CONNECTION_STRING" -c "CREATE SCHEMA $TARGET_SCHEMA;" + +echo "Cloning schema and data..." +pg_dump "$CONNECTION_STRING" -n $SOURCE_SCHEMA | sed "s/$SOURCE_SCHEMA/$TARGET_SCHEMA/g" | psql "$CONNECTION_STRING" + +# Deleting data for the specified chain ID +echo "Deleting data for chain ID: $CHAIN_ID" +psql "$CONNECTION_STRING" <