[Indexer] Integrate pocketdex into Tiltfile

Tiltfile

@@ -55,9 +55,21 @@ localnet_config_defaults = {
     "rest": {
         "enabled": True,
     },
+
+    # TODO_CONSIDERATION: Consider using git submodules for all repos where we
+    # are dependent on a sibling directory.
+
     # By default, we use the `helm_repo` function below to point to the remote repository
     # but can update it to the locally cloned repo for testing & development
-    "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"},
+    "helm_chart_local_repo": {
+        "enabled": False,
+        "path": os.path.join("..", "helm-charts")
+    },
+    "indexer": {
+        "repo_path": os.path.join("..", "pocketdex"),
+        "enabled": True,
+        "clone_if_not_present": False,
+    }
 }
 localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults)
 # Initial empty config
@@ -75,14 +87,15 @@ if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_con
 
 # Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo.
 helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/")
+# TODO_IMPROVE: Use os.path.join to make this more OS-agnostic.
 chart_prefix = "pokt-network/"
 if localnet_config["helm_chart_local_repo"]["enabled"]:
     helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"]
     hot_reload_dirs.append(helm_chart_local_repo)
     print("Using local helm chart repo " + helm_chart_local_repo)
+    # TODO_IMPROVE: Use os.path.join to make this more OS-agnostic.
     chart_prefix = helm_chart_local_repo + "/charts/"
 
-
 # Observability
 print("Observability enabled: " + str(localnet_config["observability"]["enabled"]))
 if localnet_config["observability"]["enabled"]:
@@ -383,3 +396,114 @@ if localnet_config["rest"]["enabled"]:
     print("REST enabled: " + str(localnet_config["rest"]["enabled"]))
     deployment_create("rest", image="davarski/go-rest-api-demo")
     k8s_resource("rest", labels=["data_nodes"], port_forwards=["10000"])
+
+### Pocketdex Shannon Indexer
+pocketdex_root_path = localnet_config["indexer"]["repo_path"]
+pocketdex_tilt_path = os.path.join(pocketdex_root_path, "tiltfiles", "pocketdex.tilt")
+postgres_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-postgres.yaml")
+pgadmin_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-pgadmin.yaml")
+indexer_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-indexer.yaml")
+gql_engine_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-gql-engine.yaml")
+
+
+# pocketdex_repo_exists returns true if the sibling pocketdex repo exists.
+def pocketdex_repo_exists():
+    return str(
+        local("[ -d {} ] && echo 'true' || echo 'false'".format(pocketdex_root_path),
+              echo_off=True)
+    ).strip()
+
+
+# pocketdex_disabled_resource creates a tilt resource that prints a message indicating
+# that the indexer is disabled and how to enable it.
+def pocketdex_disabled_resource(reason):
+    local_resource("⚠️ Indexer Disabled",
+                   "echo '{}'".format(reason),
+                   labels=["Pocketdex"])
+
+
+# fetch_pocketdex_main fetches the main branch from the remote from which the repo was cloned.
+def fetch_pocketdex_main():
+    local("git fetch {} main".format(main_branch_remote_name()),
+          dir=pocketdex_root_path,
+          echo_off=True)
+
+
+# main_branch_remote_name returns the name of the remote from which the repo was cloned.
+def main_branch_remote_name():
+    return str(
+        local("git rev-parse --abbrev-ref @{u} | cut -d '/' -f1",
+              dir=pocketdex_root_path,
+              echo_off=True),
+    ).strip()
+
+
+# pocketdex_changes returns a tuple of integers representing the number of local
+#  and remote changes, respectively.
+def pocketdex_changes():
+    all_changes = str(
+        local("git rev-list --left-right --count HEAD...{}/main".format(main_branch_remote_name()),
+              dir=pocketdex_root_path,
+              echo_off=True)
+    )
+
+    # Split the output into local and remote changes and convert to integers.
+    num_local_changes, num_remote_changes = [int(x) for x in all_changes.split()]
+
+    return (num_local_changes, num_remote_changes)
+
+
+# pocketdex_main_is_outdated returns true if there's a diff between the local and remote main branches.
+def pocketdex_main_is_outdated():
+    fetch_pocketdex_main()
+    _, num_remote_changes = pocketdex_changes()
+    return num_remote_changes != 0
+
+
+# pocketdex_outdated_resource creates a tilt resource that prints a message indicating
+# that the indexer is outdated and how many commits behind it is.
+def pocketdex_outdated_resource():
+    _, num_remote_changes = pocketdex_changes()
+    local_resource("🔄 Updates Available",
+                   """
+                   echo 'Pocketdex main branch is outdated; {} commits behind. Please `git pull --ff-only` to update pocketdex.'
+                   """.format(num_remote_changes),
+                   labels=["Pocketdex"])
+
+
+def load_pocketdex():
+    if pocketdex_main_is_outdated():
+        pocketdex_outdated_resource()
+
+    pocketdex_tilt = load_dynamic(pocketdex_tilt_path)
+    pocketdex_tilt["pocketdex"](pocketdex_root_path,
+                                genesis_file_name="localnet.json",
+                                postgres_values_path=postgres_values_path,
+                                pgadmin_values_path=pgadmin_values_path,
+                                indexer_values_path=indexer_values_path,
+                                gql_engine_values_path=gql_engine_values_path)
+
+# Check if sibling pocketdex repo exists.
+# If it does, load the pocketdex.tilt file from the sibling repo.
+# Otherwise check the `indexer.clone_if_not_present` flag in `localnet_config.yaml` and EITHER:
+#   1. clone pocketdex to ../pocketdex
+#   -- OR --
+#   2. Prints a message if true or false
+if localnet_config["indexer"]["enabled"]:
+    pocketdex_repo_exists = local("[ -d {} ] && echo 'true' || echo 'false'".format(pocketdex_root_path))
+    if pocketdex_repo_exists == "false":
+        if localnet_config["indexer"]["clone_if_not_present"]:
+            print("Cloning pocketdex repo")
+            # TODO_INVESTIGATE: https://github.com/tilt-dev/tilt-extensions/tree/master/git_resource
+            # TODO_IN_THIS_PR: Remove "--branch' after merging in https://github.com/pokt-network/pocketdex/pull/23/files
+            local("git clone https://github.com/pokt-network/pocketdex --branch chore/tilt ../pocketdex")
+            load_pocketdex()
+        else:
+            pocketdex_disabled_resource(
+                "Pocketdex repo not found at ../pocketdex. Set `clone_if_not_present` to `true` in `localnet_config.yaml`.".format())
+    else:
+        print("Using existing pocketdex repo")
+        load_pocketdex()
+else:
+    pocketdex_disabled_resource(
+        "Pocketdex indexer disabled. Set `indexer.enabled` to `true` in `localnet_config.yaml` to enable it.")

docusaurus/docs/develop/contributing/_category_.json

@@ -1,6 +1,6 @@
 {
   "label": "Contributing",
-  "position": 9,
+  "position": 10,
   "link": {
     "type": "generated-index",
     "description": "Guidelines on how to contribute to the poktroll repo."

docusaurus/docs/develop/localnet/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "LocalNet",
+  "position": 9,
+  "link": {
+    "type": "generated-index",
+    "description": "Tips on how to leverage the most out of your LocalNet Environment."
+  }
+}

docusaurus/docs/develop/localnet/observability.md

@@ -0,0 +1,54 @@
+---
+sidebar_position: 1
+title: Pocketdex Indexer
+---
+
+- [Usage](#usage)
+  - [GraphQL](#graphql)
+  - [Postgres - CLI](#postgres---cli)
+- [Debugging](#debugging)
+  - [Port already in use](#port-already-in-use)
+
+:::warning
+
+This document is a living WIP and assumes you are familiar with the LocalNet environment.
+
+:::
+
+## Pocketdex <!-- omit in toc -->
+
+[Pocketdex](https://github.com/pokt-network/pocketdex/), the poktroll indexer starts up as part of the default LocalNet.
+
+### Usage
+
+#### GraphQL
+
+#### Postgres - CLI
+
+You can connect using a tool of your choice or with the CLI via:
+
+```bash
+psql -h localhost -p 5432 -U postgres -d postgres
+```
+
+### Debugging
+
+#### Port already in use
+
+If you go to [http://localhost:10350/r/Postgres/overview] and see the following error:
+
+```bash
+Reconnecting... Error port-forwarding Postgres (5432 -> 5432): Unable to listen on port 5432: Listeners failed to create with the following errors: [unable to create listener: Error listen tcp4 127.0.0.1:5432: bind: address already in use unable to create listener: Error listen tcp6 [::1]:5432: bind: address already in use]
+```
+
+You likely have another local Postgres instance running. You can identify it by running
+
+```bash
+lsof -i:5432
+```
+
+On macOS, if installed via `brew`, it can be stopped with:
+
+```bash
+brew services stop postgresql
+```

docusaurus/yarn.lock

@@ -1810,15 +1810,10 @@
   dependencies:
     "@types/mdx" "^2.0.0"
 
-"@node-rs/jieba-linux-x64-gnu@1.10.0":
+"@node-rs/jieba-darwin-arm64@1.10.0":
   version "1.10.0"
-  resolved "https://registry.npmjs.org/@node-rs/jieba-linux-x64-gnu/-/jieba-linux-x64-gnu-1.10.0.tgz"
-  integrity sha512-rS5Shs8JITxJjFIjoIZ5a9O+GO21TJgKu03g2qwFE3QaN5ZOvXtz+/AqqyfT4GmmMhCujD83AGqfOGXDmItF9w==
-
-"@node-rs/[email protected]":
-  version "1.10.0"
-  resolved "https://registry.npmjs.org/@node-rs/jieba-linux-x64-musl/-/jieba-linux-x64-musl-1.10.0.tgz"
-  integrity sha512-BvSiF2rR8Birh2oEVHcYwq0WGC1cegkEdddWsPrrSmpKmukJE2zyjcxaOOggq2apb8fIRsjyeeUh6X3R5AgjvA==
+  resolved "https://registry.npmjs.org/@node-rs/jieba-darwin-arm64/-/jieba-darwin-arm64-1.10.0.tgz"
+  integrity sha512-IhR5r+XxFcfhVsF93zQ3uCJy8ndotRntXzoW/JCyKqOahUo/ITQRT6vTKHKMyD9xNmjl222OZonBSo2+mlI2fQ==
 
 "@node-rs/jieba@^1.6.0":
   version "1.10.0"
@@ -4619,6 +4614,11 @@ fs.realpath@^1.0.0:
   resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
+fsevents@~2.3.2:
+  version "2.3.3"
+  resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz"
+  integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
+
 function-bind@^1.1.2:
   version "1.1.2"
   resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz"

go.mod

@@ -82,7 +82,6 @@ require (
 require (
 	cosmossdk.io/x/tx v0.13.4
 	github.com/jhump/protoreflect v1.16.0
-	go.uber.org/mock v0.4.0
 )
 
 require (

go.sum

@@ -1213,8 +1213,6 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
-go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

localnet/kubernetes/values-pocketdex-gql-engine.yaml

@@ -0,0 +1,24 @@
+enable_graphiql_playground: true
+indexer_endpoint: "http://indexer-service:3000"
+port: 3000
+env:
+  NODE_ENV: development
+  # LocalNet
+  ENDPOINT: "http://validator-poktroll-validator:26657"
+  CHAIN_ID: "poktroll"
+  # db schema name
+  DB_SCHEMA: "localnet"
+
+  START_BLOCK: 1
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-cli
+  SUBQUERY_NODE_EXTRA_PARAMS: "--unfinalized-blocks=true"
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-query
+  SUBQUERY_GRAPHQL_EXTRA_PARAMS: ""
+
+  DB_HOST: postgres-service
+  DB_PORT: 5432
+  DB_USER: postgres
+  DB_PASS: postgres
+  DB_DATABASE: postgres

localnet/kubernetes/values-pocketdex-indexer.yaml

@@ -0,0 +1,24 @@
+port: 3000
+env:
+  NODE_ENV: development
+  # LocalNet
+  ENDPOINT: "http://validator-poktroll-validator:26657"
+  CHAIN_ID: "poktroll"
+  # db schema name
+  DB_SCHEMA: "localnet"
+
+  START_BLOCK: 1
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-cli
+  SUBQUERY_NODE_EXTRA_PARAMS: "--unfinalized-blocks=true"
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-query
+  SUBQUERY_GRAPHQL_EXTRA_PARAMS: ""
+
+  SUBQUERY_GRAPHQL_ENGINE_VERSION: "latest"
+
+  DB_HOST: postgres-service
+  DB_PORT: 5432
+  DB_USER: postgres
+  DB_PASS: postgres
+  DB_DATABASE: postgres

localnet/kubernetes/values-pocketdex-pgadmin.yaml

@@ -0,0 +1,14 @@
+env:
+  # Pgadmin4
+  # DEV_NOTE: if you modify POSTGRES_USER/PASSWORD/DB then auto-imported server will not match and you need to setup
+  # your own with the modified values on the pgadmin website after first login.
+  PGADMIN_VERSION: 8.11
+  PGADMIN_DEFAULT_EMAIL: [email protected]
+  PGADMIN_DEFAULT_PASSWORD: admin
+  PGADMIN_LISTEN_PORT: 5050
+
+  DB_HOST: postgres-service
+  DB_PORT: 5432
+  DB_USER: postgres
+  DB_PASS: postgres
+  DB_DATABASE: postgres

localnet/kubernetes/values-pocketdex-postgres.yaml

@@ -0,0 +1,4 @@
+env:
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+  POSTGRES_DB: postgres