diff --git a/Tiltfile b/Tiltfile index 8c339de7f..6b701bd8a 100644 --- a/Tiltfile +++ b/Tiltfile @@ -55,9 +55,20 @@ localnet_config_defaults = { "rest": { "enabled": True, }, + + # NOTE: git submodule usage was explicitly avoided to reduce environment complexity. + # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development - "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"}, + "helm_chart_local_repo": { + "enabled": False, + "path": os.path.join("..", "helm-charts") + }, + "indexer": { + "repo_path": os.path.join("..", "pocketdex"), + "enabled": True, + "clone_if_not_present": False, + } } localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults) # Initial empty config @@ -75,14 +86,15 @@ if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_con # Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo. helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/") +# TODO_IMPROVE: Use os.path.join to make this more OS-agnostic. chart_prefix = "pokt-network/" if localnet_config["helm_chart_local_repo"]["enabled"]: helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"] hot_reload_dirs.append(helm_chart_local_repo) print("Using local helm chart repo " + helm_chart_local_repo) + # TODO_IMPROVE: Use os.path.join to make this more OS-agnostic. chart_prefix = helm_chart_local_repo + "/charts/" - # Observability print("Observability enabled: " + str(localnet_config["observability"]["enabled"])) if localnet_config["observability"]["enabled"]: @@ -383,3 +395,14 @@ if localnet_config["rest"]["enabled"]: print("REST enabled: " + str(localnet_config["rest"]["enabled"])) deployment_create("rest", image="davarski/go-rest-api-demo") k8s_resource("rest", labels=["data_nodes"], port_forwards=["10000"]) + +### Pocketdex Shannon Indexer +load("./tiltfiles/pocketdex.tilt", "check_and_load_pocketdex") + +# Check if sibling pocketdex repo exists. +# If it does, load the pocketdex.tilt file from the sibling repo. +# Otherwise, check the `indexer.clone_if_not_present` flag in `localnet_config.yaml` and EITHER: +# 1. clone pocketdex to ../pocketdex +# -- OR -- +# 2. Prints a message if true or false +check_and_load_pocketdex(localnet_config["indexer"]) diff --git a/docusaurus/docs/develop/contributing/_category_.json b/docusaurus/docs/develop/contributing/_category_.json index a2849367a..54f7b6101 100644 --- a/docusaurus/docs/develop/contributing/_category_.json +++ b/docusaurus/docs/develop/contributing/_category_.json @@ -1,6 +1,6 @@ { "label": "Contributing", - "position": 9, + "position": 10, "link": { "type": "generated-index", "description": "Guidelines on how to contribute to the poktroll repo." diff --git a/docusaurus/docs/develop/localnet/_category_.json b/docusaurus/docs/develop/localnet/_category_.json new file mode 100644 index 000000000..eeb79d4a7 --- /dev/null +++ b/docusaurus/docs/develop/localnet/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "LocalNet", + "position": 9, + "link": { + "type": "generated-index", + "description": "Tips on how to leverage the most out of your LocalNet Environment." + } +} diff --git a/docusaurus/docs/develop/localnet/observability.md b/docusaurus/docs/develop/localnet/observability.md new file mode 100644 index 000000000..0879b009b --- /dev/null +++ b/docusaurus/docs/develop/localnet/observability.md @@ -0,0 +1,72 @@ +--- +sidebar_position: 1 +title: Pocketdex Indexer +--- + +- [Usage](#usage) + - [GraphQL](#graphql) + - [Postgres - CLI](#postgres---cli) +- [Debugging](#debugging) + - [Port already in use](#port-already-in-use) + +:::warning + +This document is a living WIP and assumes you are familiar with the LocalNet environment. + +::: + +## Pocketdex + +[Pocketdex](https://github.com/pokt-network/pocketdex/), the poktroll indexer starts up as part of the default LocalNet. + +### Usage + +#### GraphQL + +The localnet graphiql playground is available at [http://localhost:3000](http://localhost:3000), by default. + +![GraphiQL Playground](../../../static/img/pocketdex_graphiql_screenshot.png) + +A link is accessible from the ["GraphQL API" tab in tilt](http://localhost:10350/r/GraphQL%20API/overview): + +![LocalNet Dashboard](../../../static/img/pocketdex_graphiql_link.png) + +See the [pocketdex docs](https://github.com/pokt-network/pocketdex?tab=readme-ov-file#usage--query-docs) for more details. + +#### Postgres - CLI + +You can connect using a tool of your choice or with the `psql` CLI via: + +```bash +psql -h localhost -p 5432 -U postgres -d postgres +``` + +After you've connected, you MUST update your schema to `localnet` and start exploring the data: + +```sql +set schema 'localnet'; +\dt +select * from accounts limit 10; # Example query +``` + +### Debugging + +#### Port already in use + +If you go to [http://localhost:10350/r/Postgres/overview](http://localhost:10350/r/Postgres/overview) and see the following error: + +```bash +Reconnecting... Error port-forwarding Postgres (5432 -> 5432): Unable to listen on port 5432: Listeners failed to create with the following errors: [unable to create listener: Error listen tcp4 127.0.0.1:5432: bind: address already in use unable to create listener: Error listen tcp6 [::1]:5432: bind: address already in use] +``` + +You likely have another local Postgres instance running. You can identify it by running + +```bash +lsof -i:5432 +``` + +On macOS, if installed via `brew`, it can be stopped with: + +```bash +brew services stop postgresql +``` diff --git a/docusaurus/static/img/pocketdex_graphiql_link.png b/docusaurus/static/img/pocketdex_graphiql_link.png new file mode 100644 index 000000000..e17214f2f Binary files /dev/null and b/docusaurus/static/img/pocketdex_graphiql_link.png differ diff --git a/docusaurus/static/img/pocketdex_graphiql_screenshot.png b/docusaurus/static/img/pocketdex_graphiql_screenshot.png new file mode 100644 index 000000000..c84ca17b6 Binary files /dev/null and b/docusaurus/static/img/pocketdex_graphiql_screenshot.png differ diff --git a/localnet/kubernetes/values-pocketdex-gql-engine.yaml b/localnet/kubernetes/values-pocketdex-gql-engine.yaml new file mode 100644 index 000000000..2a7d81243 --- /dev/null +++ b/localnet/kubernetes/values-pocketdex-gql-engine.yaml @@ -0,0 +1,24 @@ +enable_graphiql_playground: true +indexer_endpoint: "http://indexer-service:3000" +port: 3000 +env: + NODE_ENV: development + # LocalNet + ENDPOINT: "http://validator-poktroll-validator:26657" + CHAIN_ID: "poktroll" + # db schema name + DB_SCHEMA: "localnet" + + START_BLOCK: 1 + # Check docs to see what else you can set here: + # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-cli + SUBQUERY_NODE_EXTRA_PARAMS: "--unfinalized-blocks=true" + # Check docs to see what else you can set here: + # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-query + SUBQUERY_GRAPHQL_EXTRA_PARAMS: "" + + DB_HOST: postgres-service + DB_PORT: 5432 + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres diff --git a/localnet/kubernetes/values-pocketdex-indexer.yaml b/localnet/kubernetes/values-pocketdex-indexer.yaml new file mode 100644 index 000000000..611a09e32 --- /dev/null +++ b/localnet/kubernetes/values-pocketdex-indexer.yaml @@ -0,0 +1,24 @@ +port: 3000 +env: + NODE_ENV: development + # LocalNet + ENDPOINT: "http://validator-poktroll-validator:26657" + CHAIN_ID: "poktroll" + # db schema name + DB_SCHEMA: "localnet" + + START_BLOCK: 1 + # Check docs to see what else you can set here: + # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-cli + SUBQUERY_NODE_EXTRA_PARAMS: "--unfinalized-blocks=true" + # Check docs to see what else you can set here: + # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-query + SUBQUERY_GRAPHQL_EXTRA_PARAMS: "" + + SUBQUERY_GRAPHQL_ENGINE_VERSION: "latest" + + DB_HOST: postgres-service + DB_PORT: 5432 + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres diff --git a/localnet/kubernetes/values-pocketdex-pgadmin.yaml b/localnet/kubernetes/values-pocketdex-pgadmin.yaml new file mode 100644 index 000000000..8c18ca35a --- /dev/null +++ b/localnet/kubernetes/values-pocketdex-pgadmin.yaml @@ -0,0 +1,14 @@ +env: + # Pgadmin4 + # DEV_NOTE: if you modify POSTGRES_USER/PASSWORD/DB then auto-imported server will not match and you need to setup + # your own with the modified values on the pgadmin website after first login. + PGADMIN_VERSION: 8.11 + PGADMIN_DEFAULT_EMAIL: admin@local.dev + PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_LISTEN_PORT: 5050 + + DB_HOST: postgres-service + DB_PORT: 5432 + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres diff --git a/localnet/kubernetes/values-pocketdex-postgres.yaml b/localnet/kubernetes/values-pocketdex-postgres.yaml new file mode 100644 index 000000000..76f5a11ea --- /dev/null +++ b/localnet/kubernetes/values-pocketdex-postgres.yaml @@ -0,0 +1,4 @@ +env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres diff --git a/tiltfiles/git.tilt b/tiltfiles/git.tilt new file mode 100644 index 000000000..08db67611 --- /dev/null +++ b/tiltfiles/git.tilt @@ -0,0 +1,46 @@ +# TODO_INVESTIGATE: https://github.com/tilt-dev/tilt-extensions/tree/master/git_resource + +# repo_remote_name returns the name of the remote from which the git repo was cloned. +def repo_remote_name(repo_root): + return str( + local("git rev-parse --abbrev-ref @{u} | cut -d '/' -f1", + dir=repo_root, + echo_off=True), + ).strip() + + +# fetch_repo_main fetches the main branch from the remote from which the git repo was cloned. +def fetch_repo_main(repo_root): + local("git fetch {} main".format(repo_remote_name(repo_root)), + dir=repo_root, + echo_off=True) + + +# repo_changes returns a tuple of integers representing the number of local +# and remote changes for the git repos main branch, respectively. +def repo_changes(repo_root): + all_changes = str( + local(""" + git rev-list --left-right --count HEAD...{}/main + """.format(repo_remote_name(repo_root)), + dir=repo_root, + echo_off=True) + ) + + # Split the output into local and remote changes and convert to integers. + num_local_changes, num_remote_changes = [int(x) for x in all_changes.split()] + + return (num_local_changes, num_remote_changes) + + +# repo_is_outdated returns true if there's a diff between the local and remote main branches. +def repo_is_outdated(repo_root): + fetch_repo_main(repo_root) + _, num_remote_changes = repo_changes(repo_root) + return num_remote_changes != 0 + + +# clone_repo clones the main branch from the repo at repo_url to local_path. +def clone_repo(repo_url, local_path): + print("Cloning pocketdex repo") + local("git clone {} --branch main {}".format(repo_url, local_path)) diff --git a/tiltfiles/pocketdex.tilt b/tiltfiles/pocketdex.tilt new file mode 100644 index 000000000..0b9c69c2d --- /dev/null +++ b/tiltfiles/pocketdex.tilt @@ -0,0 +1,68 @@ +load("./git.tilt", + "clone_repo", + "fetch_repo_main", + "repo_remote_name", + "repo_changes", + "repo_is_outdated") + + +# pocketdex_disabled_resource creates a tilt resource that prints a message indicating +# that the indexer is disabled and how to enable it. +def pocketdex_disabled_resource(reason): + local_resource("⚠️ Indexer Disabled", + "echo '{}'".format(reason), + labels=["Pocketdex"]) + + +# pocketdex_outdated_resource creates a tilt resource that prints a message indicating +# that the indexer is outdated and how many commits behind it is. +def pocketdex_outdated_resource(pocketdex_root_path): + _, num_remote_changes = repo_changes(pocketdex_root_path) + local_resource("🔄 Updates Available", + """ + echo 'Pocketdex main branch is outdated; {} commits behind. Please `git pull --ff-only` to update pocketdex.' + """.format(num_remote_changes), + labels=["Pocketdex"]) + + +# load_pocketdex loads the pocketdex.tilt file from the pocketdex repo at pocketdex_root_path. +# It also checks if the pocketdex repo has updates and, if so, creates a resource which prints instructions to update. +def load_pocketdex(pocketdex_root_path): + if repo_is_outdated(pocketdex_root_path): + pocketdex_outdated_resource(pocketdex_root_path) + + pocketdex_tilt_path = os.path.join(pocketdex_root_path, "tiltfiles", "pocketdex.tilt") + pocketdex_tilt = load_dynamic(pocketdex_tilt_path) + + postgres_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-postgres.yaml") + pgadmin_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-pgadmin.yaml") + indexer_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-indexer.yaml") + gql_engine_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-gql-engine.yaml") + pocketdex_tilt["pocketdex"](pocketdex_root_path, + genesis_file_name="localnet.json", + postgres_values_path=postgres_values_path, + pgadmin_values_path=pgadmin_values_path, + indexer_values_path=indexer_values_path, + gql_engine_values_path=gql_engine_values_path) + + +# check_and_load_pocketdex checks if sibling pocketdex repo exists. +# If it does, load the pocketdex.tilt file from the sibling repo. +# Otherwise, check the `indexer.clone_if_not_present` flag in `localnet_config.yaml` and EITHER: +# 1. clone pocketdex to ../pocketdex +# -- OR -- +# 2. Prints a message if true or false +def check_and_load_pocketdex(indexer_config): + if indexer_config["enabled"]: + pocketdex_root_path = indexer_config["repo_path"] + if not os.path.exists(pocketdex_root_path): + if indexer_config["clone_if_not_present"]: + clone_repo("https://github.com/pokt-network/pocketdex", pocketdex_root_path) + load_pocketdex(pocketdex_root_path) + else: + pocketdex_disabled_resource("Pocketdex repo not found at {}. Set `clone_if_not_present` to `true` in `localnet_config.yaml`.".format(pocketdex_root_path)) + else: + print("Using existing pocketdex repo") + load_pocketdex(pocketdex_root_path) + else: + pocketdex_disabled_resource("Pocketdex indexer disabled. Set `indexer.enabled` to `true` in `localnet_config.yaml` to enable it.")