From 385e4784221fcfbea844175203931bea3064f23f Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Wed, 7 Jun 2023 18:32:16 +0530 Subject: [PATCH 1/4] chore: cleanup and improve project names --- README.md | 60 ++++++++--------- config/aggregator.example.json | 114 ++++++++++++++++----------------- config/projects.example.json | 4 +- snapshotter_autofill.sh | 2 +- 4 files changed, 90 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index e9055ffc..b7689820 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The size of an epoch is configurable. Let that be referred to as `size(E)` and then publishes an epoch `(h₁, h₂)` by sending a transaction to the protocol state smart contract deployed on the Prost Chain (anchor chain) so that `h₂ - h₁ + 1 == size(E)`. The next epoch, therefore, is tracked from `h₂ + 1`. - Each such transaction emits an `EpochReleased` event + Each such transaction emits an `EpochReleased` event ``` event EpochReleased(uint256 indexed epochId, uint256 begin, uint256 end, uint256 timestamp); @@ -92,14 +92,14 @@ The size of an epoch is configurable. Let that be referred to as `size(E)` Workers in [`config/projects.json`](config/projects.example.json) calculate base snapshots against this `epochId` which corresponds to collections of state observations and event logs between the blocks at height in the range `[begin, end]`, per Uniswap v2 pair contract. Each such pair contract is assigned a project ID on the protocol state contract according to the following format: `{project_type}:{pair_contract_address}:{settings.namespace}` - + The snapshots generated by workers defined in this config are the fundamental data models on which higher order aggregates and richer datapoints are built. ### Snapshot Finalization -All snapshots per project reach consensus on the protocol state contract which results in a `SnapshotFinalized` event being triggered. +All snapshots per project reach consensus on the protocol state contract which results in a `SnapshotFinalized` event being triggered. -This helps us in building sophisticated aggregates, super-aggregates, filters and other forms of data composition on top of base snapshots. +This helps us in building sophisticated aggregates, super-aggregates, filters and other forms of data composition on top of base snapshots. ``` event SnapshotFinalized(uint256 indexed epochId, uint256 epochEnd, string projectId, string snapshotCid, uint256 timestamp); @@ -128,26 +128,26 @@ Pooler needs the following config files to be present "project_type": "snapshot_project_name_prefix_", "projects": ["array of smart contract addresses"], // Uniswap v2 pair contract addresses in this implementation "processor":{ - "module": "pooler.modules.uniswapv2.pair_total_reserves", + "module": "pooler.modules.uniswapv2.pair_total_reserves", "class_name": "PairTotalReservesProcessor" // class to be found in module pooler/modules/uniswapv2/pair_total_reserves.py } } ``` Copy over [`config/projects.example.json`](config/projects.example.json) to `config/projects.json`. For more details, read on in the [section below on extending a use case](#extending-pooler-with-a-uniswap-v2-data-point). - * **`config/aggregator.json`** : This lists out different type of aggregation work to be performed over a span of snapshots. Copy over [`config/aggregator.example.json`](config/aggregator.example.json) to `config/aggregator.json`. The span is usually calculated as a function of the epoch size and average block time on the data source network. For eg, + * **`config/aggregator.json`** : This lists out different type of aggregation work to be performed over a span of snapshots. Copy over [`config/aggregator.example.json`](config/aggregator.example.json) to `config/aggregator.json`. The span is usually calculated as a function of the epoch size and average block time on the data source network. For eg, * the following configuration calculates a snapshot of total trade volume over a 24 hour time period, based on the [snapshot finalization](#snapshot-finalization) of a project ID corresponding to a pair contract. This can be seen by the `aggregate_on` key being set to `SingleProject`. - * This is specified by the `filters` key below. If a [snapshot finalization](#snapshot-finalization) is achieved for an epoch over a project ID [(ref:generation of project ID for snapshot building workers)](#epoch-generation) `uniswap_pairContract_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod`, this would trigger the worker [`AggreagateTradeVolumeProcessor`](pooler/modules/uniswapv2/aggregate/single_uniswap_trade_volume_24h.py) as defined in the `processor` section of the config against the pair contract `0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc`. + * This is specified by the `filters` key below. If a [snapshot finalization](#snapshot-finalization) is achieved for an epoch over a project ID [(ref:generation of project ID for snapshot building workers)](#epoch-generation) `pairContract_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2`, this would trigger the worker [`AggreagateTradeVolumeProcessor`](pooler/modules/uniswapv2/aggregate/single_uniswap_trade_volume_24h.py) as defined in the `processor` section of the config against the pair contract `0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc`. ```javascript { "config": [ { - "project_type": "aggregate_uniswap_pairContract_24h_trade_volume", + "project_type": "aggregate_pairContract_24h_trade_volume", "aggregate_on": "SingleProject", "filters": { // this triggers the compute() contained in the processor class at the module location - // every time a `SnapshotFinalized` event is received for project IDs containing the prefix `uniswap_pairContract_trade_volume` + // every time a `SnapshotFinalized` event is received for project IDs containing the prefix `pairContract_trade_volume` // at each epoch ID - "projectId": "uniswap_pairContract_trade_volume" + "projectId": "pairContract_trade_volume" }, "processor": { "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_24h", @@ -158,19 +158,19 @@ Pooler needs the following config files to be present } ``` * The following configuration generates a collection of data sets of 24 hour trade volume as calculated by the worker above across multiple pair contracts. This can be seen by the `aggregate_on` key being set to `MultiProject`. - * `projects_to_wait_for` specifies the exact project IDs on which this collection will be generated once a [snapshot finalized event](#snapshot-finalization) has been received for an [`epochId`](#epoch-generation). + * `projects_to_wait_for` specifies the exact project IDs on which this collection will be generated once a [snapshot finalized event](#snapshot-finalization) has been received for an [`epochId`](#epoch-generation). ```javascript { "config": [ - "project_type": "aggregate_uniswap_24h_top_pairs", + "project_type": "aggregate_24h_top_pairs", "aggregate_on": "MultiProject", "projects_to_wait_for": [ // this triggers the compute() contained in the processor class at the module location - // after `SnapshotFinalized` events are received for project IDs containing the prefix `uniswap_pairContract_trade_volume` - "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod" + // after `SnapshotFinalized` events are received for project IDs containing the prefix `pairContract_trade_volume` + "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2", + "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2", + "pairContract_pair_total_reserves:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_24h", @@ -186,10 +186,10 @@ Pooler needs the following config files to be present - `instance_id`: This is the unique public key for your node to participate in consensus. It is currently registered on approval of an application (refer [deploy](https://github.com/PowerLoom/deploy) repo for more details on applying). - `namespace`, it is the unique key used to identify your project namespace around which all consensus activity takes place. - RPC service URL(s) and rate limit configurations. Rate limits are service provider specific, different RPC providers have different rate limits. Example rate limit config for a node looks something like this `"100000000/day;20000/minute;2500/second"` - - **`rpc.full_nodes`**: This will correspond to RPC nodes for the chain on which the data source smart contracts lives (for eg. Ethereum Mainnet, Polygon Mainnet etc). - - **`anchor_chain_rpc.full_nodes`**: This will correspond to RPC nodes for the anchor chain on which the protocol state smart contract lives (Prost Chain). + - **`rpc.full_nodes`**: This will correspond to RPC nodes for the chain on which the data source smart contracts lives (for eg. Ethereum Mainnet, Polygon Mainnet etc). + - **`anchor_chain_rpc.full_nodes`**: This will correspond to RPC nodes for the anchor chain on which the protocol state smart contract lives (Prost Chain). - **`protocol_state.address`** : This will correspond to the address at which the protocol state smart contract is deployed on the anchor chain. **`protocol_state.abi`** is already filled in the example and already available at the static path specified [`pooler/static/abis/ProtocolContract.json`](pooler/static/abis/ProtocolContract.json) - + ## Monitoring and Debugging Login to pooler docker container using `docker exec -it deploy-pooler-1 bash` (use `docker ps` to verify its presence in the list of running containers) and use the following commands for monitoring and debugging @@ -208,12 +208,12 @@ Now, whenever you commit anything, it'll automatically check the files you've ch ## Extending pooler with a Uniswap v2 data point -In this section, let us take a look at the data composition abilities of Pooler to build on the base snapshot being built that captures information on Uniswap trades. +In this section, let us take a look at the data composition abilities of Pooler to build on the base snapshot being built that captures information on Uniswap trades. ### Step 1. Review: Base snapshot extraction logic for trade information -Required reading: -* [Base Snapshot Generation](#base-snapshot-generation) and +Required reading: +* [Base Snapshot Generation](#base-snapshot-generation) and * [configuring `config/projects.json`](#configuration) * [Aggregation and data composition](#aggregation-and-data-composition---snapshot-generation-of-higher-order-datapoints-on-base-snapshots) @@ -257,9 +257,9 @@ https://github.com/PowerLoom/pooler/blob/1452c166bef7534568a61b3a2ab0ff94535d722 ### Step 2. Review: 24 hour aggregate of trade volume snapshots over a single pair contract -* As demonstrated in the previous section, the `TradeVolumeProcessor` logic takes care of capturing a snapshot of information regarding Uniswap v2 trades between the block heights of `min_chain_height` and `max_chain_height`. +* As demonstrated in the previous section, the `TradeVolumeProcessor` logic takes care of capturing a snapshot of information regarding Uniswap v2 trades between the block heights of `min_chain_height` and `max_chain_height`. -* The epoch size as described in the prior section on [epoch generation](#epoch-generation) can be considered to be constant for this specific implementation of the Uniswap v2 use case on PowerLoom Protocol, and by extension, the time duration captured within the epoch. +* The epoch size as described in the prior section on [epoch generation](#epoch-generation) can be considered to be constant for this specific implementation of the Uniswap v2 use case on PowerLoom Protocol, and by extension, the time duration captured within the epoch. * As shown in the section on [dependency graph of data composition](#aggregation-and-data-composition---snapshot-generation), every aggregate is calculated relative to the `epochId` at which the dependee [`SnapshotFinalized` event](#snapshot-finalization) is receieved. @@ -294,7 +294,7 @@ From the information provided above, the following is left as an exercise for th * Add a new configuration entry in `config/aggregator.json` for this new aggregation worker class -* Define a new data model in [`utils/message_models.py`](pooler/modules/uniswapv2/utils/models/message_models.py) referring to +* Define a new data model in [`utils/message_models.py`](pooler/modules/uniswapv2/utils/models/message_models.py) referring to * `UniswapTradesAggregateSnapshot` as used in above example * `UniswapTradesSnapshot` used to capture each epoch's trade snapshots which includes the raw event logs as well @@ -313,12 +313,12 @@ Related information and other services depending on these can be found in previo ### Process Hub Core -The Process Hub Core, defined in [`process_hub_core.py`](pooler/process_hub_core.py), serves as the primary process manager in Pooler. -* Operated by the CLI tol [`processhub_cmd.py`](pooler/processhub_cmd.py), it is responsible for starting and managing the `SystemEventDetector` and `ProcessorDistributor` processes. +The Process Hub Core, defined in [`process_hub_core.py`](pooler/process_hub_core.py), serves as the primary process manager in Pooler. +* Operated by the CLI tol [`processhub_cmd.py`](pooler/processhub_cmd.py), it is responsible for starting and managing the `SystemEventDetector` and `ProcessorDistributor` processes. * Additionally, it spawns the base snapshot and aggregator workers required for processing tasks from the `powerloom-backend-callback` queue. The number of workers and their configuration path can be adjusted in `config/settings.json`. ### Processor Distributor -The Processor Distributor, defined in [`processor_distributor.py`](pooler/processor_distributor.py), is initiated using the `processhub_cmd.py` CLI. +The Processor Distributor, defined in [`processor_distributor.py`](pooler/processor_distributor.py), is initiated using the `processhub_cmd.py` CLI. * It loads the base snapshotting and aggregator config information from settings * It reads the events forwarded by the event detector to the `f'powerloom-event-detector:{settings.namespace}:{settings.instance_id}'` RabbitMQ queue bound to a topic exchange as configured in `settings.rabbitmq.setup.event_detector.exchange`([code-ref: RabbitMQ exchanges and queue setup in pooler](pooler/init_rabbitmq.py)) @@ -380,4 +380,4 @@ try { * [Twitter](https://twitter.com/PowerLoomHQ) * [Github](https://github.com/PowerLoom) * [Careers](https://wellfound.com/company/powerloom/jobs) -* [Medium Engineering Blog](https://medium.com/powerloom) \ No newline at end of file +* [Medium Engineering Blog](https://medium.com/powerloom) diff --git a/config/aggregator.example.json b/config/aggregator.example.json index c8eb8e1f..1af42230 100644 --- a/config/aggregator.example.json +++ b/config/aggregator.example.json @@ -1,9 +1,9 @@ { "config": [{ - "project_type": "aggregate_uniswap_pairContract_24h_trade_volume", + "project_type": "aggregate_pairContract_24h_trade_volume", "aggregate_on": "SingleProject", "filters": { - "projectId": "uniswap_pairContract_trade_volume" + "projectId": "pairContract_trade_volume" }, "processor": { "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_24h", @@ -11,10 +11,10 @@ } }, { - "project_type": "aggregate_uniswap_pairContract_7d_trade_volume", + "project_type": "aggregate_pairContract_7d_trade_volume", "aggregate_on": "SingleProject", "filters": { - "projectId": "aggregate_uniswap_pairContract_24h_trade_volume" + "projectId": "aggregate_pairContract_24h_trade_volume" }, "processor": { "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_7d", @@ -22,23 +22,23 @@ } }, { - "project_type": "aggregate_uniswap_24h_top_pairs_lite", + "project_type": "aggregate_24h_top_pairs_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ - "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" + "aggregate_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2", + "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_24h", @@ -46,23 +46,23 @@ } }, { - "project_type": "aggregate_uniswap_24h_top_tokens_lite", + "project_type": "aggregate_24h_top_tokens_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ - "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" + "aggregate_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2", + "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_tokens", @@ -70,16 +70,16 @@ } }, { - "project_type": "aggregate_uniswap_7d_top_pairs_lite", + "project_type": "aggregate_7d_top_pairs_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ - "aggregate_uniswap_pairContract_7d_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" + "aggregate_pairContract_7d_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "aggregate_pairContract_7d_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "aggregate_pairContract_7d_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "aggregate_pairContract_7d_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "aggregate_pairContract_7d_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "aggregate_pairContract_7d_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "aggregate_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_7d", @@ -87,23 +87,23 @@ } }, { - "project_type": "aggregate_uniswap_24h_stats_lite", + "project_type": "aggregate_24h_stats_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ - "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" + "aggregate_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2", + "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2", + "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_stats", diff --git a/config/projects.example.json b/config/projects.example.json index 8c8576ef..d57c1d5d 100644 --- a/config/projects.example.json +++ b/config/projects.example.json @@ -1,6 +1,6 @@ { "config": [{ - "project_type": "uniswap_pairContract_pair_total_reserves", + "project_type": "pairContract_pair_total_reserves", "projects":[ "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc", "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5", @@ -16,7 +16,7 @@ } }, { - "project_type": "uniswap_pairContract_trade_volume", + "project_type": "pairContract_trade_volume", "projects":[ "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc", "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5", diff --git a/snapshotter_autofill.sh b/snapshotter_autofill.sh index fd4aa0cc..8464473a 100755 --- a/snapshotter_autofill.sh +++ b/snapshotter_autofill.sh @@ -45,7 +45,7 @@ cp config/aggregator.example.json config/aggregator.json cp config/auth_settings.example.json config/auth_settings.json cp config/settings.example.json config/settings.json -export namespace=UNISWAPV2-ph15-prod +export namespace=UNISWAPV2 export prost_rpc_url="${PROST_RPC_URL:-https://rpc-prost1b.powerloom.io}" export ipfs_url="${IPFS_URL:-/dns/ipfs/tcp/5001}" From e7a5bd62debfe726d1473f0dfb68856dab43ef25 Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Wed, 7 Jun 2023 18:57:56 +0530 Subject: [PATCH 2/4] fix: double assignment of variable --- pooler/utils/aggregation_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pooler/utils/aggregation_worker.py b/pooler/utils/aggregation_worker.py index 5f8bd7c2..59a07282 100644 --- a/pooler/utils/aggregation_worker.py +++ b/pooler/utils/aggregation_worker.py @@ -110,7 +110,7 @@ async def _processor_task( def _gen_single_type_project_id(self, type_, epoch): contract = epoch.projectId.split(':')[-2] - project_id = project_id = f'{type_}:{contract}:{settings.namespace}' + project_id = f'{type_}:{contract}:{settings.namespace}' return project_id def _gen_multiple_type_project_id(self, type_, epoch): From 1982b127a78b3b0162307159c6bfaa6b55c4e4b5 Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Wed, 7 Jun 2023 19:00:38 +0530 Subject: [PATCH 3/4] doc: include references to project ID generation --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index b7689820..8120c6fd 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ The size of an epoch is configurable. Let that be referred to as `size(E)` `{project_type}:{pair_contract_address}:{settings.namespace}` +https://github.com/PowerLoom/pooler/blob/e7a5bd62debfe726d1473f0dfb68856dab43ef25/pooler/utils/snapshot_worker.py#L114 + The snapshots generated by workers defined in this config are the fundamental data models on which higher order aggregates and richer datapoints are built. ### Snapshot Finalization @@ -114,6 +116,9 @@ Workers as defined in `config/aggregator.json` are triggered by the appropriate ![Aggregation Workflow](pooler/static/docs/assets/AggregationWorkflow.png) +In case of aggregation over multiple projects, their project IDs are generated with a combination of the hash of the dependee project IDs along with the namespace + +https://github.com/PowerLoom/pooler/blob/e7a5bd62debfe726d1473f0dfb68856dab43ef25/pooler/utils/aggregation_worker.py#L116-L124 ## Development Instructions These instructions are needed if you're planning to run the system using `build-dev.sh` from [deploy](https://github.com/PowerLoom/deploy). From af9b173456edd7e8e5bd05a6dbddd5f5dbcd5f5d Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Wed, 7 Jun 2023 19:49:37 +0530 Subject: [PATCH 4/4] chore: udpate base image to python3.10 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 874c4c2c..bda420dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nikolaik/python-nodejs:python3.9-nodejs18-bullseye +FROM nikolaik/python-nodejs:python3.10-nodejs18 # Install the PM2 process manager for Node.js RUN npm install pm2 -g