Skip to content

Commit

Permalink
DeepBook indexer - add event_digest, replace id, add checkpoint_times…
Browse files Browse the repository at this point in the history 
…tamp_ms (#20008)

## Description 

Describe the changes or additions included in this PR.
* Added a new column event_digest - digest + event index. This is a
unique value. It is used as primary key and will ensure an event is not
added twice.
* Added a new column checkpoint_timestamp_ms - best way to know when the
event was emitted, since some events don't emit the onchain timestamp.

## Test plan 

How did you test the new or updated feature?
Local end to end testing. I manipulated the local progress_store to
force reread checkpoints, which resulted in an error when adding
duplicate events (as expected).

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:
  • Loading branch information
@0xaslan
0xaslan authored Oct 24, 2024
1 parent df3cc8b commit ba49d22
Show file tree
Hide file tree
Showing 6 changed files with 215 additions and 71 deletions.
7 changes: 7 additions & 0 deletions crates/sui-deepbook-indexer/src/migrations/00000000000000_diesel_initial_setup/down.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,12 @@ DROP TABLE IF EXISTS order_updates;
DROP TABLE IF EXISTS order_fills;
DROP TABLE IF EXISTS flashloans;
DROP TABLE IF EXISTS pool_prices;
DROP TABLE IF EXISTS balances;
DROP TABLE IF EXISTS trade_params_update;
DROP TABLE IF EXISTS stakes;
DROP TABLE IF EXISTS proposals;
DROP TABLE IF EXISTS votes;
DROP TABLE IF EXISTS rebates;
DROP TABLE IF EXISTS sui_error_transactions;
DROP TABLE IF EXISTS progress_store;
DROP TABLE IF EXISTS pools;
30 changes: 20 additions & 10 deletions crates/sui-deepbook-indexer/src/migrations/00000000000000_diesel_initial_setup/up.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@

CREATE TABLE IF NOT EXISTS order_updates
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
status TEXT NOT NULL,
pool_id TEXT NOT NULL,
Expand All @@ -24,11 +25,12 @@ CREATE TABLE IF NOT EXISTS order_updates

CREATE TABLE IF NOT EXISTS order_fills
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
pool_id TEXT NOT NULL,
maker_order_id TEXT NOT NULL,
Expand All @@ -50,11 +52,12 @@ CREATE TABLE IF NOT EXISTS order_fills

CREATE TABLE IF NOT EXISTS flashloans
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
borrow BOOLEAN NOT NULL,
pool_id TEXT NOT NULL,
Expand All @@ -64,11 +67,12 @@ CREATE TABLE IF NOT EXISTS flashloans

CREATE TABLE IF NOT EXISTS pool_prices
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
target_pool TEXT NOT NULL,
reference_pool TEXT NOT NULL,
Expand All @@ -77,11 +81,12 @@ CREATE TABLE IF NOT EXISTS pool_prices

CREATE TABLE IF NOT EXISTS balances
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
balance_manager_id TEXT NOT NULL,
asset TEXT NOT NULL,
Expand All @@ -91,11 +96,12 @@ CREATE TABLE IF NOT EXISTS balances

CREATE TABLE IF NOT EXISTS trade_params_update
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
pool_id TEXT NOT NULL,
taker_fee BIGINT NOT NULL,
Expand All @@ -105,11 +111,12 @@ CREATE TABLE IF NOT EXISTS trade_params_update

CREATE TABLE IF NOT EXISTS stakes
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
pool_id TEXT NOT NULL,
balance_manager_id TEXT NOT NULL,
Expand All @@ -120,11 +127,12 @@ CREATE TABLE IF NOT EXISTS stakes

CREATE TABLE IF NOT EXISTS proposals
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
pool_id TEXT NOT NULL,
balance_manager_id TEXT NOT NULL,
Expand All @@ -136,11 +144,12 @@ CREATE TABLE IF NOT EXISTS proposals

CREATE TABLE IF NOT EXISTS votes
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
pool_id TEXT NOT NULL,
balance_manager_id TEXT NOT NULL,
Expand All @@ -152,11 +161,12 @@ CREATE TABLE IF NOT EXISTS votes

CREATE TABLE IF NOT EXISTS rebates
(
id SERIAL PRIMARY KEY,
event_digest TEXT PRIMARY KEY,
digest TEXT NOT NULL,
sender TEXT NOT NULL,
checkpoint BIGINT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
checkpoint_timestamp_ms BIGINT NOT NULL,
package TEXT NOT NULL,
pool_id TEXT NOT NULL,
balance_manager_id TEXT NOT NULL,
Expand Down
40 changes: 30 additions & 10 deletions crates/sui-deepbook-indexer/src/models.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@ use crate::schema::{
};

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = order_updates, primary_key(digest))]
#[diesel(table_name = order_updates, primary_key(event_digest))]
pub struct OrderUpdate {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub status: String,
pub pool_id: String,
Expand All @@ -34,11 +36,13 @@ pub struct OrderUpdate {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = order_fills, primary_key(digest))]
#[diesel(table_name = order_fills, primary_key(event_digest))]
pub struct OrderFill {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub maker_order_id: String, // u128
Expand Down Expand Up @@ -74,11 +78,13 @@ pub struct BalancesSummary {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = flashloans, primary_key(digest))]
#[diesel(table_name = flashloans, primary_key(event_digest))]
pub struct Flashloan {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub borrow_quantity: i64,
Expand All @@ -87,23 +93,27 @@ pub struct Flashloan {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = pool_prices, primary_key(digest))]
#[diesel(table_name = pool_prices, primary_key(event_digest))]
pub struct PoolPrice {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub target_pool: String,
pub reference_pool: String,
pub conversion_rate: i64,
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = balances, primary_key(digest))]
#[diesel(table_name = balances, primary_key(event_digest))]
pub struct Balances {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub balance_manager_id: String,
pub asset: String,
Expand All @@ -112,11 +122,13 @@ pub struct Balances {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = proposals, primary_key(digest))]
#[diesel(table_name = proposals, primary_key(event_digest))]
pub struct Proposals {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub balance_manager_id: String,
Expand All @@ -127,11 +139,13 @@ pub struct Proposals {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = rebates, primary_key(digest))]
#[diesel(table_name = rebates, primary_key(event_digest))]
pub struct Rebates {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub balance_manager_id: String,
Expand All @@ -140,11 +154,13 @@ pub struct Rebates {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = stakes, primary_key(digest))]
#[diesel(table_name = stakes, primary_key(event_digest))]
pub struct Stakes {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub balance_manager_id: String,
Expand All @@ -154,11 +170,13 @@ pub struct Stakes {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = trade_params_update, primary_key(digest))]
#[diesel(table_name = trade_params_update, primary_key(event_digest))]
pub struct TradeParamsUpdate {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub taker_fee: i64,
Expand All @@ -167,11 +185,13 @@ pub struct TradeParamsUpdate {
}

#[derive(Queryable, Selectable, Insertable, Identifiable, Debug)]
#[diesel(table_name = votes, primary_key(digest))]
#[diesel(table_name = votes, primary_key(event_digest))]
pub struct Votes {
pub event_digest: String,
pub digest: String,
pub sender: String,
pub checkpoint: i64,
pub checkpoint_timestamp_ms: i64,
pub package: String,
pub pool_id: String,
pub balance_manager_id: String,
Expand Down
Loading

0 comments on commit ba49d22

Please sign in to comment.