From 7ec4cf2926f9f8c7ca5713ceff4982a4c3612d07 Mon Sep 17 00:00:00 2001 From: Mansi <122999960+IrishLatte19@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:42:50 +0530 Subject: [PATCH 01/59] Update tokens_viction_erc20.sql (#7270) * Update tokens_viction_erc20.sql * Update tokens_viction_erc20.sql --- .../tokens/models/tokens/viction/tokens_viction_erc20.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/tokens/models/tokens/viction/tokens_viction_erc20.sql b/dbt_subprojects/tokens/models/tokens/viction/tokens_viction_erc20.sql index 3a9a1a3bd53..9bc999f4031 100644 --- a/dbt_subprojects/tokens/models/tokens/viction/tokens_viction_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/viction/tokens_viction_erc20.sql @@ -15,4 +15,7 @@ FROM (VALUES (0xC054751BdBD24Ae713BA3Dc9Bd9434aBe2abc1ce, 'WVIC', 18) , (0x381B31409e4D220919B2cFF012ED94d70135A59e, 'USDT', 6) , (0x20cC4574f263C54eb7aD630c9AC6d4d9068Cf127, 'USDC', 6) -) AS temp_table (contract_address, symbol, decimals) \ No newline at end of file + , (0xb786d9c8120d311b948cf1e5aa48d8fbacf477e2, 'SAROS', 18) + , (0x0fd0288aaae91eaf935e2ec14b23486f86516c8c, 'C98', 18) + , (0xCdde1f5D971A369eB952192F9a5C367f33a0A891, 'SVIC', 18) +) AS temp_table (contract_address, symbol, decimals) From 1f8001ea5babb64c5042d6f15601bb94f8609d59 Mon Sep 17 00:00:00 2001 From: Lydia Puspita <113081740+lydiapuspita@users.noreply.github.com> Date: Tue, 10 Dec 2024 00:13:01 +0700 Subject: [PATCH 02/59] Add SHRUB, SBR, MSTR, SIGMA, FWOG, LESTER, & HAPPY token price (#7267) * Update prices_ethereum_tokens.sql Add SPX6900 token price * Update prices_solana_tokens.sql Add MOODENG token price * Update prices_solana_tokens.sql Add SCF token price * Update prices_ethereum_tokens.sql Add KEKEC token price * Update prices_solana_tokens.sql Add GOAT and LUNA by Virtuals token price * Update prices_solana_tokens.sql add FWOG, LESTER, & HAPPY token price * Update prices_ethereum_tokens.sql add SHRUB, SBR, SIGMA, & MSTR token price --------- Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../models/prices/ethereum/prices_ethereum_tokens.sql | 6 +++++- .../tokens/models/prices/solana/prices_solana_tokens.sql | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql b/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql index f2e49568891..e70afa9db2d 100644 --- a/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql +++ b/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql @@ -1864,7 +1864,11 @@ FROM ('oas-oases-chain', 'ethereum', 'OAS', 0x89885fc1f76c3f4cc719640e33c315227da7003a, 18), ('sidus-sidus', 'ethereum', 'SIDUS', 0x549020a9Cb845220D66d3E9c6D9F9eF61C981102, 18), ('mcg-metalcore', 'ethereum', 'MCG', 0xc9E503562d0Db0A2629288a5D3b1c94Ea7741869, 18), - ('mavia-heroes-of-mavia', 'ethereum', 'MAVIA', 0x24fcfc492c1393274b6bcd568ac9e225bec93584, 18) + ('mavia-heroes-of-mavia', 'ethereum', 'MAVIA', 0x24fcfc492c1393274b6bcd568ac9e225bec93584, 18), + ('shrub-shrub', 'ethereum', 'SHRUB', 0x3b991130eae3cca364406d718da22fa1c3e7c256, 18), + ('sbr-strategic-bitcoin-reserve', 'ethereum', 'SBR', 0xd6203889c22d9fe5e938a9200f50fdffe9dd8e02, 9), + ('sigma2-sigma', 'ethereum', 'SIGMA', 0x18a8D75F70eAEad79B5A55903D036ce337F623A5, 18), + ('mstr-mstr2100', 'ethereum', 'MSTR', 0x42069026EAC8Eee0Fd9b5f7aDFa4f6E6D69a2B39, 9) ) as temp (token_id, blockchain, symbol, contract_address, decimals) where contract_address not in ( -- bad price feeds diff --git a/dbt_subprojects/tokens/models/prices/solana/prices_solana_tokens.sql b/dbt_subprojects/tokens/models/prices/solana/prices_solana_tokens.sql index 37cc907ebd8..6feb9cdbd4d 100644 --- a/dbt_subprojects/tokens/models/prices/solana/prices_solana_tokens.sql +++ b/dbt_subprojects/tokens/models/prices/solana/prices_solana_tokens.sql @@ -707,5 +707,8 @@ FROM ('moodeng-moo-deng-moodengsolcom', 'solana', 'MOODENG', 'ED5nyyWEzpPPiWimP8vYm7sD7TD3LAt3Q3gRTWHzPJBY', 6), ('scf-smoking-chicken-fish', 'solana', 'SCF', 'GiG7Hr61RVm4CSUxJmgiCoySFQtdiwxtqf64MsRppump', 6), ('luna-luna-by-virtuals', 'solana', 'LUNA', '9se6kma7LeGcQWyRBNcYzyxZPE3r9t9qWZ8SnjnN3jJ7', 8), - ('goat-goatseus-maximus', 'solana', 'GOAT', 'CzLSujWBLFsSjncfkh59rUFqvafWcY5tzedWJSuypump', 6) + ('goat-goatseus-maximus', 'solana', 'GOAT', 'CzLSujWBLFsSjncfkh59rUFqvafWcY5tzedWJSuypump', 6), + ('fwog-fwog', 'solana', 'FWOG', 'A8C3xuqscfmyLrte3VmTqrAq8kgMASius9AFNANwpump', 6), + ('lester-litecoin-mascot', 'solana', 'LESTER', '5z3iCe53hUANTiG8Js8RjHNE2Arjik7L2CXLyr2rpump', 6), + ('happy-happycat', 'solana', 'HAPPY', 'HAPPYwgFcjEJDzRtfWE6tiHE9zGdzpNky2FvjPHsvvGZ', 9) ) as temp (token_id, blockchain, symbol, contract_address, decimals) From 9665ddd32315883df073dbf7a92cdd83385bd92d Mon Sep 17 00:00:00 2001 From: whale_hunter <143016036+whalehunting@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:13:14 +0100 Subject: [PATCH 03/59] chore: add new chainswap deployers across all chains (#7269) --- .../models/chainswap/arbitrum/chain_swap_arbitrum_trades.sql | 2 ++ .../chainswap/avalanche_c/chain_swap_avalanche_c_trades.sql | 2 ++ .../models/chainswap/base/chain_swap_base_trades.sql | 2 ++ .../models/chainswap/bnb/chain_swap_bnb_trades.sql | 2 ++ .../models/chainswap/ethereum/chain_swap_ethereum_trades.sql | 2 ++ .../models/chainswap/optimism/chain_swap_optimism_trades.sql | 2 ++ .../models/chainswap/polygon/chain_swap_polygon_trades.sql | 2 ++ 7 files changed, 14 insertions(+) diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/arbitrum/chain_swap_arbitrum_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/arbitrum/chain_swap_arbitrum_trades.sql index 99e946ced3c..591bf524275 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/arbitrum/chain_swap_arbitrum_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/arbitrum/chain_swap_arbitrum_trades.sql @@ -23,6 +23,7 @@ {% set deployer_6 = '0xb252f0ab7bdf1be4d5bbf607eb5c220b2d902a2c' %} {% set deployer_7 = '0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94' %} {% set deployer_8 = '0xc8378819fbB95130c34D62f520167F745B13C305' %} +{% set deployer_9 = '0xde7Cb3d58D4004ff0De70995C0604089cc945EAF' %} {% set weth_contract_address = '0x82aF49447D8a07e3bd95BD0d56f35241523fBab1' %} {% set usdc_contract_address = '0xaf88d065e77c8cC2239327C5EDb3A432268e5831' %} {% set fee_recipient_1 = '0x415EEc63c95e944D544b3088bc682B759edB8548' %} @@ -42,6 +43,7 @@ with or "from" = {{ deployer_6 }} or "from" = {{ deployer_7 }} or "from" = {{ deployer_8 }} + or "from" = {{ deployer_9 }} ) and block_time >= timestamp '{{project_start_date}}' ), diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/avalanche_c/chain_swap_avalanche_c_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/avalanche_c/chain_swap_avalanche_c_trades.sql index b2a0547ac21..34d5ead3a0b 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/avalanche_c/chain_swap_avalanche_c_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/avalanche_c/chain_swap_avalanche_c_trades.sql @@ -20,6 +20,7 @@ {% set deployer_3 = '0xc1cc1a300Dcfe5359eBe37f2007A77d1F91533ba' %} {% set deployer_4 = '0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94' %} {% set deployer_5 = '0xc8378819fbB95130c34D62f520167F745B13C305' %} +{% set deployer_6 = '0xde7Cb3d58D4004ff0De70995C0604089cc945EAF' %} {% set wavax_contract_address = '0xB31f66AA3C1e785363F0875A1B74E27b85FD66c7' %} {% set usdc_contract_address = '0xB97EF9Ef8734C71904D8002F8b6Bc66Dd9c48a6E' %} {% set fee_recipient_1 = '0x415EEc63c95e944D544b3088bc682B759edB8548' %} @@ -36,6 +37,7 @@ with or "from" = {{ deployer_3 }} or "from" = {{ deployer_4 }} or "from" = {{ deployer_5 }} + or "from" = {{ deployer_6 }} ) and block_time >= timestamp '{{project_start_date}}' ), diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/base/chain_swap_base_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/base/chain_swap_base_trades.sql index 5e021ef8056..4ad4612a9d9 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/base/chain_swap_base_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/base/chain_swap_base_trades.sql @@ -22,6 +22,7 @@ {% set deployer_5 = '0xb252f0ab7bdf1be4d5bbf607eb5c220b2d902a2c' %} {% set deployer_6 = '0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94' %} {% set deployer_7 = "0xc8378819fbB95130c34D62f520167F745B13C305" %} +{% set deployer_8 = "0xde7Cb3d58D4004ff0De70995C0604089cc945EAF" %} {% set weth_contract_address = '0x4200000000000000000000000000000000000006' %} {% set usdc_contract_address = '0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913' %} {% set fee_recipient_1 = '0x415EEc63c95e944D544b3088bc682B759edB8548' %} @@ -40,6 +41,7 @@ with or "from" = {{ deployer_5 }} or "from" = {{ deployer_6 }} or "from" = {{ deployer_7 }} + or "from" = {{ deployer_8 }} ) and block_time >= timestamp '{{project_start_date}}' ), diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/bnb/chain_swap_bnb_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/bnb/chain_swap_bnb_trades.sql index c0a18a2e654..ca2c8015251 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/bnb/chain_swap_bnb_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/bnb/chain_swap_bnb_trades.sql @@ -19,6 +19,7 @@ {% set deployer_2 = '0x3A510C5a32bCb381c53704AED9c02b0c70041F7A' %} {% set deployer_3 = '0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94' %} {% set deployer_4 = '0xc8378819fbB95130c34D62f520167F745B13C305' %} +{% set deployer_5 = '0xde7Cb3d58D4004ff0De70995C0604089cc945EAF' %} {% set wbnb_contract_address = '0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c' %} {% set usdc_contract_address = '0x8AC76a51cc950d9822D68b83fE1Ad97B32Cd580d' %} {% set fee_recipient_1 = '0x415EEc63c95e944D544b3088bc682B759edB8548' %} @@ -34,6 +35,7 @@ with or "from" = {{ deployer_2 }} or "from" = {{ deployer_3 }} or "from" = {{ deployer_4 }} + or "from" = {{ deployer_5 }} ) and block_time >= timestamp '{{project_start_date}}' diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/ethereum/chain_swap_ethereum_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/ethereum/chain_swap_ethereum_trades.sql index 8a7689ce356..4ad0c4894ab 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/ethereum/chain_swap_ethereum_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/ethereum/chain_swap_ethereum_trades.sql @@ -22,6 +22,7 @@ {% set deployer_5 = "0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94" %} {% set deployer_6 = "0x686c0072dF3Df7A13ef666a3b661803a48558A90" %} {% set deployer_7 = "0xc8378819fbB95130c34D62f520167F745B13C305" %} +{% set deployer_8 = "0xde7Cb3d58D4004ff0De70995C0604089cc945EAF" %} {% set weth_contract_address = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" %} {% set usdc_contract_address = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" %} {% set usdt_contract_address = "0xdac17f958d2ee523a2206206994597c13d831ec7" %} @@ -41,6 +42,7 @@ with or "from" = {{ deployer_5 }} or "from" = {{ deployer_6 }} or "from" = {{ deployer_7 }} + or "from" = {{ deployer_8 }} ) and block_time >= timestamp '{{project_start_date}}' ), diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/optimism/chain_swap_optimism_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/optimism/chain_swap_optimism_trades.sql index 08677164306..2a110861880 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/optimism/chain_swap_optimism_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/optimism/chain_swap_optimism_trades.sql @@ -23,6 +23,7 @@ {% set deployer_6 = '0x415EEc63c95e944D544b3088bc682B759edB8548' %} {% set deployer_7 = '0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94' %} {% set deployer_8 = '0xc8378819fbB95130c34D62f520167F745B13C305' %} +{% set deployer_9 = '0xde7Cb3d58D4004ff0De70995C0604089cc945EAF' %} {% set weth_contract_address = '0x4200000000000000000000000000000000000006' %} {% set usdc_contract_address = '0x0b2C639c533813f4Aa9D7837CAf62653d097Ff85' %} {% set fee_recipient_1 = '0x415EEc63c95e944D544b3088bc682B759edB8548' %} @@ -42,6 +43,7 @@ with or "from" = {{ deployer_6 }} or "from" = {{ deployer_7 }} or "from" = {{ deployer_8 }} + or "from" = {{ deployer_9 }} ) and block_time >= timestamp '{{project_start_date}}' ), diff --git a/dbt_subprojects/daily_spellbook/models/chainswap/polygon/chain_swap_polygon_trades.sql b/dbt_subprojects/daily_spellbook/models/chainswap/polygon/chain_swap_polygon_trades.sql index f0a9fdbf3b3..5629a66a60d 100644 --- a/dbt_subprojects/daily_spellbook/models/chainswap/polygon/chain_swap_polygon_trades.sql +++ b/dbt_subprojects/daily_spellbook/models/chainswap/polygon/chain_swap_polygon_trades.sql @@ -22,6 +22,7 @@ {% set deployer_5 = '0xB7B953e81612c57256fF0aebD62B6a2F0546F7dA' %} {% set deployer_6 = '0xa24e8cE77D4A7Ce869DA3730e6560BfB66553F94' %} {% set deployer_7 = "0xc8378819fbB95130c34D62f520167F745B13C305" %} +{% set deployer_8 = "0xde7Cb3d58D4004ff0De70995C0604089cc945EAF" %} {% set wmatic_contract_address = '0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270' %} {% set usdc_contract_address = '0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359' %} {% set usdt_contract_address = '0xc2132d05d31c914a87c6611c10748aeb04b58e8f' %} @@ -41,6 +42,7 @@ with or "from" = {{ deployer_5 }} or "from" = {{ deployer_6 }} or "from" = {{ deployer_7 }} + or "from" = {{ deployer_8 }} ) and block_time >= timestamp '{{project_start_date}}' ), From 1f9f3f58e906edd010a59409f354fad900f0b389 Mon Sep 17 00:00:00 2001 From: tomfutago <35136350+tomfutago@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:13:23 +0000 Subject: [PATCH 04/59] Nexus Mutual - Capital Pool rETH/ETH fix (#7263) * fix: rETH/ETH * fix: sources --- .../nexusmutual_ethereum_capital_pool_totals.sql | 12 +++++++++++- .../ethereum/rocketpool_ethereum_sources.yml | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 sources/rocketpool/ethereum/rocketpool_ethereum_sources.yml diff --git a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/capital_pool/nexusmutual_ethereum_capital_pool_totals.sql b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/capital_pool/nexusmutual_ethereum_capital_pool_totals.sql index 0b7019fac62..c0bbf2369a4 100644 --- a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/capital_pool/nexusmutual_ethereum_capital_pool_totals.sql +++ b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/capital_pool/nexusmutual_ethereum_capital_pool_totals.sql @@ -269,6 +269,15 @@ daily_avg_cbbtc_prices as ( group by 1 ), +daily_reth_eth_prices as ( + select + date_trunc('day', call_block_time) as block_date, + avg(output_0 / 1e18) as avg_reth_eth_price + from {{ source('rocketpool_ethereum', 'RocketTokenRETH_call_getExchangeRate') }} + where call_block_time >= timestamp '2023-04-18' + group by 1 +), + day_sequence as ( select cast(d.seq_date as timestamp) as block_date from (select sequence(date '2019-05-23', current_date, interval '1' day) as days) as days_s @@ -337,7 +346,7 @@ daily_running_totals_enriched as ( -- rETH coalesce(drt.reth_total, 0) as reth_total, coalesce(drt.reth_total * p_avg_reth.price_usd, 0) as avg_reth_usd_total, - coalesce(drt.reth_total * p_avg_reth.price_usd / p_avg_eth.price_usd, 0) as avg_reth_eth_total, + coalesce(drt.reth_total * p_reth_eth.avg_reth_eth_price, 0) as avg_reth_eth_total, -- USDC coalesce(drt.usdc_total, 0) as usdc_total, coalesce(drt.usdc_total * p_avg_usdc.price_usd, 0) as avg_usdc_usd_total, @@ -362,6 +371,7 @@ daily_running_totals_enriched as ( left join daily_avg_reth_prices p_avg_reth on drt.block_date = p_avg_reth.block_date left join daily_avg_usdc_prices p_avg_usdc on drt.block_date = p_avg_usdc.block_date left join daily_avg_cbbtc_prices p_avg_cbbtc on drt.block_date = p_avg_cbbtc.block_date + left join daily_reth_eth_prices p_reth_eth on drt.block_date = p_reth_eth.block_date ) select diff --git a/sources/rocketpool/ethereum/rocketpool_ethereum_sources.yml b/sources/rocketpool/ethereum/rocketpool_ethereum_sources.yml new file mode 100644 index 00000000000..0e9a9d500fb --- /dev/null +++ b/sources/rocketpool/ethereum/rocketpool_ethereum_sources.yml @@ -0,0 +1,6 @@ +version: 2 + +sources: + - name: rocketpool_ethereum + tables: + - name: RocketTokenRETH_call_getExchangeRate From dfd137c0cebe04eb0248a97c7975afd8f1147454 Mon Sep 17 00:00:00 2001 From: Phu <33654804+lequangphu@users.noreply.github.com> Date: Tue, 10 Dec 2024 00:13:34 +0700 Subject: [PATCH 05/59] Bridges of LI.FI across chains, the contract is LiFiDiamond_v2 (#7172) * Add Odos on Arbitrum to dex_aggregator * Change to WETH address on Arbitrum * Add sources of Odos on Arbitrum * Add odos/arbitrum seeds * Change data type * Add bridges of LiFi across chains * replace tests which is deprecated * fix name error of avalanche source * add index to unique test * fix concat issue * fix unique test of the main model * try to fix unique key * use surrogate key for data tests * use generate_surrogate_key inside the models * fix and remove redundant schema properties * data_tests * move generate_surrogate_key to schema.yml * 1. create macro lifi_extract_bridge_data_macro.sql and apply it in the models 2. update the models' config 3. flip materialization strategy * data_ again * fix naming of the main model * replace avalanche with avalanche_c * avalanche_c again * 1. add block_date column to the macro to use it in the add_tx_columns macro 2. update model of avalanche_c * rename columns to use add_tx_columns macro * remove evt_ * a few minor changes * add amount_usd column * data_tests again * map native token to wrapped token to avoid null price data * update the main model and schema * move type conversion from models to macro * fix source name * varbinary type doesn't need single quotes * correct collumn name * add post_hook --------- Co-authored-by: Huang Geyang Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../lifi/lifi_extract_bridge_data_macro.sql | 37 ++++++++++ .../lifi/arbitrum/lifi_arbitrum_transfers.sql | 49 +++++++++++++ .../lifi_avalanche_c_transfers.sql | 47 ++++++++++++ .../_projects/lifi/bnb/lifi_bnb_transfers.sql | 47 ++++++++++++ .../lifi/ethereum/lifi_ethereum_transfers.sql | 47 ++++++++++++ .../lifi/fantom/lifi_fantom_transfers.sql | 47 ++++++++++++ .../lifi/gnosis/lifi_gnosis_transfers.sql | 47 ++++++++++++ .../models/_projects/lifi/lifi_transfers.sql | 61 ++++++++++++++++ .../models/_projects/lifi/schema.yml | 73 +++++++++++++++++++ .../lifi/zksync/lifi_zksync_transfers.sql | 47 ++++++++++++ sources/lifi/sources.yml | 56 ++++++++++++++ 11 files changed, 558 insertions(+) create mode 100644 dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche_c/lifi_avalanche_c_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/lifi_transfers.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql create mode 100644 sources/lifi/sources.yml diff --git a/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql b/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql new file mode 100644 index 00000000000..7210078cee3 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql @@ -0,0 +1,37 @@ +{% macro lifi_extract_bridge_data(blockchain) %} + +{% set bridge_data_fields = [ + 'transactionId', + 'bridge', + 'integrator', + 'referrer', + 'sendingAssetId', + 'receiver', + 'minAmount', + 'destinationChainId' +] %} + +select + contract_address, + evt_tx_hash as tx_hash, + evt_index, + evt_block_time as block_time, + evt_block_number as block_number, + cast(date_trunc('day', evt_block_time) as date) as block_date, + {% for field in bridge_data_fields %} + {% if field in ['transactionId', 'referrer', 'sendingAssetId', 'receiver'] %} + from_hex(json_extract_scalar(bridgedata, '$.{{ field }}')) as {{ field }}, + {% elif field == 'minAmount' %} + cast(json_extract_scalar(bridgedata, '$.{{ field }}') as double) as {{ field }}, + {% else %} + json_extract_scalar(bridgedata, '$.{{ field }}') as {{ field }}, + {% endif %} + {% endfor %} + '{{ blockchain }}' as source_chain, + {{ dbt_utils.generate_surrogate_key(['evt_tx_hash', 'evt_index']) }} as transfer_id +from {{ source('lifi_' ~ blockchain, 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} +{% if is_incremental() %} +where {{ incremental_predicate('evt_block_time') }} +{% endif %} + +{% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql new file mode 100644 index 00000000000..50c4011e59c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql @@ -0,0 +1,49 @@ +{{ config( + schema = 'lifi_arbitrum', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('arbitrum') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0x82af49447d8a07e3bd95bd0d56f35241523fbab1 -- WETH + when sendingAssetId = 0x3405a1bd46b85c5c029483fbecf2f3e611026e45 + then 0xff970a61a04b1ca14834a43f5de4533ebddb5cc8 -- USDC + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'arbitrum' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'arbitrum' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche_c/lifi_avalanche_c_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche_c/lifi_avalanche_c_transfers.sql new file mode 100644 index 00000000000..90f8be57686 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche_c/lifi_avalanche_c_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + schema = 'lifi_avalanche_c', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('avalanche_c') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7 -- WAVAX + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'avalanche_c' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'avalanche_c' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql new file mode 100644 index 00000000000..e0182da42f6 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + schema = 'lifi_bnb', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('bnb') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c -- WBNB + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'bnb' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'bnb' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql new file mode 100644 index 00000000000..3b9e32d6055 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + schema = 'lifi_ethereum', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('ethereum') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 -- WETH + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'ethereum' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'ethereum' + , columns = ['from'] + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql new file mode 100644 index 00000000000..51660d5e855 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + schema = 'lifi_fantom', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('fantom') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83 -- WFTM + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'fantom' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'fantom' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql new file mode 100644 index 00000000000..d52d1b9ec92 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + schema = 'lifi_gnosis', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('gnosis') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0xe91d153e0b41518a2ce8dd3d7944fa863463a97d -- WXDAI + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'gnosis' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'gnosis' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/lifi_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/lifi_transfers.sql new file mode 100644 index 00000000000..74c65ad2842 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/lifi_transfers.sql @@ -0,0 +1,61 @@ +{{ + config( + schema = 'lifi', + alias = 'transfers', + materialized = 'view', + post_hook='{{ expose_spells(\'[ + "arbitrum" + , "avalanche_c" + , "bnb" + , "ethereum" + , "fantom" + , "gnosis" + , "zksync" + ]\', + "project", + "lifi", + \'["lequangphu"]\') }}' + ) +}} + +{% set chains = [ + 'ethereum', + 'arbitrum', + 'avalanche_c', + 'bnb', + 'fantom', + 'gnosis', + 'zksync' +] %} + +with chain_transfers as ( + {% for chain in chains %} + select + contract_address, + tx_hash, + evt_index, + block_time, + block_number, + block_date, + transactionId, + bridge, + integrator, + referrer, + sendingAssetId, + receiver, + minAmount, + destinationChainId, + source_chain, + transfer_id, + sendingAssetId_adjusted, + amount_usd, + tx_from + from {{ ref('lifi_' ~ chain ~ '_transfers') }} + {% if not loop.last %} + union all + {% endif %} + {% endfor %} +) + +select * +from chain_transfers \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml new file mode 100644 index 00000000000..b24cab85118 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml @@ -0,0 +1,73 @@ +version: 2 + +models: + - name: lifi_ethereum_transfers + description: "Ethereum LiFi transfer events" + columns: &common_columns + - name: contract_address + description: "Contract address of LiFi Diamond" + - name: tx_hash + description: "Transaction hash" + - name: evt_index + description: "Event index" + - name: block_time + description: "Timestamp of the block" + - name: block_number + description: "Block number" + - name: transactionId + description: "Unique transaction ID from LiFi" + - name: bridge + description: "Bridge used for the transfer" + - name: integrator + description: "Integrator of the transfer" + - name: referrer + description: "Referrer of the transfer" + - name: sendingAssetId + description: "Asset being transferred" + - name: receiver + description: "Address receiving the transfer" + - name: minAmount + description: "Minimum amount to be received" + - name: destinationChainId + description: "Target chain ID for the transfer" + - name: source_chain + description: "Source blockchain" + - name: transfer_id + description: "Unique identifier for each transfer" + data_tests: + - unique + - not_null + - name: amount_usd + description: "USD value of the transfer amount at the time of transfer" + - name: sendingAssetId_adjusted + description: "Sending asset address adjusted for native token price lookup" + - name: tx_from + description: "Address that initiated the transaction" + + - name: lifi_arbitrum_transfers + description: "Arbitrum LiFi transfer events" + columns: *common_columns + + - name: lifi_avalanche_c_transfers + description: "Avalanche C-Chain LiFi transfer events" + columns: *common_columns + + - name: lifi_bnb_transfers + description: "BNB LiFi transfer events" + columns: *common_columns + + - name: lifi_fantom_transfers + description: "Fantom LiFi transfer events" + columns: *common_columns + + - name: lifi_gnosis_transfers + description: "Gnosis LiFi transfer events" + columns: *common_columns + + - name: lifi_zksync_transfers + description: "zkSync LiFi transfer events" + columns: *common_columns + + - name: lifi_transfers + description: "Combined LiFi transfer events across all supported chains" + columns: *common_columns \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql new file mode 100644 index 00000000000..d1560138355 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + schema = 'lifi_zksync', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with source_data as ( + {{ lifi_extract_bridge_data('zksync') }} +), + +tokens_mapped as ( + select + *, + case + when sendingAssetId = 0x0000000000000000000000000000000000000000 + then 0x000000000000000000000000000000000000800a -- WETH + else sendingAssetId + end as sendingAssetId_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * minAmount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.sendingAssetId_adjusted + and p.blockchain = 'zksync' + and p.minute = date_trunc('minute', tokens_mapped.block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +{{ + add_tx_columns( + model_cte = 'price_data' + , blockchain = 'zksync' + , columns = ['from'] + ) +}} diff --git a/sources/lifi/sources.yml b/sources/lifi/sources.yml new file mode 100644 index 00000000000..87486c607d9 --- /dev/null +++ b/sources/lifi/sources.yml @@ -0,0 +1,56 @@ +version: 2 + +sources: + - name: lifi_ethereum + description: "Ethereum LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: &common_columns + - name: contract_address + description: "Contract address of the LiFi Diamond" + - name: evt_tx_hash + description: "Transaction hash of the event" + - name: evt_index + description: "Event index in the transaction" + - name: evt_block_time + description: "Timestamp of the block when the event was emitted" + - name: evt_block_number + description: "Block number when the event was emitted" + - name: bridgeData + description: "JSON data containing bridge transfer information including transactionId, bridge, integrator, referrer, sendingAssetId, receiver, minAmount, and destinationChainId" + + - name: lifi_arbitrum + description: "Arbitrum LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: *common_columns + + - name: lifi_avalanche_c + description: "Avalanche C-Chain LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: *common_columns + + - name: lifi_bnb + description: "BNB Chain LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: *common_columns + + - name: lifi_fantom + description: "Fantom LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: *common_columns + + - name: lifi_gnosis + description: "Gnosis Chain LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: *common_columns + + - name: lifi_zksync + description: "zkSync LiFi events" + tables: + - name: LiFiDiamond_v2_evt_LiFiTransferStarted + columns: *common_columns From 27841739699595d6f77632f0da74103855ca3ca6 Mon Sep 17 00:00:00 2001 From: AIDataMaster Date: Mon, 9 Dec 2024 19:13:46 +0200 Subject: [PATCH 06/59] Add New Event Models for GMX v2 Project (#7187) * Add New Event Models * Fixed error in OraclePriceUpdate event model * Fixed error in PoolAmountUpdated * Fix Error in SetUint Event model * Make small updates * empty commit to run action again * Modify gmx_v2_arbitrum_open_interest_updated to incremental view * Fix incremental strategy * Add 2 new Event Models * Added Incremental Materialization to Main Event Models * Updated OraclePriceUpdate Event Model * Empty commit * Removed tests for the market column This error arises because the other model has not been updated yet. I believe that is the reason. The corresponding table will be updated on schedule, and then the market column will no longer be NULL. * added data_tests --------- Co-authored-by: Huang Geyang --- .../arbitrum/gmx_v2_arbitrum_event_schema.yml | 422 +++++++++++++++++- .../gmx_v2_arbitrum_execution_fee_refund.sql | 147 ++++++ .../gmx_v2_arbitrum_keeper_execution_fee.sql | 147 ++++++ ...bitrum_open_interest_in_tokens_updated.sql | 187 ++++++++ .../gmx_v2_arbitrum_open_interest_updated.sql | 186 ++++++++ .../gmx_v2_arbitrum_oracle_price_update.sql | 157 +++++++ .../gmx_v2_arbitrum_order_cancelled.sql | 12 +- .../gmx_v2_arbitrum_order_created.sql | 12 +- .../gmx_v2_arbitrum_order_executed.sql | 12 +- .../arbitrum/gmx_v2_arbitrum_order_frozen.sql | 12 +- .../gmx_v2_arbitrum_order_updated.sql | 12 +- .../gmx_v2_arbitrum_pool_amount_updated.sql | 165 +++++++ .../gmx_v2_arbitrum_position_decrease.sql | 12 +- ...mx_v2_arbitrum_position_fees_collected.sql | 12 +- .../gmx_v2_arbitrum_position_fees_info.sql | 12 +- .../gmx_v2_arbitrum_position_increase.sql | 12 +- .../arbitrum/gmx_v2_arbitrum_set_uint.sql | 165 +++++++ .../gmx_v2_avalanche_c_event_schema.yml | 421 ++++++++++++++++- ...mx_v2_avalanche_c_execution_fee_refund.sql | 147 ++++++ ...mx_v2_avalanche_c_keeper_execution_fee.sql | 147 ++++++ ...nche_c_open_interest_in_tokens_updated.sql | 187 ++++++++ ...x_v2_avalanche_c_open_interest_updated.sql | 187 ++++++++ ...gmx_v2_avalanche_c_oracle_price_update.sql | 157 +++++++ .../gmx_v2_avalanche_c_order_cancelled.sql | 12 +- .../gmx_v2_avalanche_c_order_created.sql | 12 +- .../gmx_v2_avalanche_c_order_executed.sql | 12 +- .../gmx_v2_avalanche_c_order_frozen.sql | 12 +- .../gmx_v2_avalanche_c_order_updated.sql | 12 +- ...gmx_v2_avalanche_c_pool_amount_updated.sql | 165 +++++++ .../gmx_v2_avalanche_c_position_decrease.sql | 12 +- ...v2_avalanche_c_position_fees_collected.sql | 12 +- .../gmx_v2_avalanche_c_position_fees_info.sql | 12 +- .../gmx_v2_avalanche_c_position_increase.sql | 12 +- .../gmx_v2_avalanche_c_set_uint.sql | 165 +++++++ .../gmx/event/gmx_v2_event_schema.yml | 350 ++++++++++++++- .../gmx/event/gmx_v2_execution_fee_refund.sql | 36 ++ .../gmx/event/gmx_v2_keeper_execution_fee.sql | 36 ++ ...gmx_v2_open_interest_in_tokens_updated.sql | 40 ++ .../event/gmx_v2_open_interest_updated.sql | 41 ++ .../gmx/event/gmx_v2_oracle_price_update.sql | 40 ++ .../gmx/event/gmx_v2_pool_amount_updated.sql | 39 ++ .../gmx/event/gmx_v2_position_decrease.sql | 2 +- .../_projects/gmx/event/gmx_v2_set_uint.sql | 38 ++ 43 files changed, 3907 insertions(+), 83 deletions(-) create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_execution_fee_refund.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_keeper_execution_fee.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_in_tokens_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_oracle_price_update.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_pool_amount_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_set_uint.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_execution_fee_refund.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_keeper_execution_fee.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_in_tokens_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_oracle_price_update.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_pool_amount_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_set_uint.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_execution_fee_refund.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_keeper_execution_fee.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_in_tokens_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_oracle_price_update.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_pool_amount_updated.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_set_uint.sql diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_event_schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_event_schema.yml index a234b79c6fb..dfc436cc271 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_event_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_event_schema.yml @@ -13,7 +13,11 @@ models: Extracts and decodes the `OrderCreated` event log data from the Arbitrum blockchain. This model processes log entries related to created orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - &blockchain name: blockchain @@ -187,7 +191,11 @@ models: Extracts and decodes the `OrderExecuted` event log data from the Arbitrum blockchain. This model processes log entries related to executed orders, extracting key variables such as account addresses, event keys, and integer values. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -230,7 +238,11 @@ models: Extracts and decodes the `OrderCancelled` event log data from the Arbitrum blockchain. This model processes log entries related to cancelled orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -344,7 +356,11 @@ models: This model extracts and parses `PositionIncrease` events from the Arbitrum blockchain. It includes transaction details, addresses, and various metrics related to the event, such as market data, collateral amounts, price impacts, and more. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -477,7 +493,11 @@ models: This model extracts and parses `PositionDecrease` events from the Arbitrum blockchain. It includes transaction details, addresses, and various metrics related to the event, such as market data, collateral amounts, price impacts, and more. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -558,7 +578,11 @@ models: This model extracts and parses `PositionFeesCollected` events from the Arbitrum blockchain. It includes details about the transaction, addresses involved, and various metrics related to fees collected from positions, including fee amounts and related market data. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -775,7 +799,11 @@ models: This model extracts and parses `PositionFeesInfo` events from the Arbitrum blockchain. It includes details about the transaction, addresses involved, and various metrics related to position fees, including fee amounts and related market data. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -858,8 +886,7 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - tx_hash - - key - + - index columns: - *blockchain - *block_time @@ -903,7 +930,11 @@ models: Extracts and decodes the `OrderUpdated` event log data from the Arbitrum blockchain. This model processes log entries related to updated orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -925,8 +956,6 @@ models: - not_null - name: market description: The market in which the order was updated - data_tests: - - not_null - name: account description: The address associated with the order. - *size_delta_usd @@ -939,4 +968,371 @@ models: - name: valid_from_time description: The time from which the order becomes valid for execution. - name: auto_cancel - description: Boolean indicating whether the order will be automatically canceled under certain conditions. \ No newline at end of file + description: Boolean indicating whether the order will be automatically canceled under certain conditions. + + + - name: gmx_v2_arbitrum_open_interest_in_tokens_updated + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'open_interest_in_tokens_updated'] + description: | + Combines and processes decoded event data to produce a normalized dataset of updated orders + on the GMX platform for the Arbitrum blockchain. + This model integrates data from the `open_interest_in_tokens_updated` table, enriching it with market-specific details. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OpenInterestInTokensUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market in which the order was updated, decoded from a hexadecimal value. + data_tests: + - not_null + - name: collateral_token + description: The token used as collateral for the order, decoded from a hexadecimal value. + data_tests: + - not_null + - name: is_long + description: A boolean flag indicating whether the order is a long position (true) or short position (false). + - name: next_value + description: | + Represents the updated open interest vlaue after the transaction. It is derived by normalizing + the raw value using index token decimals. + This ensures consistent scaling of the value across tokens with varying decimals. + - name: delta + description: | + Indicates the change in open interest. It is calculated by normalizing the raw delta value + using index token decimals. + This normalization accounts for token decimal differences and provides a standardized measure of change. + - name: tx_from + description: The address that initiated the transaction + - name: tx_to + description: The destination address of the transaction + - name: tx_index + description: Index value related to the transaction + + + - name: gmx_v2_arbitrum_open_interest_updated + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'open_interest_updated'] + description: | + Extracts and processes decoded event data for open interest updates on the GMX platform, + specifically on the Arbitrum blockchain. This model normalizes data, including key metrics like + delta and next_value, ensuring consistent scaling based on token decimals. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OpenInterestUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market associated with the open interest update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: collateral_token + description: The token used as collateral for the open interest, decoded from a hexadecimal value. + data_tests: + - not_null + - name: is_long + description: | + A boolean flag indicating whether the open interest update is for a long position (true) or short position (false). + data_type: boolean + - name: delta + description: | + Represents the change in open interest value, normalized by dividing the raw delta by `POWER(10, 30)`. + This ensures consistent scaling for values across tokens with varying decimals. + data_type: float + data_tests: + - not_null + - name: next_value + description: | + Indicates the updated open interest value after the transaction, normalized by dividing + the raw value by `POWER(10, 30)` to ensure consistent scaling across tokens. + data_type: float + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction. + + + - name: gmx_v2_arbitrum_oracle_price_update + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'oracle_price_update'] + description: | + Processes decoded event data for Oracle Price Updates on the GMX platform in the Arbitrum blockchain. + The model normalizes price data (min_price and max_price) using token-specific decimals and extracts + additional details such as the provider and token involved in the update. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OraclePriceUpdate' for this model + data_tests: + - not_null + - *msg_sender + - name: token + description: The token associated with the oracle price update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: provider + description: The address of the price provider, decoded from a hexadecimal value. + - name: min_price + description: The minimum price reported by the oracle. + data_tests: + - not_null + - name: max_price + description: The maximum price reported by the oracle. + data_tests: + - not_null + - name: timestamp + description: The timestamp of the oracle price update. + data_type: timestamp + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_arbitrum_pool_amount_updated + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'pool_amount_updated'] + description: | + Processes decoded event data for Pool Amount Updates on the GMX platform in the Arbitrum blockchain. + This model normalizes token values (next_value and delta) based on token-specific decimals and extracts + additional details such as the market and token involved in the pool update. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'PoolAmountUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market associated with the pool update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: token + description: The token involved in the pool update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: next_value + description: | + The amount in the pool for the specified token, normalized by dividing the raw value + by `POWER(10, token_decimals)`, where `token_decimals` is derived from ERC-20 token metadata. + data_tests: + - not_null + - name: delta + description: | + The change in the pool amount for the specified token, normalized by dividing the raw value + by `POWER(10, token_decimals)`, where `token_decimals` is derived from ERC-20 token metadata. + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_arbitrum_set_uint + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'set_uint'] + description: | + Processes decoded event data for `SetUint` events on the GMX platform in the Arbitrum blockchain. + This model normalizes the value field and extracts additional details such as the base key and associated data, + ensuring compatibility with downstream analyses. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'SetUint' for this model + data_tests: + - not_null + - *msg_sender + - name: base_key + description: The base key associated with the event, decoded from a hexadecimal value. + data_tests: + - not_null + - name: data + description: The associated data field, decoded from a hexadecimal value. + data_tests: + - not_null + - name: value_raw + description: | + The value associated with the event. + data_type: float + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_arbitrum_execution_fee_refund + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'execution_fee_refund'] + description: | + Processes decoded event data for `ExecutionFeeRefund` events on the GMX platform in the Arbitrum blockchain. + This model extracts key details such as the refund amount, receiver, and transaction details to + enable analysis of refunded execution fees. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: event_name + description: The type of event recorded, always `ExecutionFeeRefund` for this model. + data_tests: + - not_null + - *msg_sender + - name: receiver + description: The address that received the refunded execution fee. + - name: refund_fee_amount + description: The amount of the refunded execution fee, denominated in the native token of the blockchain (e.g., ETH). + + + - name: gmx_v2_arbitrum_keeper_execution_fee + meta: + blockchain: arbitrum + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'gmx', 'event', 'keeper_execution_fee'] + description: | + Processes decoded event data for `KeeperExecutionFee` events on the GMX platform in the Arbitrum blockchain. + This model extracts key details such as the keeper's address, execution fee amount, and transaction details, enabling + analysis of fees associated with keeper-based executions. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: event_name + description: The type of event recorded, always `KeeperExecutionFee` for this model. + data_tests: + - not_null + - *msg_sender + - name: keeper + description: The address of the keeper responsible for executing the transaction. + - name: execution_fee_amount + description: The amount of the execution fee, denominated in the native token of the blockchain (e.g., ETH, AVAX). diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_execution_fee_refund.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_execution_fee_refund.sql new file mode 100644 index 00000000000..c20cd8fe4f9 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_execution_fee_refund.sql @@ -0,0 +1,147 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'execution_fee_refund', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'ExecutionFeeRefund' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'receiver' THEN value END) AS receiver, + MAX(CASE WHEN key_name = 'refundFeeAmount' THEN value END) AS refund_fee_amount + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.receiver) AS receiver, + CAST(EDP.refund_fee_amount AS DOUBLE) / POWER(10, 18) AS refund_fee_amount + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_keeper_execution_fee.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_keeper_execution_fee.sql new file mode 100644 index 00000000000..e7f2e0ee0f8 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_keeper_execution_fee.sql @@ -0,0 +1,147 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'keeper_execution_fee', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'KeeperExecutionFee' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'keeper' THEN value END) AS keeper, + MAX(CASE WHEN key_name = 'executionFeeAmount' THEN value END) AS execution_fee_amount + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.keeper) AS keeper, + CAST(EDP.execution_fee_amount AS DOUBLE) / POWER(10, 18) AS execution_fee_amount + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_in_tokens_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_in_tokens_updated.sql new file mode 100644 index 00000000000..3d73a35efa3 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_in_tokens_updated.sql @@ -0,0 +1,187 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'open_interest_in_tokens_updated', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'OpenInterestInTokensUpdated' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items, + json_query(data, 'lax $.intItems' OMIT QUOTES) AS int_items, + json_query(data, 'lax $.boolItems' OMIT QUOTES) AS bool_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, int_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(int_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, bool_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bool_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed + UNION ALL + SELECT * + FROM int_items_parsed + UNION ALL + SELECT * + FROM bool_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'isLong' THEN value END) AS is_long, + MAX(CASE WHEN key_name = 'nextValue' THEN value END) AS next_value, + MAX(CASE WHEN key_name = 'market' THEN value END) AS market, + MAX(CASE WHEN key_name = 'collateralToken' THEN value END) AS collateral_token, + MAX(CASE WHEN key_name = 'delta' THEN value END) AS delta + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.market) AS market, + from_hex(EDP.collateral_token) AS collateral_token, + CAST(EDP.is_long AS BOOLEAN) AS is_long, + CAST(EDP.next_value AS DOUBLE) / POWER(10, MD.index_token_decimals) AS next_value, + CAST(EDP.delta AS DOUBLE) / POWER(10, MD.index_token_decimals) AS delta + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index + LEFT JOIN {{ ref('gmx_v2_arbitrum_markets_data') }} AS MD + ON from_hex(EDP.market) = MD.market +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_updated.sql new file mode 100644 index 00000000000..acbf9774525 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_open_interest_updated.sql @@ -0,0 +1,186 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'open_interest_updated', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'OpenInterestUpdated' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items, + json_query(data, 'lax $.intItems' OMIT QUOTES) AS int_items, + json_query(data, 'lax $.boolItems' OMIT QUOTES) AS bool_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, int_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(int_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, bool_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bool_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed + UNION ALL + SELECT * + FROM int_items_parsed + UNION ALL + SELECT * + FROM bool_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'market' THEN value END) AS market, + MAX(CASE WHEN key_name = 'collateralToken' THEN value END) AS collateral_token, + MAX(CASE WHEN key_name = 'isLong' THEN value END) AS is_long, + MAX(CASE WHEN key_name = 'delta' THEN value END) AS delta, + MAX(CASE WHEN key_name = 'nextValue' THEN value END) AS next_value + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(market) AS market, + from_hex(collateral_token) AS collateral_token, + TRY_CAST(is_long AS BOOLEAN) AS is_long, + TRY_CAST(delta AS DOUBLE) / POWER(10, 30) AS delta, + TRY_CAST(next_value AS DOUBLE) / POWER(10, 30) AS next_value + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_oracle_price_update.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_oracle_price_update.sql new file mode 100644 index 00000000000..1e9216afe01 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_oracle_price_update.sql @@ -0,0 +1,157 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'oracle_price_update', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'OraclePriceUpdate' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% else %} + AND evt_block_time > DATE '2023-08-01' + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% else %} + AND evt_block_time > DATE '2023-08-01' + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'token' THEN value END) AS token, + MAX(CASE WHEN key_name = 'provider' THEN value END) AS provider, + MAX(CASE WHEN key_name = 'minPrice' THEN value END) AS min_price, + MAX(CASE WHEN key_name = 'maxPrice' THEN value END) AS max_price, + MAX(CASE WHEN key_name = 'timestamp' THEN value END) AS "timestamp" + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + ED.blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + ED.contract_address, + event_name, + msg_sender, + + from_hex(token) AS token, + from_hex(provider) AS provider, + TRY_CAST(min_price AS DOUBLE) / POWER(10, 30 - ERC20.decimals) AS min_price, + TRY_CAST(max_price AS DOUBLE) / POWER(10, 30 - ERC20.decimals) AS max_price, + CASE + WHEN TRY_CAST("timestamp" AS DOUBLE) = 0 THEN NULL + ELSE TRY_CAST("timestamp" AS DOUBLE) + END AS "timestamp" + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index + LEFT JOIN {{ ref('gmx_v2_arbitrum_erc20') }} AS ERC20 + ON from_hex(EDP.token) = ERC20.contract_address +) + +-- can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_cancelled.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_cancelled.sql index 84fe7f7b326..b24ca5fafe2 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_cancelled.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_cancelled.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'order_cancelled', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_created.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_created.sql index 37823907ffc..12f38efb078 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_created.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_created.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'order_created', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_executed.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_executed.sql index b373d356708..a7a9c6c1c62 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_executed.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_executed.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'order_executed', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_frozen.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_frozen.sql index b0cc9d09f62..289f146dcb3 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_frozen.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_frozen.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'order_frozen', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_updated.sql index edb33308631..38d6a895854 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_updated.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_order_updated.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'order_updated', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_pool_amount_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_pool_amount_updated.sql new file mode 100644 index 00000000000..d1cba68b7fa --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_pool_amount_updated.sql @@ -0,0 +1,165 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'pool_amount_updated', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'PoolAmountUpdated' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items, + json_query(data, 'lax $.intItems' OMIT QUOTES) AS int_items + FROM + evt_data +) +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, int_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(int_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed + UNION ALL + SELECT * + FROM int_items_parsed +) +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'market' THEN value END) AS market, + MAX(CASE WHEN key_name = 'token' THEN value END) AS token, + MAX(CASE WHEN key_name = 'delta' THEN value END) AS delta, + MAX(CASE WHEN key_name = 'nextValue' THEN value END) AS next_value + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + ED.blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + ED.contract_address, + event_name, + msg_sender, + + from_hex(market) AS market, + from_hex(token) AS token, + TRY_CAST(next_value AS DOUBLE) / POWER(10, ERC20.decimals) AS next_value, + TRY_CAST(delta AS DOUBLE) / POWER(10, ERC20.decimals) AS delta + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index + LEFT JOIN {{ ref('gmx_v2_arbitrum_erc20') }} AS ERC20 + ON from_hex(EDP.token) = ERC20.contract_address +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_decrease.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_decrease.sql index d5cfa5c134e..b05f68e1501 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_decrease.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_decrease.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'position_decrease', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_collected.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_collected.sql index 7019c6c497d..5b142f5b386 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_collected.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_collected.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'position_fees_collected', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_info.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_info.sql index 6665accfbd1..8502f185856 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_info.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_fees_info.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'position_fees_info', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_increase.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_increase.sql index bddd6f5c9f6..1129f3b860f 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_increase.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_position_increase.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_arbitrum', alias = 'position_increase', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_set_uint.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_set_uint.sql new file mode 100644 index 00000000000..c3a1085676b --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/arbitrum/gmx_v2_arbitrum_set_uint.sql @@ -0,0 +1,165 @@ +{{ + config( + schema = 'gmx_v2_arbitrum', + alias = 'set_uint', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'SetUint' -%} +{%- set blockchain_name = 'arbitrum' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_arbitrum','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.bytes32Items' OMIT QUOTES) AS bytes32_items, + json_query(data, 'lax $.bytesItems' OMIT QUOTES) AS bytes_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) + +, bytes32_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bytes32_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, bytes_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bytes_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM bytes32_items_parsed + UNION ALL + SELECT * + FROM bytes_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'baseKey' THEN value END) AS base_key, + MAX(CASE WHEN key_name = 'data' THEN value END) AS "data", + MAX(CASE WHEN key_name = 'value' THEN value END) AS "value" + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.base_key) AS base_key, + from_hex(EDP."data") AS "data", + TRY_CAST(EDP."value" AS DOUBLE) AS "value_raw" + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_event_schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_event_schema.yml index ba40d0e98e5..bd0636b0db1 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_event_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_event_schema.yml @@ -13,7 +13,11 @@ models: Extracts and decodes the `OrderCreated` event log data from the Avalanche blockchain. This model processes log entries related to created orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - &blockchain name: blockchain @@ -187,7 +191,11 @@ models: Extracts and decodes the `OrderExecuted` event log data from the Avalanche blockchain. This model processes log entries related to executed orders, extracting key variables such as account addresses, event keys, and integer values. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -230,7 +238,11 @@ models: Extracts and decodes the `OrderCancelled` event log data from the Avalanche blockchain. This model processes log entries related to cancelled orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -344,7 +356,11 @@ models: This model extracts and parses `PositionIncrease` events from the Avalanche blockchain. It includes transaction details, addresses, and various metrics related to the event, such as market data, collateral amounts, price impacts, and more. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -477,7 +493,11 @@ models: This model extracts and parses `PositionDecrease` events from the Avalanche blockchain. It includes transaction details, addresses, and various metrics related to the event, such as market data, collateral amounts, price impacts, and more. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -558,7 +578,11 @@ models: This model extracts and parses `PositionFeesCollected` events from the Avalanche blockchain. It includes details about the transaction, addresses involved, and various metrics related to fees collected from positions, including fee amounts and related market data. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -775,7 +799,11 @@ models: This model extracts and parses `PositionFeesInfo` events from the Avalanche blockchain. It includes details about the transaction, addresses involved, and various metrics related to position fees, including fee amounts and related market data. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -858,7 +886,7 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - tx_hash - - key + - index columns: - *blockchain @@ -903,7 +931,11 @@ models: Extracts and decodes the `OrderUpdated` event log data from the Avalanche blockchain. This model processes log entries related to updated orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. - + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - *blockchain - *block_time @@ -925,8 +957,6 @@ models: - not_null - name: market description: The market in which the order was updated. - data_tests: - - not_null - name: account description: The address associated with the order. - *size_delta_usd @@ -939,4 +969,371 @@ models: - name: valid_from_time description: The time from which the order becomes valid for execution. - name: auto_cancel - description: Boolean indicating whether the order will be automatically canceled under certain conditions. \ No newline at end of file + description: Boolean indicating whether the order will be automatically canceled under certain conditions. + + + - name: gmx_v2_avalanche_c_open_interest_in_tokens_updated + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'open_interest_in_tokens_updated'] + description: | + Combines and processes decoded event data to produce a normalized dataset of updated orders + on the GMX platform for the Avalanche blockchain. + This model integrates data from the `open_interest_in_tokens_updated` table, enriching it with market-specific details. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OpenInterestInTokensUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market in which the order was updated, decoded from a hexadecimal value. + data_tests: + - not_null + - name: collateral_token + description: The token used as collateral for the order, decoded from a hexadecimal value. + data_tests: + - not_null + - name: is_long + description: A boolean flag indicating whether the order is a long position (true) or short position (false). + - name: next_value + description: | + Represents the updated open interest vlaue after the transaction. It is derived by normalizing + the raw value using index token decimals. + This ensures consistent scaling of the value across tokens with varying decimals. + - name: delta + description: | + Indicates the change in open interest. It is calculated by normalizing the raw delta value + using index token decimals. + This normalization accounts for token decimal differences and provides a standardized measure of change. + - name: tx_from + description: The address that initiated the transaction + - name: tx_to + description: The destination address of the transaction + - name: tx_index + description: Index value related to the transaction + + + - name: gmx_v2_avalanche_c_open_interest_updated + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'open_interest_updated'] + description: | + Extracts and processes decoded event data for open interest updates on the GMX platform, + specifically on the Avalanche blockchain. This model normalizes data, including key metrics like + delta and next_value, ensuring consistent scaling based on token decimals. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OpenInterestUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market associated with the open interest update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: collateral_token + description: The token used as collateral for the open interest, decoded from a hexadecimal value. + data_tests: + - not_null + - name: is_long + description: | + A boolean flag indicating whether the open interest update is for a long position (true) or short position (false). + data_type: boolean + - name: delta + description: | + Represents the change in open interest value, normalized by dividing the raw delta by `POWER(10, 30)`. + This ensures consistent scaling for values across tokens with varying decimals. + data_type: float + data_tests: + - not_null + - name: next_value + description: | + Indicates the updated open interest value after the transaction, normalized by dividing + the raw value by `POWER(10, 30)` to ensure consistent scaling across tokens. + data_type: float + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction. + + + - name: gmx_v2_avalanche_c_oracle_price_update + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'oracle_price_update'] + description: | + Processes decoded event data for Oracle Price Updates on the GMX platform in the Avalanche blockchain. + The model normalizes price data (min_price and max_price) using token-specific decimals and extracts + additional details such as the provider and token involved in the update. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OraclePriceUpdate' for this model + data_tests: + - not_null + - *msg_sender + - name: token + description: The token associated with the oracle price update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: provider + description: The address of the price provider, decoded from a hexadecimal value. + - name: min_price + description: The minimum price reported by the oracle. + data_tests: + - not_null + - name: max_price + description: The maximum price reported by the oracle. + data_tests: + - not_null + - name: timestamp + description: The timestamp of the oracle price update. + data_type: timestamp + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_avalanche_c_pool_amount_updated + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'pool_amount_updated'] + description: | + Processes decoded event data for Pool Amount Updates on the GMX platform in the Avalanche blockchain. + This model normalizes token values (next_value and delta) based on token-specific decimals and extracts + additional details such as the market and token involved in the pool update. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'PoolAmountUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market associated with the pool update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: token + description: The token involved in the pool update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: next_value + description: | + The amount in the pool for the specified token, normalized by dividing the raw value + by `POWER(10, token_decimals)`, where `token_decimals` is derived from ERC-20 token metadata. + data_tests: + - not_null + - name: delta + description: | + The change in the pool amount for the specified token, normalized by dividing the raw value + by `POWER(10, token_decimals)`, where `token_decimals` is derived from ERC-20 token metadata. + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_avalanche_c_set_uint + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'set_uint'] + description: | + Processes decoded event data for `SetUint` events on the GMX platform in the Avalanche blockchain. + This model normalizes the value field and extracts additional details such as the base key and associated data, + ensuring compatibility with downstream analyses. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'SetUint' for this model + data_tests: + - not_null + - *msg_sender + - name: base_key + description: The base key associated with the event, decoded from a hexadecimal value. + data_tests: + - not_null + - name: data + description: The associated data field, decoded from a hexadecimal value. + data_tests: + - not_null + - name: value_raw + description: | + The value associated with the event. + data_type: float + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_avalanche_c_execution_fee_refund + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'execution_fee_refund'] + description: | + Processes decoded event data for `ExecutionFeeRefund` events on the GMX platform in the Avalanche blockchain. + This model extracts key details such as the refund amount, receiver, and transaction details to + enable analysis of refunded execution fees. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: event_name + description: The type of event recorded, always `ExecutionFeeRefund` for this model. + data_tests: + - not_null + - *msg_sender + - name: receiver + description: The address that received the refunded execution fee. + - name: refund_fee_amount + description: The amount of the refunded execution fee, denominated in the native token of the blockchain (e.g., ETH). + + + - name: gmx_v2_avalanche_c_keeper_execution_fee + meta: + blockchain: avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['avalanche_c', 'gmx', 'event', 'keeper_execution_fee'] + description: | + Processes decoded event data for `KeeperExecutionFee` events on the GMX platform in the Avalanche blockchain. + This model extracts key details such as the keeper's address, execution fee amount, and transaction details, enabling + analysis of fees associated with keeper-based executions. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: event_name + description: The type of event recorded, always `KeeperExecutionFee` for this model. + data_tests: + - not_null + - *msg_sender + - name: keeper + description: The address of the keeper responsible for executing the transaction. + - name: execution_fee_amount + description: The amount of the execution fee, denominated in the native token of the blockchain (e.g., ETH, AVAX). diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_execution_fee_refund.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_execution_fee_refund.sql new file mode 100644 index 00000000000..2fa2e95fe43 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_execution_fee_refund.sql @@ -0,0 +1,147 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'execution_fee_refund', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'ExecutionFeeRefund' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'receiver' THEN value END) AS receiver, + MAX(CASE WHEN key_name = 'refundFeeAmount' THEN value END) AS refund_fee_amount + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.receiver) AS receiver, + CAST(EDP.refund_fee_amount AS DOUBLE) / POWER(10, 18) AS refund_fee_amount + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_keeper_execution_fee.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_keeper_execution_fee.sql new file mode 100644 index 00000000000..3a6c16d6009 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_keeper_execution_fee.sql @@ -0,0 +1,147 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'keeper_execution_fee', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'KeeperExecutionFee' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'keeper' THEN value END) AS keeper, + MAX(CASE WHEN key_name = 'executionFeeAmount' THEN value END) AS execution_fee_amount + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.keeper) AS keeper, + CAST(EDP.execution_fee_amount AS DOUBLE) / POWER(10, 18) AS execution_fee_amount + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_in_tokens_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_in_tokens_updated.sql new file mode 100644 index 00000000000..d601703949e --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_in_tokens_updated.sql @@ -0,0 +1,187 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'open_interest_in_tokens_updated', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'OpenInterestInTokensUpdated' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items, + json_query(data, 'lax $.intItems' OMIT QUOTES) AS int_items, + json_query(data, 'lax $.boolItems' OMIT QUOTES) AS bool_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, int_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(int_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, bool_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bool_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed + UNION ALL + SELECT * + FROM int_items_parsed + UNION ALL + SELECT * + FROM bool_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'isLong' THEN value END) AS is_long, + MAX(CASE WHEN key_name = 'nextValue' THEN value END) AS next_value, + MAX(CASE WHEN key_name = 'market' THEN value END) AS market, + MAX(CASE WHEN key_name = 'collateralToken' THEN value END) AS collateral_token, + MAX(CASE WHEN key_name = 'delta' THEN value END) AS delta + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.market) AS market, + from_hex(EDP.collateral_token) AS collateral_token, + CAST(EDP.is_long AS BOOLEAN) AS is_long, + CAST(EDP.next_value AS DOUBLE) / POWER(10, MD.index_token_decimals) AS next_value, + CAST(EDP.delta AS DOUBLE) / POWER(10, MD.index_token_decimals) AS delta + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index + LEFT JOIN {{ ref('gmx_v2_avalanche_c_markets_data') }} AS MD + ON from_hex(EDP.market) = MD.market +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_updated.sql new file mode 100644 index 00000000000..cd88f284349 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_open_interest_updated.sql @@ -0,0 +1,187 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'open_interest_updated', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'OpenInterestUpdated' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items, + json_query(data, 'lax $.intItems' OMIT QUOTES) AS int_items, + json_query(data, 'lax $.boolItems' OMIT QUOTES) AS bool_items + FROM + evt_data +) + +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, int_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(int_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, bool_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bool_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed + UNION ALL + SELECT * + FROM int_items_parsed + UNION ALL + SELECT * + FROM bool_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'market' THEN value END) AS market, + MAX(CASE WHEN key_name = 'collateralToken' THEN value END) AS collateral_token, + MAX(CASE WHEN key_name = 'isLong' THEN value END) AS is_long, + MAX(CASE WHEN key_name = 'delta' THEN value END) AS delta, + MAX(CASE WHEN key_name = 'nextValue' THEN value END) AS next_value + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(market) AS market, + from_hex(collateral_token) AS collateral_token, + TRY_CAST(is_long AS BOOLEAN) AS is_long, + TRY_CAST(delta AS DOUBLE) / POWER(10, 30) AS delta, + TRY_CAST(next_value AS DOUBLE) / POWER(10, 30) AS next_value + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_oracle_price_update.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_oracle_price_update.sql new file mode 100644 index 00000000000..14f2540c8f6 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_oracle_price_update.sql @@ -0,0 +1,157 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'oracle_price_update', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'OraclePriceUpdate' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% else %} + AND evt_block_time > DATE '2023-08-01' + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% else %} + AND evt_block_time > DATE '2023-08-01' + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'token' THEN value END) AS token, + MAX(CASE WHEN key_name = 'provider' THEN value END) AS provider, + MAX(CASE WHEN key_name = 'minPrice' THEN value END) AS min_price, + MAX(CASE WHEN key_name = 'maxPrice' THEN value END) AS max_price, + MAX(CASE WHEN key_name = 'timestamp' THEN value END) AS "timestamp" + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + ED.blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + ED.contract_address, + event_name, + msg_sender, + + from_hex(token) AS token, + from_hex(provider) AS provider, + TRY_CAST(min_price AS DOUBLE) / POWER(10, 30 - ERC20.decimals) AS min_price, + TRY_CAST(max_price AS DOUBLE) / POWER(10, 30 - ERC20.decimals) AS max_price, + CASE + WHEN TRY_CAST("timestamp" AS DOUBLE) = 0 THEN NULL + ELSE TRY_CAST("timestamp" AS DOUBLE) + END AS "timestamp" + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index + LEFT JOIN {{ ref('gmx_v2_avalanche_c_erc20') }} AS ERC20 + ON from_hex(EDP.token) = ERC20.contract_address +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_cancelled.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_cancelled.sql index ce8206ba85c..e974c4d20e5 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_cancelled.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_cancelled.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'order_cancelled', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_created.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_created.sql index ba428954368..9822561b7bc 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_created.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_created.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'order_created', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_executed.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_executed.sql index 677afcce2bd..d0f775d745e 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_executed.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_executed.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'order_executed', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_frozen.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_frozen.sql index 0de9538f4a7..6e9b7ce230c 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_frozen.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_frozen.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'order_frozen', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_updated.sql index 0f56ec17a91..76d244245ce 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_updated.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_order_updated.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'order_updated', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_pool_amount_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_pool_amount_updated.sql new file mode 100644 index 00000000000..17c7d2934da --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_pool_amount_updated.sql @@ -0,0 +1,165 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'pool_amount_updated', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'PoolAmountUpdated' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.addressItems' OMIT QUOTES) AS address_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items, + json_query(data, 'lax $.intItems' OMIT QUOTES) AS int_items + FROM + evt_data +) +, address_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(address_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, int_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(int_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) +, combined AS ( + SELECT * + FROM address_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed + UNION ALL + SELECT * + FROM int_items_parsed +) +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'market' THEN value END) AS market, + MAX(CASE WHEN key_name = 'token' THEN value END) AS token, + MAX(CASE WHEN key_name = 'delta' THEN value END) AS delta, + MAX(CASE WHEN key_name = 'nextValue' THEN value END) AS next_value + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + ED.blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + ED.contract_address, + event_name, + msg_sender, + + from_hex(market) AS market, + from_hex(token) AS token, + TRY_CAST(next_value AS DOUBLE) / POWER(10, ERC20.decimals) AS next_value, + TRY_CAST(delta AS DOUBLE) / POWER(10, ERC20.decimals) AS delta + + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index + LEFT JOIN {{ ref('gmx_v2_avalanche_c_erc20') }} AS ERC20 + ON from_hex(EDP.token) = ERC20.contract_address +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_decrease.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_decrease.sql index e82cce31001..a5edbae87fc 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_decrease.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_decrease.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'position_decrease', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_collected.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_collected.sql index 5e19b65ebfd..ad4015404f2 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_collected.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_collected.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'position_fees_collected', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_info.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_info.sql index 6dff079f071..a4aae5dd017 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_info.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_fees_info.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'position_fees_info', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_increase.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_increase.sql index a4b6b107a40..52d9bd0cddb 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_increase.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_position_increase.sql @@ -2,7 +2,9 @@ config( schema = 'gmx_v2_avalanche_c', alias = 'position_increase', - materialized = 'table' + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' ) }} @@ -23,7 +25,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) , evt_data_2 AS ( @@ -40,7 +44,9 @@ WITH evt_data_1 AS ( msgSender AS msg_sender FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} WHERE eventName = '{{ event_name }}' - ORDER BY evt_block_time ASC + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} ) -- unite 2 tables diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_set_uint.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_set_uint.sql new file mode 100644 index 00000000000..41e06b529d5 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/avalanche_c/gmx_v2_avalanche_c_set_uint.sql @@ -0,0 +1,165 @@ +{{ + config( + schema = 'gmx_v2_avalanche_c', + alias = 'set_uint', + materialized = 'incremental', + unique_key = ['tx_hash', 'index'], + incremental_strategy = 'merge' + ) +}} + +{%- set event_name = 'SetUint' -%} +{%- set blockchain_name = 'avalanche_c' -%} + + +WITH evt_data_1 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog1')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +, evt_data_2 AS ( + SELECT + -- Main Variables + '{{ blockchain_name }}' AS blockchain, + evt_block_time AS block_time, + evt_block_number AS block_number, + evt_tx_hash AS tx_hash, + evt_index AS index, + contract_address, + eventName AS event_name, + eventData AS data, + msgSender AS msg_sender + FROM {{ source('gmx_v2_avalanche_c','EventEmitter_evt_EventLog2')}} + WHERE eventName = '{{ event_name }}' + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} +) + +-- unite 2 tables +, evt_data AS ( + SELECT * + FROM evt_data_1 + UNION + SELECT * + FROM evt_data_2 +) + +, parsed_data AS ( + SELECT + tx_hash, + index, + json_query(data, 'lax $.bytes32Items' OMIT QUOTES) AS bytes32_items, + json_query(data, 'lax $.bytesItems' OMIT QUOTES) AS bytes_items, + json_query(data, 'lax $.uintItems' OMIT QUOTES) AS uint_items + FROM + evt_data +) + +, bytes32_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bytes32_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, bytes_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(bytes_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, uint_items_parsed AS ( + SELECT + tx_hash, + index, + json_extract_scalar(CAST(item AS VARCHAR), '$.key') AS key_name, + json_extract_scalar(CAST(item AS VARCHAR), '$.value') AS value + FROM + parsed_data, + UNNEST( + CAST(json_extract(uint_items, '$.items') AS ARRAY(JSON)) + ) AS t(item) +) + +, combined AS ( + SELECT * + FROM bytes32_items_parsed + UNION ALL + SELECT * + FROM bytes_items_parsed + UNION ALL + SELECT * + FROM uint_items_parsed +) + +, evt_data_parsed AS ( + SELECT + tx_hash, + index, + MAX(CASE WHEN key_name = 'baseKey' THEN value END) AS base_key, + MAX(CASE WHEN key_name = 'data' THEN value END) AS "data", + MAX(CASE WHEN key_name = 'value' THEN value END) AS "value" + FROM + combined + GROUP BY tx_hash, index +) + +-- full data +, full_data AS ( + SELECT + blockchain, + block_time, + DATE(block_time) AS block_date, + block_number, + ED.tx_hash, + ED.index, + contract_address, + event_name, + msg_sender, + + from_hex(EDP.base_key) AS base_key, + from_hex(EDP."data") AS "data", + TRY_CAST(EDP."value" AS DOUBLE) AS "value_raw" + FROM evt_data AS ED + LEFT JOIN evt_data_parsed AS EDP + ON ED.tx_hash = EDP.tx_hash + AND ED.index = EDP.index +) + +--can be removed once decoded tables are fully denormalized +{{ + add_tx_columns( + model_cte = 'full_data' + , blockchain = blockchain_name + , columns = ['from', 'to', 'index'] + ) +}} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_event_schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_event_schema.yml index c772b056d86..39ea55ccbab 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_event_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_event_schema.yml @@ -13,6 +13,11 @@ models: Extracts and decodes the `OrderCreated` event log data from the Arbitrum and Avalanche blockchains. This model processes log entries related to created orders, extracting key variables such as account addresses, event keys, and strings representing the reason for cancellation. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - index columns: - &blockchain @@ -925,8 +930,6 @@ models: - not_null - name: market description: The market in which the order was updated. - data_tests: - - not_null - name: account description: The address associated with the order. - *size_delta_usd @@ -939,4 +942,345 @@ models: - name: valid_from_time description: The time from which the order becomes valid for execution. - name: auto_cancel - description: Boolean indicating whether the order will be automatically canceled under certain conditions. \ No newline at end of file + description: Boolean indicating whether the order will be automatically canceled under certain conditions. + + + - name: gmx_v2_open_interest_in_tokens_updated + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'open_interest_in_tokens_updated'] + description: | + Combines and processes decoded event data to produce a normalized dataset of updated orders + on the GMX platform for the Arbitrum and Avalanche blockchains. + This model integrates data from the `open_interest_in_tokens_updated` table, enriching it with market-specific details. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OpenInterestInTokensUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market in which the order was updated, decoded from a hexadecimal value. + data_tests: + - not_null + - name: collateral_token + description: The token used as collateral for the order, decoded from a hexadecimal value. + data_tests: + - not_null + - name: is_long + description: A boolean flag indicating whether the order is a long position (true) or short position (false). + - name: next_value + description: | + Represents the updated open interest vlaue after the transaction. It is derived by normalizing + the raw value using index token decimals. + This ensures consistent scaling of the value across tokens with varying decimals. + - name: delta + description: | + Indicates the change in open interest. It is calculated by normalizing the raw delta value + using index token decimals. + This normalization accounts for token decimal differences and provides a standardized measure of change. + - name: tx_from + description: The address that initiated the transaction + - name: tx_to + description: The destination address of the transaction + - name: tx_index + description: Index value related to the transaction + + + - name: gmx_v2_open_interest_updated + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'open_interest_updated'] + description: | + Extracts and processes decoded event data for open interest updates on the GMX platform, + specifically on the Arbitrum and Avalanche blockchains. This model normalizes data, including key metrics like + delta and next_value, ensuring consistent scaling based on token decimals. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OpentInterestUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market associated with the open interest update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: collateral_token + description: The token used as collateral for the open interest, decoded from a hexadecimal value. + data_tests: + - not_null + - name: is_long + description: | + A boolean flag indicating whether the open interest update is for a long position (true) or short position (false). + data_type: boolean + - name: delta + description: | + Represents the change in open interest value, normalized by dividing the raw delta by `POWER(10, 30)`. + This ensures consistent scaling for values across tokens with varying decimals. + data_type: float + data_tests: + - not_null + - name: next_value + description: | + Indicates the updated open interest value after the transaction, normalized by dividing + the raw value by `POWER(10, 30)` to ensure consistent scaling across tokens. + data_type: float + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction. + + + - name: gmx_v2_oracle_price_update + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'oracle_price_update'] + description: | + Processes decoded event data for Oracle Price Updates on the GMX platform in the Arbitrum and Avalanche blockchains. + The model normalizes price data (min_price and max_price) using token-specific decimals and extracts + additional details such as the provider and token involved in the update. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'OraclePriceUpdate' for this model + data_tests: + - not_null + - *msg_sender + - name: token + description: The token associated with the oracle price update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: provider + description: The address of the price provider, decoded from a hexadecimal value. + - name: min_price + description: The minimum price reported by the oracle. + data_tests: + - not_null + - name: max_price + description: The maximum price reported by the oracle. + data_tests: + - not_null + - name: timestamp + description: The timestamp of the oracle price update. + data_type: timestamp + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_pool_amount_updated + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'pool_amount_updated'] + description: | + Processes decoded event data for Pool Amount Updates on the GMX platform in the Arbitrum and Avalanche blockchains. + This model normalizes token values (next_value and delta) based on token-specific decimals and extracts + additional details such as the market and token involved in the pool update. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'PoolAmountUpdated' for this model + data_tests: + - not_null + - *msg_sender + - name: market + description: The market associated with the pool update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: token + description: The token involved in the pool update, decoded from a hexadecimal value. + data_tests: + - not_null + - name: next_value + description: | + The amount in the pool for the specified token, normalized by dividing the raw value + by `POWER(10, token_decimals)`, where `token_decimals` is derived from ERC-20 token metadata. + data_tests: + - not_null + - name: delta + description: | + The change in the pool amount for the specified token, normalized by dividing the raw value + by `POWER(10, token_decimals)`, where `token_decimals` is derived from ERC-20 token metadata. + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_set_uint + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'set_uint'] + description: | + Processes decoded event data for `SetUint` events on the GMX platform in the Arbitrum and Avalanche blockchains. + This model normalizes the value field and extracts additional details such as the base key and associated data, + ensuring compatibility with downstream analyses. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: event_name + description: The type of event recorded, always 'SetUint' for this model + data_tests: + - not_null + - *msg_sender + - name: base_key + description: The base key associated with the event, decoded from a hexadecimal value. + data_tests: + - not_null + - name: data + description: The associated data field, decoded from a hexadecimal value. + data_tests: + - not_null + - name: value_raw + description: | + The value associated with the event. + data_type: float + data_tests: + - not_null + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: tx_index + description: The index value related to the transaction, used for unique identification. + + + - name: gmx_v2_execution_fee_refund + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'execution_fee_refund'] + description: | + Processes decoded event data for `ExecutionFeeRefund` events on the GMX platform in the Arbitrum and Avalanche blockchains. + This model extracts key details such as the refund amount, receiver, and transaction details to + enable analysis of refunded execution fees. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: event_name + description: The type of event recorded, always `ExecutionFeeRefund` for this model. + data_tests: + - not_null + - *msg_sender + - name: receiver + description: The address that received the refunded execution fee. + - name: refund_fee_amount + description: The amount of the refunded execution fee, denominated in the native token of the blockchain (e.g., ETH). + + + - name: gmx_v2_keeper_execution_fee + meta: + blockchain: arbitrum, avalanche_c + sector: dex + project: gmx + contributors: ai_data_master, gmx-io + config: + tags: ['arbitrum', 'avalanche_c', 'gmx', 'event', 'keeper_execution_fee'] + description: | + Processes decoded event data for `KeeperExecutionFee` events on the GMX platform in the Arbitrum and Avalanche blockchains. + This model extracts key details such as the keeper's address, execution fee amount, and transaction details, enabling + analysis of fees associated with keeper-based executions. + + columns: + - *blockchain + - *block_time + - *block_date + - *block_number + - *tx_hash + - *index + - *contract_address + - name: tx_from + description: The address that initiated the transaction. + - name: tx_to + description: The destination address of the transaction. + - name: event_name + description: The type of event recorded, always `KeeperExecutionFee` for this model. + data_tests: + - not_null + - *msg_sender + - name: keeper + description: The address of the keeper responsible for executing the transaction. + - name: execution_fee_amount + description: The amount of the execution fee, denominated in the native token of the blockchain (e.g., ETH, AVAX). + + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_execution_fee_refund.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_execution_fee_refund.sql new file mode 100644 index 00000000000..a4f2675416f --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_execution_fee_refund.sql @@ -0,0 +1,36 @@ +{{ config( + schema='gmx_v2', + alias = 'execution_fee_refund', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_from, + tx_to, + event_name, + msg_sender, + receiver, + refund_fee_amount +FROM {{ ref('gmx_v2_' ~ chain ~ '_execution_fee_refund') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_keeper_execution_fee.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_keeper_execution_fee.sql new file mode 100644 index 00000000000..a24684122d9 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_keeper_execution_fee.sql @@ -0,0 +1,36 @@ +{{ config( + schema='gmx_v2', + alias = 'keeper_execution_fee', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_from, + tx_to, + event_name, + msg_sender, + keeper, + execution_fee_amount +FROM {{ ref('gmx_v2_' ~ chain ~ '_keeper_execution_fee') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_in_tokens_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_in_tokens_updated.sql new file mode 100644 index 00000000000..1bdd950e9a0 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_in_tokens_updated.sql @@ -0,0 +1,40 @@ +{{ config( + schema='gmx_v2', + alias = 'open_interest_in_tokens_updated', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_index, + tx_from, + tx_to, + event_name, + msg_sender, + market, + collateral_token, + is_long, + next_value, + delta +FROM {{ ref('gmx_v2_' ~ chain ~ '_open_interest_in_tokens_updated') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_updated.sql new file mode 100644 index 00000000000..60d0182af5c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_open_interest_updated.sql @@ -0,0 +1,41 @@ +{{ config( + schema='gmx_v2', + alias = 'open_interest_updated', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_index, + tx_from, + tx_to, + event_name, + msg_sender, + market, + collateral_token, + is_long, + next_value, + delta +FROM {{ ref('gmx_v2_' ~ chain ~ '_open_interest_updated') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_oracle_price_update.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_oracle_price_update.sql new file mode 100644 index 00000000000..5296ab83672 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_oracle_price_update.sql @@ -0,0 +1,40 @@ +{{ config( + schema='gmx_v2', + alias = 'oracle_price_update', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_index, + tx_from, + tx_to, + event_name, + msg_sender, + token, + "provider", + min_price, + max_price, + "timestamp" +FROM {{ ref('gmx_v2_' ~ chain ~ '_oracle_price_update') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_pool_amount_updated.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_pool_amount_updated.sql new file mode 100644 index 00000000000..f55114c2cc6 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_pool_amount_updated.sql @@ -0,0 +1,39 @@ +{{ config( + schema='gmx_v2', + alias = 'pool_amount_updated', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_index, + tx_from, + tx_to, + event_name, + msg_sender, + market, + token, + next_value, + delta +FROM {{ ref('gmx_v2_' ~ chain ~ '_pool_amount_updated') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_position_decrease.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_position_decrease.sql index 1970473375e..30f8ebfff2b 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_position_decrease.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_position_decrease.sql @@ -57,4 +57,4 @@ FROM {{ ref('gmx_v2_' ~ chain ~ '_position_decrease') }} {% if not loop.last %} UNION ALL {% endif %} -{%- endfor -%} \ No newline at end of file +{%- endfor -%} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_set_uint.sql b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_set_uint.sql new file mode 100644 index 00000000000..4dbe5e181f8 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/gmx/event/gmx_v2_set_uint.sql @@ -0,0 +1,38 @@ +{{ config( + schema='gmx_v2', + alias = 'set_uint', + post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c"]\', + "project", + "gmx", + \'["ai_data_master","gmx-io"]\') }}' + ) +}} + +{%- set chains = [ + 'arbitrum', + 'avalanche_c', +] -%} + +{%- for chain in chains -%} +SELECT + blockchain, + block_time, + block_date, + block_number, + tx_hash, + index, + contract_address, + tx_index, + tx_from, + tx_to, + event_name, + msg_sender, + base_key, + "data", + "value_raw" +FROM {{ ref('gmx_v2_' ~ chain ~ '_set_uint') }} +{% if not loop.last %} +UNION ALL +{% endif %} +{%- endfor -%} + From 231a014460a321d7bf1b1ab8f790f13b6c901bd5 Mon Sep 17 00:00:00 2001 From: tomfutago <35136350+tomfutago@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:13:58 +0000 Subject: [PATCH 07/59] feat: add cover ipfs data column (#7264) --- .../models/nexusmutual/ethereum/covers/_schema.yml | 1 + .../ethereum/covers/nexusmutual_ethereum_covers_v2.sql | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/_schema.yml b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/_schema.yml index edee74ce8e9..9d9778dd957 100644 --- a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/_schema.yml @@ -246,6 +246,7 @@ models: - &is_migrated name: is_migrated description: "is migrated flag" + - name: cover_ipfs_data - name: trace_address - *tx_hash diff --git a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql index 9e6f803b750..ec1f72e4071 100644 --- a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql +++ b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql @@ -32,6 +32,7 @@ cover_sales as ( from_hex(json_query(c.params, 'lax $.commissionDestination' omit quotes)) as commission_destination, cast(json_query(t.pool_allocation, 'lax $.coverAmountInAsset') as uint256) as cover_amount_in_asset, cast(json_query(t.pool_allocation, 'lax $.skip') as boolean) as pool_allocation_skip, + cast(json_query(c.params, 'lax $.ipfsData' omit quotes) as varchar) as cover_ipfs_data, c.call_trace_address as trace_address, c.call_tx_hash as tx_hash from {{ source('nexusmutual_ethereum', 'Cover_call_buyCover') }} c @@ -99,6 +100,7 @@ cover_premiums as ( end as premium_asset, c.cover_owner, c.commission_destination, + c.cover_ipfs_data, c.trace_address, c.tx_hash from cover_sales c @@ -138,6 +140,7 @@ covers_v2 as ( cp.commission, cp.commission_ratio, cp.commission_destination, + cp.cover_ipfs_data, cp.trace_address, cp.tx_hash from cover_premiums cp @@ -195,6 +198,7 @@ covers as ( commission_ratio, commission_destination, false as is_migrated, + cover_ipfs_data, trace_address, tx_hash from covers_v2 @@ -223,6 +227,7 @@ covers as ( commission_ratio, commission_destination, true as is_migrated, + null as cover_ipfs_data, null as trace_address, tx_hash from covers_v1_migrated @@ -255,6 +260,7 @@ select commission_ratio, commission_destination, is_migrated, + cover_ipfs_data, trace_address, tx_hash from covers From 5e397b7393d2290357343688e6ac918088c9503d Mon Sep 17 00:00:00 2001 From: Peter <157015397+safeintern@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:14:39 +0800 Subject: [PATCH 08/59] Add Safe Balances for Scroll (#7250) * Update safe_scroll_schema.yml * Create safe_scroll_balances * Rename safe_scroll_balances to safe_scroll_balances.sql --------- Co-authored-by: Huang Geyang --- .../safe/scroll/safe_scroll_balances.sql | 36 ++++++++++++++ .../safe/scroll/safe_scroll_schema.yml | 48 ++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql b/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql new file mode 100644 index 00000000000..5c4389542bb --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql @@ -0,0 +1,36 @@ +{{ + config( + schema = 'safe_scroll', + alias = 'balances', + partition_by = ['day'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['day', 'address', 'token_address'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.day')] + ) + }} + +with safes as ( + select + address, + blockchain + from {{ ref('safe_scroll_safes') }} + where blockchain = 'scroll' +), + +balances as ( + {{ + balances_incremental_subset_daily( + blockchain = 'scroll', + address_list = 'safes', + start_date = '2021-07-01' + ) + }} +) + +select * from balances +where token_standard in ('native', 'erc20') +and token_address not in ( + 0xd74f5255d557944cf7dd0e45ff521520002d5748, --$9.8B were minted in a hack in 2023, all of which are stored in a Safe. Filtering out. + 0xe9689028ede16c2fdfe3d11855d28f8e3fc452a3 ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml index e5c5aec30ca..0d1ad4706e2 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml @@ -158,4 +158,50 @@ models: description: "Output data" - &method name: method - description: "Function method" \ No newline at end of file + description: "Function method" + + - name: safe_scroll_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: scroll + project: safe + contributors: safeintern + config: + tags: ['safe', 'scroll'] + description: “Safe addresses balances” + columns: + - name: day + - name: blockchain + - name: address + - &token_symbol + name: token_symbol + description: "Symbol for the token" + - &token_address + name: token_address + description: "Address for the token" + - &token_standard + name: token_standard + description: "Standard for the token" + - &token_id + name: token_id + description: "ID for the token" + - &balance + name: balance + description: "Balance for the user" + - &balance_usd + name: balance_usd + description: "USD value of balance for the user" + - &last_updated + name: last_updated + description: "UTC timestamp when data was last updated" + - &next_update + name: next_update + description: "UTC timestamp when data is next updated" + - &unique_key_id + name: unique_key_id + description: "unique key" From caf92993c9cbbcc325cfda519cde950b12fe352c Mon Sep 17 00:00:00 2001 From: Peter <157015397+safeintern@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:14:49 +0800 Subject: [PATCH 09/59] Create safe_linea_balances.sql (#7249) * Create safe_linea_balances.sql * Update safe_linea_schema.yml * Update safe_linea_schema.yml v2 * Update safe_linea_schema.yml v3 --------- Co-authored-by: Huang Geyang --- .../safe/linea/safe_linea_balances.sql | 36 ++++++++++++++ .../_project/safe/linea/safe_linea_schema.yml | 47 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql new file mode 100644 index 00000000000..038ed2990d5 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql @@ -0,0 +1,36 @@ +{{ + config( + schema = 'safe_linea', + alias = 'balances', + partition_by = ['day'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['day', 'address', 'token_address'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.day')] + ) + }} + +with safes as ( + select + address, + blockchain + from {{ ref('safe_linea_safes') }} + where blockchain = 'linea' +), + +balances as ( + {{ + balances_incremental_subset_daily( + blockchain = 'linea', + address_list = 'safes', + start_date = '2021-07-01' + ) + }} +) + +select * from balances +where token_standard in ('native', 'erc20') +and token_address not in ( + 0xd74f5255d557944cf7dd0e45ff521520002d5748, --$9.8B were minted in a hack in 2023, all of which are stored in a Safe. Filtering out. + 0xe9689028ede16c2fdfe3d11855d28f8e3fc452a3 ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml index a08fbe09642..2903c2ec94c 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml @@ -147,3 +147,50 @@ models: - &output name: output description: "Output data" + + + - name: safe_linea_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: linea + project: safe + contributors: safeintern + config: + tags: ['safe', 'linea'] + description: “Safe addresses balances” + columns: + - name: day + - name: blockchain + - name: address + - &token_symbol + name: token_symbol + description: "Symbol for the token" + - &token_address + name: token_address + description: "Address for the token" + - &token_standard + name: token_standard + description: "Standard for the token" + - &token_id + name: token_id + description: "ID for the token" + - &balance + name: balance + description: "Balance for the user" + - &balance_usd + name: balance_usd + description: "USD value of balance for the user" + - &last_updated + name: last_updated + description: "UTC timestamp when data was last updated" + - &next_update + name: next_update + description: "UTC timestamp when data is next updated" + - &unique_key_id + name: unique_key_id + description: " unique key" From 209df703480da3c30e288bebcc007ac1db01278a Mon Sep 17 00:00:00 2001 From: Peter <157015397+safeintern@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:15:10 +0800 Subject: [PATCH 10/59] Create safe_avalanche_c_balances.sql (#7248) * Create safe_avalanche_c_balances.sql * Update safe_avalanche_c_schema.yml --- .../avalanche_c/safe_avalanche_c_balances.sql | 36 ++++++++++++++ .../avalanche_c/safe_avalanche_c_schema.yml | 48 ++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql b/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql new file mode 100644 index 00000000000..432cd070a0a --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql @@ -0,0 +1,36 @@ +{{ + config( + schema = 'safe_avalanche_c', + alias = 'balances', + partition_by = ['day'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['day', 'address', 'token_address'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.day')] + ) + }} + +with safes as ( + select + address, + blockchain + from {{ ref('safe_avalanche_c_safes') }} + where blockchain = 'avalanche_c' +), + +balances as ( + {{ + balances_incremental_subset_daily( + blockchain = 'avalanche_c', + address_list = 'safes', + start_date = '2021-07-01' + ) + }} +) + +select * from balances +where token_standard in ('native', 'erc20') +and token_address not in ( + 0xd74f5255d557944cf7dd0e45ff521520002d5748, --$9.8B were minted in a hack in 2023, all of which are stored in a Safe. Filtering out. + 0xe9689028ede16c2fdfe3d11855d28f8e3fc452a3 ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml index c446cffc6c0..629b3491f2d 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml @@ -148,4 +148,50 @@ models: - *trace_address - &amount_usd name: amount_usd - description: "USD amount of transferred AVAX" \ No newline at end of file + description: "USD amount of transferred AVAX" + + - name: safe_avalanche_c_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: avalanche_c + project: safe + contributors: safeintern + config: + tags: ['safe', 'avalanche_c'] + description: “Safe addresses balances” + columns: + - name: day + - name: blockchain + - name: address + - &token_symbol + name: token_symbol + description: "Symbol for the token" + - &token_address + name: token_address + description: "Address for the token" + - &token_standard + name: token_standard + description: "Standard for the token" + - &token_id + name: token_id + description: "ID for the token" + - &balance + name: balance + description: "Balance for the user" + - &balance_usd + name: balance_usd + description: "USD value of balance for the user" + - &last_updated + name: last_updated + description: "UTC timestamp when data was last updated" + - &next_update + name: next_update + description: "UTC timestamp when data is next updated" + - &unique_key_id + name: unique_key_id + description: "unique key" From d84a67a5bf5e646cd6c27fa393f16048f868d011 Mon Sep 17 00:00:00 2001 From: Peter <157015397+safeintern@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:15:20 +0800 Subject: [PATCH 11/59] Create safe_base_balances.sql (#7239) * Create safe_base_balances.sql * Update safe_base_schema.yml * Update safe_base_schema.yml * Update safe_base_schema.yml v2 * Update safe_base_schema.yml v3 * Update safe_base_schema.yml v4 --------- Co-authored-by: Huang Geyang --- .../_project/safe/base/safe_base_balances.sql | 36 +++++++++++++++ .../_project/safe/base/safe_base_schema.yml | 46 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql b/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql new file mode 100644 index 00000000000..84aeba4f700 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql @@ -0,0 +1,36 @@ +{{ + config( + schema = 'safe_base', + alias = 'balances', + partition_by = ['day'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['day', 'address', 'token_address'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.day')] + ) + }} + +with safes as ( + select + address, + blockchain + from {{ ref('safe_base_safes') }} + where blockchain = 'base' +), + +balances as ( + {{ + balances_incremental_subset_daily( + blockchain = 'base', + address_list = 'safes', + start_date = '2021-07-01' + ) + }} +) + +select * from balances +where token_standard in ('native', 'erc20') +and token_address not in ( + 0xd74f5255d557944cf7dd0e45ff521520002d5748, --$9.8B were minted in a hack in 2023, all of which are stored in a Safe. Filtering out. + 0xe9689028ede16c2fdfe3d11855d28f8e3fc452a3 ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml index 4e73519cadc..fae0ee71228 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml @@ -141,3 +141,49 @@ models: - &output name: output description: "Output data" + + - name: safe_base_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: base + project: safe + contributors: safeintern + config: + tags: ['safe', 'base'] + description: “Safe addresses balances” + columns: + - name: day + - name: blockchain + - name: address + - &token_symbol + name: token_symbol + description: "Symbol for the token" + - &token_address + name: token_address + description: "Address for the token" + - &token_standard + name: token_standard + description: "Standard for the token" + - &token_id + name: token_id + description: "ID for the token" + - &balance + name: balance + description: "Balance for the user" + - &balance_usd + name: balance_usd + description: "USD value of balance for the user" + - &last_updated + name: last_updated + description: "UTC timestamp when data was last updated" + - &next_update + name: next_update + description: "UTC timestamp when data is next updated" + - &unique_key_id + name: unique_key_id + description: " unique key" From 68382e2fc9ae98aa5073c7804f70d565e292672f Mon Sep 17 00:00:00 2001 From: Peter <157015397+safeintern@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:15:30 +0800 Subject: [PATCH 12/59] Update safe_linea_safes.sql (#7268) Changed project_start_date, as safes were deployed earlier than the original stated date --- .../models/_project/safe/linea/safe_linea_safes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_safes.sql b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_safes.sql index ff805dcb211..e60f2d7ac53 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_safes.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_safes.sql @@ -16,7 +16,7 @@ ) }} -{% set project_start_date = '2024-02-24' %} +{% set project_start_date = '2023-07-01' %} select 'linea' as blockchain, From 6cf630996aa287fa83d2537bb2295fc677b57069 Mon Sep 17 00:00:00 2001 From: clizzard <145153355+clizzard7@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:15:44 +0100 Subject: [PATCH 13/59] add tradewiz bot (#7262) * add tradewiz bot * fix seed --- .../_sector/dex/bot_trades/solana/_schema.yml | 22 +++ .../platforms/tradewiz_solana_bot_trades.sql | 152 ++++++++++++++++++ .../tradewiz/tradewiz_solana_trades_seed.csv | 21 +++ 3 files changed, 195 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql create mode 100644 dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml index c6d9f401458..a48042380b2 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml @@ -666,3 +666,25 @@ models: - check_bot_trades_seed: seed_file: ref('bitfoot_solana_trades_seed') blockchain: solana + + - name: tradewiz_solana_bot_trades + meta: + blockchain: solana + sector: dex + project: tradewiz + contributors: whale_hunter + config: + tags: ["solana", "dex", "tradewiz", "trades"] + description: > + Tradewiz trades on Solana + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_id + - tx_index + - outer_instruction_index + - inner_instruction_index + - check_bot_trades_seed: + seed_file: ref('tradewiz_solana_trades_seed') + blockchain: solana diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql new file mode 100644 index 00000000000..526385d028c --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql @@ -0,0 +1,152 @@ +{{ config( + alias = 'bot_trades', + schema = 'tradewiz', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] + ) +}} + +{% set project_start_date = '2024-11-27' %} +{% set fee_receiver_1 = '97VmzkjX9w8gMFS2RnHTSjtMEDbifGXBq9pgosFdFnM' %} +{% set wsol_token = 'So11111111111111111111111111111111111111112' %} + +WITH + allFeePayments AS ( + SELECT + tx_id, + 'SOL' AS feeTokenType, + balance_change / 1e9 AS fee_token_amount, + '{{wsol_token}}' AS fee_token_mint_address + FROM + {{ source('solana','account_activity') }} + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND tx_success + AND balance_change > 0 + AND ( + address = '{{fee_receiver_1}}' + ) + ), + botTrades AS ( + SELECT + trades.block_time, + CAST(date_trunc('day', trades.block_time) AS date) AS block_date, + CAST(date_trunc('month', trades.block_time) AS date) AS block_month, + 'solana' AS blockchain, + amount_usd, + IF( + token_sold_mint_address = '{{wsol_token}}', + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_mint_address AS token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_mint_address AS token_sold_address, + fee_token_amount * price AS fee_usd, + fee_token_amount, + IF(feeTokenType = 'SOL', 'SOL', symbol) AS fee_token_symbol, + fee_token_mint_address AS fee_token_address, + project, + version, + token_pair, + project_program_id AS project_contract_address, + trader_id AS user, + trades.tx_id, + tx_index, + outer_instruction_index, + inner_instruction_index + FROM + {{ ref('dex_solana_trades') }} AS trades + JOIN allFeePayments AS feePayments ON trades.tx_id = feePayments.tx_id + LEFT JOIN {{ source('prices', 'usd') }} AS feeTokenPrices ON ( + feeTokenPrices.blockchain = 'solana' + AND fee_token_mint_address = toBase58 (feeTokenPrices.contract_address) + AND date_trunc('minute', block_time) = minute + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + JOIN {{ source('solana','transactions') }} AS transactions ON ( + trades.tx_id = id + {% if is_incremental() %} + AND {{ incremental_predicate('transactions.block_time') }} + {% else %} + AND transactions.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + WHERE + trades.trader_id != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestInnerInstructionIndexForEachTrade AS ( + SELECT + tx_id, + outer_instruction_index, + MAX(inner_instruction_index) AS highestInnerInstructionIndex + FROM + botTrades + GROUP BY + tx_id, + outer_instruction_index + ) +SELECT + block_time, + block_date, + block_month, + 'Solgun' as bot, + blockchain, + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + project, + version, + token_pair, + project_contract_address, + user, + botTrades.tx_id, + tx_index, + botTrades.outer_instruction_index, + COALESCE(inner_instruction_index, 0) AS inner_instruction_index, + IF( + inner_instruction_index = highestInnerInstructionIndex, + true, + false + ) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestInnerInstructionIndexForEachTrade ON ( + botTrades.tx_id = highestInnerInstructionIndexForEachTrade.tx_id + AND botTrades.outer_instruction_index = highestInnerInstructionIndexForEachTrade.outer_instruction_index + ) +ORDER BY + block_time DESC, + tx_index DESC, + outer_instruction_index DESC, + inner_instruction_index DESC diff --git a/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv new file mode 100644 index 00000000000..2525f27cce6 --- /dev/null +++ b/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction +2024-12-03 03:41:57.000 UTC,2024-12-03,2024-12-01,Solgun,solana,5.10029703516,Buy,130284.164414,VIBE,AB4oELjeXLyeV3t3evMgq3zXrxXAJ1WdRy9qgfinH1oq,0.022402148,SOL,So11111111111111111111111111111111111111112,0.04940438999999999,0.000217,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,VIBE-SOL,AKtxTJHHtyDTBUerXQhWQSY4L8oBreBqPHkiPbkyoL3s,8dRLn2NMHsN7Fu1nxCbfQDujQ4wyPaqmnfqHyfVvBnmj,5xYqUmnvW78WSa1VL9G3H2fkRcBbaGRfr95iDTH9JWNTAheT7pnNguneMDzkge9Bqs5KTK8boRHKHPupBtW4hBL1,1139,7,4,true +2024-12-07 15:17:10.000 UTC,2024-12-07,2024-12-01,Solgun,solana,2.1149015703,Sell,0.008739985,SOL,So11111111111111111111111111111111111111112,312509.10149,CELLAI3,5q4maTRYm1ZPuFZqFzwYRJ7jAA5xkprmndJdGHUnpjwT,0.0209373195,0.000086525,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-CELLAI3,ARUgR3UzG2KCXWAXQNSHdPuuZ5Spmmm8vTENUn3UK4HU,F6coj9TDxLDw1wSFYsQMPmA2iPYhNxu17qBSEZ8vZuDm,5769zBmZrWqPGqWU9QA2LH5CrjPnJTWePmtC8bgTtciupjiLNuNaYuLS3FEkN7eThaQj57NcTc1Hz39VnkdtDgbB,2030,3,2,true +2024-12-06 19:47:39.000 UTC,2024-12-06,2024-12-01,Solgun,solana,5.72664660432,Buy,103332.858778,PAWBLEMS,4fLAZZCA3Mrbj6Wa9PkTUHfUWu3JE2RqGHvDrXiMpump,0.023872964,SOL,So11111111111111111111111111111111111111112,0.047976000000000005,0.0002,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-PAWBLEMS,kLCF7gruvuMfBLvwSL4XZQK2ENKYKm97LFuLAKkxC5b,DksfCf1U4arH6eq9KJJJtHDiYypi3gzwnxbup2266uae,szuK2Sdz9ps9Go88t1FVjFRezggBdh1RqWETVD8voF78mJm97H3pnt6EYrN9aUmYHUmGAvKfrTs46tJfC2F1GCt,1583,5,4,true +2024-12-06 11:48:01.000 UTC,2024-12-06,2024-12-01,Solgun,solana,65.36809866600001,Sell,0.279350849,SOL,So11111111111111111111111111111111111111112,371324.880761,HYPHAL,2gsxJ1fyLi3kzvjQx8Wd8y323Ptigzj9ti54grHkpump,0.658593702,0.002814503,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-HYPHAL,2YeDSYdr9mMvEtiWskGvHbt6WSZ6wHW9xGi8DuB9GjdJ,Ed1rA4YjgtJMExjQ6sjWrEEj7xaeL6f4yvT39sm8fZo8,3VN42UsvXsxqnUDiTDQaHhrWmGCwoWnfJJWyPbz7nvja5n9EZohisb7ZR4ojD3LzDG8rquC74kJY3Z3bHac6FYqh,64,4,0,true +2024-12-03 06:01:15.000 UTC,2024-12-03,2024-12-01,Solgun,solana,12.54215028618,Sell,0.055040814,SOL,So11111111111111111111111111111111111111112,434296.831722,Pooch,csrNabqJ6eKoueCiVxGWTTbN3Bri1MhwbwoCN2Wpump,0.12432632773999999,0.000545602,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Pooch,H82eA83r3Djyt1s4T6213puNzM1As8w2tCSYjgv6kjFz,66gfZJ3yVCxhRY3VvSKxTYhrW4HAUvhbVFRCRrgoATkE,3hhXSyxnoA3U1eYsY2P8EjAqen3mWBscPoPK4pudmobfENN1VKCzAHKPaBQRk4S98W9fB41tm4MgUJ1NE4F4ZjNt,413,3,2,true +2024-12-02 18:50:09.000 UTC,2024-12-02,2024-12-01,Solgun,solana,33.5145,Buy,1090000.477547,FWILL,8aMmdyEH97qmtxge5odgJE45KwwXQkaSZEjM13EDpump,0.15,SOL,So11111111111111111111111111111111111111112,0.335145,0.0015,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FWILL,5QiczEwrbzHNooa54TVkt4oe2HFHLd9u2TCJNX46TJjj,A3kLfkrESnvmZUs54BKYS48bPq4DJT11psfFC5ADMSwS,op8TBkDy4D5gr7xX4VCiC6VvEXCuYV4zo3fhHCEJMFu4Athih23wEzHw24tA1WmjQd4eZ4swA8o7Dii6fk1vC6j,181,6,0,true +2024-12-02 05:59:04.000 UTC,2024-12-02,2024-12-01,Solgun,solana,0.012848601869999999,Sell,0.000056277,SOL,So11111111111111111111111111111111111111112,58135.635755705,LIR,Agdtmev8iBbBJZjXDf6u18M51Bu173gsJTzcWB1patsJ,0.00012876684,5.64e-7,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LIR,84PapxynnSyNL7LRg4vKg41SVrmU2NqRxaudeQe1NYUb,9gxBDddAvsVUmcLAwJWhnKr5dMN8rWkdqn8ZZ4V272bo,3UBAiPDUWx2jpU4H7u3VV4LkkYXMjxUtbnK2uj71mkHb2T22iWFxvtAnjac4rwQA5itbZb9VahYM4XRbvpEEyxdr,552,4,0,true +2024-12-02 20:48:46.000 UTC,2024-12-02,2024-12-01,Solgun,solana,174.81549384187,Buy,14986480.13779,FAITH,CK3TMoqHy5ApzRwooxbxMAbpGDK2w6KpQMdv2rDhpump,0.782137237,SOL,So11111111111111111111111111111111111111112,1.6763249999999998,0.0075,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-FAITH,5PBojHVrS9C6F6kjTosfVEUoBzED8PCxpFdeouWzDQbd,2kNoYHNtpaefWZ3JptWug7H7pyzXZXYYTQufNbnt3Zzy,3gPFS1BtQArtTN2hZwbrUxKETHiHRts4YCcCEZqFxoGo9E8zpYztsjdtNsUmbK14qNT7VdieNJXbY9tBTembavCB,413,7,4,true +2024-12-02 18:06:48.000 UTC,2024-12-02,2024-12-01,Solgun,solana,30.016174407599998,Buy,1809787.710499,ART,CuSXYspmP7khJiov8Kr3XaExcuhPadVDHGm3K3rvpump,0.13517146,SOL,So11111111111111111111111111111111111111112,0.29361128496,0.001322216,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-ART,GD5CvxQZwKLfqMpvMTeLkTecnMVZGNPXnUk597W1j8hb,H9kHjTaJqVrrSuZGwJWgQnjyhVmGANpqsA6eoqbbZrTk,2SZ1av6twfq14Um3gM8KuRopdqdJpDtESqidS725tKMiDMa1zQu2wnr76JMsbXZMrH5LiUyGv9XPUqMkmwbjW4S9,2628,5,4,true +2024-12-01 23:25:30.000 UTC,2024-12-01,2024-12-01,Solgun,solana,12.139543894200001,Buy,1715893.906365,Bio AI,7LL7cbEfKLMiH4oUmfnfYgAhUnnVRWGeksE6iXjDpump,0.050942274,SOL,So11111111111111111111111111111111111111112,0.2383,0.001,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Bio AI,76BfryV9K9e9o8zF69csrqFLaar2PpdYZ8hN7ZRE4Y3e,H5PQNrAj7uKYK44ksRqym8sAfuasNC4xHJZW88WbzDyT,3YXaTrff9qmYHko2ZD5veVtAcF7XfJa8eCmFBmMtV5dLdowbCn2D2ArfkymQ2ZLon5ExhEYr5qbM54JJbE4NiZtU,475,7,4,true +2024-12-03 20:40:46.000 UTC,2024-12-03,2024-12-01,Solgun,solana,148.38664547177999,Buy,17917109.708311,WILDMNKY,75fv3J8gw9mp64ZVcAXjmr1qT8PqJjE1Cc4ocjmCpump,0.645552273,SOL,So11111111111111111111111111111111111111112,1.3791600000000002,0.006,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,WILDMNKY-SOL,7SV8yk5hXtcXNqgohc9tJMdmAbBnLpbKr8oURHdh8s8g,4Nw2v6XZJgaWHmtETqMWtQyTHaGDtQZ27x3otcvjQAHT,ep8K6Nvu25kw1yVWqENPHv2Y4mGRW5T7kMBxeXnmujd6WoDxzSfVtqASa2fz5toNTyX1JDWecAoMHxTdEsXnkxU,601,7,4,true +2024-12-03 00:15:12.000 UTC,2024-12-03,2024-12-01,Solgun,solana,5.12252529562,Sell,0.022631993,SOL,So11111111111111111111111111111111111111112,27129.461569,anthrupad,2okJ4wqqDFFD2Tce3QaU8L866KxBJzJXVoMtLR1ppump,0.052636498700000006,0.000232555,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-anthrupad,FAfDgkWFexH1BA2D1i5pc2uRR8ZYcvt8jY84CK7e9My8,31M2cNExVg8bwB2HwzAzA1w9pHwLeP67ejygZ44UB1UR,2HryhZK2dRLqZeTowRQ4Rd7BGwL9ekAgyjDuaFvYAJyyVZHn5C81cTGGc66eMRg9MhiiWEdU7j1yWKfJDjdfbMRU,499,4,0,true +2024-12-07 18:47:05.000 UTC,2024-12-07,2024-12-01,Solgun,solana,47.17441114759,Sell,0.195040357,SOL,So11111111111111111111111111111111111111112,12392.36946,AFITR,83Ur4XYc5QyRNBshoW2Yu9cCLyNDV9s5krBu11JWDxxq,0.47612907671000004,0.001968533,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-AFITR,5JtbDjPqfN6K4mDF2c5rvJZXXryQbihjViSP2XoMohSV,8uAied3jRqG1mPRzqsmLaeRDz3F22BkarmfgAvoqNWN,3Lr37rTZUVk9m1iYz3h8sg6Dt4dMHm4PhLmuPUe7G1Z3tMxraN54M8boZMP5wdXGqs8NGQPHMDaXLZoKAq1zMZLE,238,4,0,true +2024-12-01 18:04:29.000 UTC,2024-12-01,2024-12-01,Solgun,solana,32.99628642063,Buy,438388.5076,Metanoia,2dPFQnktvPKZe3hjkstWfJx8tDrH8FW9xoiBjZMEpump,0.139218963,SOL,So11111111111111111111111111111111111111112,0.2986326,0.00126,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Metanoia,FpTruBXx6UcyBiVr2Rj7UMf9tNTpJgTyGpXynpuo9HzA,GhqrMrC4tY8zTDvWHiHG7ag4MWPqQ7qLZmeochY6aosY,29TPJAT4vP7rUmMETneBknCmH8UpzdeFaB6ZpJTmzVobE6kTou8KBgqvPPLb588ettce6HRVFUPcSbuSrRJpJnzG,620,5,4,true +2024-12-01 02:27:09.000 UTC,2024-12-01,2024-12-01,Solgun,solana,19.69581542709,Sell,0.083524089,SOL,So11111111111111111111111111111111111111112,313146.63071,GSC,F86FtKmg8vnaCfDoJF7U21NbmLprVazE3L2PshmCpump,0.26366694273,0.001118133,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GSC,7H6MS1nqg9Y6NN8srD9UCovDQwVxuZaozgXrMT7kcH61,35cx6mhp2Nsi33d7QfwnqVrYoEcyWpc17PtgMrCnwfxt,5K5isZKhK8mwM4rZ7akSejvwByViNxZ5GcCMcsqWENmT42WkJ4kUoYsAJQkeq73pwiEBgjwEoUEgRNB8jCt9Bugp,541,4,0,true +2024-12-04 17:48:52.000 UTC,2024-12-04,2024-12-01,Solgun,solana,29.19816884426,Sell,0.128270302,SOL,So11111111111111111111111111111111111111112,1239313.36229,Pretty,8pTpjjcq5pG7NXDm99NinoT1vRoH99X7s3C1Go3upump,0.30598138415,0.001344205,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Pretty,HBiCDY9CVNWENM8vTkHekJoNAeaLZAJoHr7EwvmAf5WY,GhqrMrC4tY8zTDvWHiHG7ag4MWPqQ7qLZmeochY6aosY,53KrU2unKY95pnSKSq3empC6woBXPxQShDv3EmpDNnn1knesLfxHtCetCkbuNPRWPXr3tj3WKuu8B63Ypawx8yvb,977,3,2,true +2024-12-04 04:50:48.000 UTC,2024-12-04,2024-12-01,Solgun,solana,77.97821513972,Buy,1428724.561159,eigenrobot,9qpaeuNMNWwHfa7keQ51D5VJNt2jessw8khxWis5pump,0.327832402,SOL,So11111111111111111111111111111111111111112,0.7135800000000001,0.003,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-eigenrobot,7Ys5V2wmjz3LihubgfnCdSZnwEUvBNBXXLvpAWZLRiS5,5q853P8PLeYiE5PjXg24kQ9nXrf5Qn6r55A9urS4hYF6,2XPV2bKhVjYGjuLGWhxjUK69mPFqyfFYcBM6HTSnErYLxHP1cNSksWp4i3VbF7QesYJtQnPvtt5aixEbkr137nz1,959,7,4,true +2024-12-04 13:04:31.000 UTC,2024-12-04,2024-12-01,Solgun,solana,0.00010256337,Sell,4.41e-7,SOL,So11111111111111111111111111111111111111112,117382.719111186,FCI,7s9q7HcfMhhf4XoFtHUzJWyAPAN3mSEmBDfB17iM67QJ,9.3028e-7,4e-9,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FCI,8AfbS9Ji49Jj4UkZmm4whYACyUN8QQQa1dsj9JuYQjFs,EcpfuULFUmouh7XU4uESrksywAwoND2PBu7T3LJwXJmM,4N2Fw8hjiaKSBHnkc7nbH4C6TbjFtpE5RHas2uQKkJvNVUSbwUuTST6GhTueLsrMJe4ho9Sr13xm3V1QQtNzGcSf,460,4,0,true +2024-12-04 20:35:20.000 UTC,2024-12-04,2024-12-01,Solgun,solana,46.792,Buy,66517.719092,convo,KnrmAHo1wW2fGqVXcSZSpo6kmiKam5vNbDByrURpump,0.2,SOL,So11111111111111111111111111111111111111112,0.46792,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-convo,Exy58ud7t8LHN1EBnrcRRwRQmz1uSUC3hQF9oHpM1Box,5CEbKj8WbqKkah7nhYxkXGvN97xvz5hxd8fQjYnW5NTV,rNFm4bvTJvPVtqQGoBhyFBPhFbJftPVxoasZccwEhaaJqBA4GMVurr3aJTexbDm4p673RSnpTHpnmxLshHmGEK6,38,5,0,true +2024-12-06 07:59:32.000 UTC,2024-12-06,2024-12-01,Solgun,solana,47.796,Buy,8548.498569,PIZZAGUY,5cN4dWZKqxrpwCCBcwHpsAgus1s7WWyRaguR8YbRiECY,0.2,SOL,So11111111111111111111111111111111111111112,0.47796,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-PIZZAGUY,FgYufZ6c7NpcMvRnbB6tYupsEoyiPCtBw9dZ3bVo8kHE,1EjTzPniAGBxzvi4TBfocfq4Zxuj2GCK42aWnNJEoSS,619fmC5opZwwwAnnci5snSqtVcp9cqPYW2SAiwq8DZQrPVS8F1cLDZW9fzwxdjBYbBKZtBTeMyA9rjeCkLU1U7uE,1700,6,0,true From 0fc254cef1c3a3fe224aef9fc9931019a96afca8 Mon Sep 17 00:00:00 2001 From: max-morrow Date: Mon, 9 Dec 2024 20:59:57 +0300 Subject: [PATCH 14/59] fix (#7256) --- .../oneinch/_meta/oneinch_mapped_contracts_macro.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql b/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql index c41474053bc..fffe4792f2d 100644 --- a/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql +++ b/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql @@ -324,10 +324,10 @@ , ('0xb971ef87ede563556b2ed4b1c0b0019111dd85d2', 'true', 'Uniswap' , 'UNDEFINED' , ['bnb']) , ('0x2626664c2603336e57b271c5c0b26f421741e481', 'true', 'Uniswap' , 'UNDEFINED' , ['base']) , ('0x198ef79f1f515f02dfe9e3115ed9fc07183f02fc', 'true', 'Uniswap' , 'UNDEFINED' , ['base']) - , ('0xe80bf394d190851e215d5f67b67f8f5a52783f1e', 'true', 'Uniswap' , 'UniswapXV1' , ['ethereum']) - , ('0x6000da47483062a0d734ba3dc7576ce6a0b645c4', 'true', 'Uniswap' , 'UniswapXV1' , ['ethereum']) - , ('0x00000011f84b9aa48e5f8aa8b9897600006289be', 'true', 'Uniswap' , 'UniswapXV2' , ['ethereum']) - , ('0x1bd1aAdc9E230626C44a139d7E70d842749351eb', 'true', 'Uniswap' , 'UniswapXV2' , ['arbitrum']) + , ('0xe80bf394d190851e215d5f67b67f8f5a52783f1e', 'true', 'UniswapX' , 'UniswapXV1' , ['ethereum']) + , ('0x6000da47483062a0d734ba3dc7576ce6a0b645c4', 'true', 'UniswapX' , 'UniswapXV1' , ['ethereum']) + , ('0x00000011f84b9aa48e5f8aa8b9897600006289be', 'true', 'UniswapX' , 'UniswapXV2' , ['ethereum']) + , ('0x1bd1aAdc9E230626C44a139d7E70d842749351eb', 'true', 'UniswapX' , 'UniswapXV2' , ['arbitrum']) , ('0x9c12939390052919af3155f41bf4160fd3666a6f', 'true', 'Velodrome' , 'Router' , ['optimism']) , ('0xa132dab612db5cb9fc9ac426a0cc215a3423f9c9', 'true', 'Velodrome' , 'Router' , ['optimism']) , ('0x777777773fdd8b28bb03377d10fcea75ad9768da', 'true', 'Viarouter' , 'ViaRouter' , ['polygon','arbitrum','optimism']) From db0c129aa121c27e91811eed22bbb33c1441bed5 Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:28:59 +0000 Subject: [PATCH 15/59] add upstream balancer_v3 sources (#7277) * add upstream balancer v3 sources * add pool labels * fix labels * again --- .../_projects/balancer/labels/_schema.yml | 87 ++++++++++++++- .../labels_balancer_v2_pools_arbitrum.sql | 1 + .../labels_balancer_v2_pools_avalanche_c.sql | 1 + .../base/labels_balancer_v2_pools_base.sql | 1 + .../labels_balancer_v2_pools_ethereum.sql | 1 + .../labels_balancer_v3_pools_ethereum.sql | 101 ++++++++++++++++++ .../labels_balancer_v2_pools_gnosis.sql | 1 + .../labels_balancer_v3_pools_gnosis.sql | 101 ++++++++++++++++++ .../labels/labels_balancer_v3_pools.sql | 11 ++ .../labels_balancer_v2_pools_optimism.sql | 1 + .../labels_balancer_v2_pools_polygon.sql | 1 + .../zkevm/labels_balancer_v2_pools_zkevm.sql | 1 + sources/_subprojects_outputs/dex/_sources.yml | 5 +- .../aave/ethereum/aave_ethereum_sources.yml | 9 ++ sources/aave/gnosis/aave_gnosis_sources.yml | 7 ++ .../ethereum/balancer_ethereum_sources.yml | 34 ++++++ .../gnosis/balancer_gnosis_sources.yml | 33 ++++++ .../ethereum/morpho_ethereum_sources.yml | 8 ++ 18 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql create mode 100644 dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql create mode 100644 dbt_subprojects/dex/models/_projects/balancer/labels/labels_balancer_v3_pools.sql create mode 100644 sources/aave/gnosis/aave_gnosis_sources.yml create mode 100644 sources/morpho/ethereum/morpho_ethereum_sources.yml diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml b/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml index 9e2d6f55d00..86bc33222c3 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml @@ -304,4 +304,89 @@ models: - *created_at - *updated_at - *model_name - - *label_type \ No newline at end of file + - *label_type + + - name: labels_balancer_v3_pools_ethereum + meta: + blockchain: ethereum + sector: labels + project: balancer_v3 + contributors: balancerlabs, viniabussafi + config: + tags: ['labels', 'ethereum', 'balancer', 'pools'] + description: 'Balancer V3 liquidity pools created on Ethereum.' + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - address + columns: + - *blockchain + - *address + - *name + - *category + - *contributor + - *source + - *created_at + - *updated_at + - *model_name + - *label_type + + - name: labels_balancer_v3_pools_gnosis + meta: + blockchain: gnosis + sector: labels + project: balancer_v3 + contributors: balancerlabs, viniabussafi + config: + tags: ['labels', 'zkevm', 'balancer', 'pools'] + description: 'Balancer V3 liquidity pools created on Gnosis.' + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - address + columns: + - *blockchain + - *address + - *name + - *category + - *contributor + - *source + - *created_at + - *updated_at + - *model_name + - *label_type + + + - name: labels_balancer_v3_pools + meta: + blockchain: ethereum, gnosis + sector: labels + project: balancer_v3 + contributors: balancerlabs, viniabussafi + config: + tags: + [ + 'labels', + 'ethereum', + 'polygon', + 'arbitrum', + 'optimism', + 'avalanche_c', + 'base', + 'gnosis', + 'zkevm', + 'balancer', + 'pools', + ] + description: 'Balancer V3 liquidity pools created across blockchains.' + columns: + - *blockchain + - *address + - *name + - *category + - *contributor + - *source + - *created_at + - *updated_at + - *model_name + - *label_type \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/arbitrum/labels_balancer_v2_pools_arbitrum.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/arbitrum/labels_balancer_v2_pools_arbitrum.sql index 28698790a26..10f6c91decb 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/arbitrum/labels_balancer_v2_pools_arbitrum.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/arbitrum/labels_balancer_v2_pools_arbitrum.sql @@ -275,6 +275,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'arbitrum' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/avalanche_c/labels_balancer_v2_pools_avalanche_c.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/avalanche_c/labels_balancer_v2_pools_avalanche_c.sql index 15ea79d74de..9ae2b794382 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/avalanche_c/labels_balancer_v2_pools_avalanche_c.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/avalanche_c/labels_balancer_v2_pools_avalanche_c.sql @@ -151,6 +151,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'avalanche_c' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/base/labels_balancer_v2_pools_base.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/base/labels_balancer_v2_pools_base.sql index 1b4c00b5f9e..bbf7e0c347c 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/base/labels_balancer_v2_pools_base.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/base/labels_balancer_v2_pools_base.sql @@ -126,6 +126,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'base' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v2_pools_ethereum.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v2_pools_ethereum.sql index 9e9c26f9489..5fdeff93667 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v2_pools_ethereum.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v2_pools_ethereum.sql @@ -275,6 +275,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'ethereum' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql new file mode 100644 index 00000000000..8e28104d6ff --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql @@ -0,0 +1,101 @@ +{{config( + schema = 'labels', + alias = 'balancer_v3_pools_ethereum', + post_hook = '{{ expose_spells(\'["ethereum"]\', + "sector", + "labels", + \'["viniabussafi"]\') }}' +)}} + +WITH token_data AS ( + SELECT + pool, + ARRAY_AGG(json_extract_scalar(token, '$.token') ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source('balancer_v3_ethereum', 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + + pools AS ( + SELECT + pool_id, + zip.tokens AS token_address, + zip.weights / POWER(10, 18) AS normalized_weight, + symbol, + pool_type + FROM ( + SELECT + c.pool AS pool_id, + t.tokens, + w.weights, + cc.symbol, + 'weighted' AS pool_type + FROM token_data c + INNER JOIN {{ source('balancer_v3_ethereum', 'WeightedPoolFactory_call_create') }} cc + ON c.pool = cc.output_pool + CROSS JOIN UNNEST(c.tokens) WITH ORDINALITY t(tokens, pos) + CROSS JOIN UNNEST(cc.normalizedWeights) WITH ORDINALITY w(weights, pos) + WHERE t.pos = w.pos + ) zip + + + UNION ALL + + SELECT + c.pool AS pool_id, + t.tokens, + 0 AS weights, + cc.symbol, + 'stable' AS pool_type + FROM {{ source('balancer_v3_ethereum', 'Vault_evt_PoolRegistered') }} c + INNER JOIN {{ source('balancer_v3_ethereum', 'StablePoolFactory_call_create') }} cc + ON c.pool = cc.output_pool + CROSS JOIN UNNEST(cc.tokens) AS t(tokens) + ), + + settings AS ( + SELECT + pool_id, + coalesce(t.symbol, '?') AS token_symbol, + normalized_weight, + p.symbol AS pool_symbol, + p.pool_type + FROM pools p + LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = CAST(t.contract_address AS VARCHAR) + AND t.blockchain = 'ethereum' + ) + +SELECT + 'ethereum' AS blockchain, + bytearray_substring(pool_id, 1, 20) AS address, + CASE WHEN pool_type IN ('stable') + THEN lower(pool_symbol) + ELSE lower(concat(array_join(array_agg(token_symbol ORDER BY token_symbol), '/'), ' ', + array_join(array_agg(cast(norm_weight AS varchar) ORDER BY token_symbol), '/'))) + END AS name, + pool_type, + 'balancer_v3_pool' AS category, + 'balancerlabs' AS contributor, + 'query' AS source, + TIMESTAMP'2024-12-01 00:00' AS created_at, + now() AS updated_at, + 'balancer_v3_pools_ethereum' AS model_name, + 'identifier' AS label_type +FROM ( + SELECT + s1.pool_id, + token_symbol, + pool_symbol, + cast(100 * normalized_weight AS integer) AS norm_weight, + pool_type + FROM settings s1 + GROUP BY s1.pool_id, token_symbol, pool_symbol, normalized_weight, pool_type +) s +GROUP BY pool_id, pool_symbol, pool_type +ORDER BY 1 \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v2_pools_gnosis.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v2_pools_gnosis.sql index c907edd95e4..0cba149d634 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v2_pools_gnosis.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v2_pools_gnosis.sql @@ -192,6 +192,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'gnosis' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql new file mode 100644 index 00000000000..5997e2be864 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql @@ -0,0 +1,101 @@ +{{config( + schema = 'labels', + alias = 'balancer_v3_pools_gnosis', + post_hook = '{{ expose_spells(\'["gnosis"]\', + "sector", + "labels", + \'["viniabussafi"]\') }}' +)}} + +WITH token_data AS ( + SELECT + pool, + ARRAY_AGG(json_extract_scalar(token, '$.token') ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source('balancer_v3_gnosis', 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + + pools AS ( + SELECT + pool_id, + zip.tokens AS token_address, + zip.weights / POWER(10, 18) AS normalized_weight, + symbol, + pool_type + FROM ( + SELECT + c.pool AS pool_id, + t.tokens, + w.weights, + cc.symbol, + 'weighted' AS pool_type + FROM token_data c + INNER JOIN {{ source('balancer_v3_gnosis', 'WeightedPoolFactory_call_create') }} cc + ON c.pool = cc.output_pool + CROSS JOIN UNNEST(c.tokens) WITH ORDINALITY t(tokens, pos) + CROSS JOIN UNNEST(cc.normalizedWeights) WITH ORDINALITY w(weights, pos) + WHERE t.pos = w.pos + ) zip + + + UNION ALL + + SELECT + c.pool AS pool_id, + t.tokens, + 0 AS weights, + cc.symbol, + 'stable' AS pool_type + FROM {{ source('balancer_v3_gnosis', 'Vault_evt_PoolRegistered') }} c + INNER JOIN {{ source('balancer_v3_gnosis', 'StablePoolFactory_call_create') }} cc + ON c.pool = cc.output_pool + CROSS JOIN UNNEST(cc.tokens) AS t(tokens) + ), + + settings AS ( + SELECT + pool_id, + coalesce(t.symbol, '?') AS token_symbol, + normalized_weight, + p.symbol AS pool_symbol, + p.pool_type + FROM pools p + LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = CAST(t.contract_address AS VARCHAR) + AND t.blockchain = 'gnosis' + ) + +SELECT + 'gnosis' AS blockchain, + bytearray_substring(pool_id, 1, 20) AS address, + CASE WHEN pool_type IN ('stable') + THEN lower(pool_symbol) + ELSE lower(concat(array_join(array_agg(token_symbol ORDER BY token_symbol), '/'), ' ', + array_join(array_agg(cast(norm_weight AS varchar) ORDER BY token_symbol), '/'))) + END AS name, + pool_type, + 'balancer_v3_pool' AS category, + 'balancerlabs' AS contributor, + 'query' AS source, + TIMESTAMP'2024-12-01 00:00' AS created_at, + now() AS updated_at, + 'balancer_v3_pools_gnosis' AS model_name, + 'identifier' AS label_type +FROM ( + SELECT + s1.pool_id, + token_symbol, + pool_symbol, + cast(100 * normalized_weight AS integer) AS norm_weight, + pool_type + FROM settings s1 + GROUP BY s1.pool_id, token_symbol, pool_symbol, normalized_weight, pool_type +) s +GROUP BY pool_id, pool_symbol, pool_type +ORDER BY 1 \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/labels_balancer_v3_pools.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/labels_balancer_v3_pools.sql new file mode 100644 index 00000000000..1e9e933869d --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/labels_balancer_v3_pools.sql @@ -0,0 +1,11 @@ +{{config( + schema = 'labels', + alias = 'balancer_v3_pools', + post_hook='{{ expose_spells(\'["ethereum", "gnosis"]\', + "sector", + "labels", + \'["balancerlabs", "viniabussafi"]\') }}')}} + +SELECT * FROM {{ ref('labels_balancer_v3_pools_ethereum') }} +UNION +SELECT * FROM {{ ref('labels_balancer_v3_pools_gnosis') }} diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/optimism/labels_balancer_v2_pools_optimism.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/optimism/labels_balancer_v2_pools_optimism.sql index 22a66efaaa4..139d8a5882a 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/optimism/labels_balancer_v2_pools_optimism.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/optimism/labels_balancer_v2_pools_optimism.sql @@ -196,6 +196,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'optimism' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/polygon/labels_balancer_v2_pools_polygon.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/polygon/labels_balancer_v2_pools_polygon.sql index fdcc3adcc08..4ff7daa683e 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/polygon/labels_balancer_v2_pools_polygon.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/polygon/labels_balancer_v2_pools_polygon.sql @@ -273,6 +273,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'polygon' ) SELECT diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/zkevm/labels_balancer_v2_pools_zkevm.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/zkevm/labels_balancer_v2_pools_zkevm.sql index 8353fd05ce4..f72c672acd5 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/zkevm/labels_balancer_v2_pools_zkevm.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/zkevm/labels_balancer_v2_pools_zkevm.sql @@ -154,6 +154,7 @@ settings AS ( p.pool_type FROM pools p LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address + AND t.blockchain = 'zkevm' ) SELECT diff --git a/sources/_subprojects_outputs/dex/_sources.yml b/sources/_subprojects_outputs/dex/_sources.yml index 88751103645..e590eaae00d 100644 --- a/sources/_subprojects_outputs/dex/_sources.yml +++ b/sources/_subprojects_outputs/dex/_sources.yml @@ -56,7 +56,10 @@ sources: - name: balancer_v2_pools_optimism - name: balancer_v2_pools_polygon - name: balancer_v2_pools_zkevm - - name: balancer_cowswap_amm_pools + - name: balancer_cowswap_amm_pools + - name: balancer_v3_pools_ethereum + - name: balancer_v3_pools_gnosis + - name: balancer_v3_pools - name: uniswap_v3_optimism tables: - name: ovm1_pool_mapping diff --git a/sources/aave/ethereum/aave_ethereum_sources.yml b/sources/aave/ethereum/aave_ethereum_sources.yml index 25e06170c4f..28958196631 100644 --- a/sources/aave/ethereum/aave_ethereum_sources.yml +++ b/sources/aave/ethereum/aave_ethereum_sources.yml @@ -117,3 +117,12 @@ sources: - *evt_index - *evt_tx_hash - *id + + - name: erc4626TokenLM_evt_Initialized + + - name: erc4626TokenFactory_evt_StaticTokenCreated + + - name: aave_v3_ethereum + description: "Decoded contracts for Aave v3 on Ethereum" + tables: + - name: VariableDebtToken_evt_Initialized \ No newline at end of file diff --git a/sources/aave/gnosis/aave_gnosis_sources.yml b/sources/aave/gnosis/aave_gnosis_sources.yml new file mode 100644 index 00000000000..9d44db97d25 --- /dev/null +++ b/sources/aave/gnosis/aave_gnosis_sources.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: aave_v3_gnosis + description: "Decoded contracts for Aave v3 on Gnosis" + tables: + - name: AToken_evt_Initialized \ No newline at end of file diff --git a/sources/balancer/ethereum/balancer_ethereum_sources.yml b/sources/balancer/ethereum/balancer_ethereum_sources.yml index d110ed98492..256f89463c4 100644 --- a/sources/balancer/ethereum/balancer_ethereum_sources.yml +++ b/sources/balancer/ethereum/balancer_ethereum_sources.yml @@ -352,3 +352,37 @@ sources: description: "veBAL holder that voted for the gauge" - name: weight description: "Weight of the user for the gauge" + + - name: balancer_v3_ethereum + description: > + Decoded tables related to Balancer V3, an automated portfolio manager and trading platform, on Ethereum. + tables: + - name: Vault_evt_PoolRegistered + + - name: WeightedPoolFactory_evt_PoolCreated + + - name: WeightedPoolFactory_call_create + + - name: Vault_evt_LiquidityAdded + + - name: Vault_evt_LiquidityRemoved + + - name: StablePoolFactory_evt_PoolCreated + + - name: StablePoolFactory_call_create + + - name: Vault_evt_Swap + + - name: ProtocolFeeController_evt_ProtocolSwapFeeCollected + + - name: ProtocolFeeController_evt_ProtocolYieldFeeCollected + + - name: Vault_evt_Wrap + + - name: Vault_evt_Unwrap + + - name: Vault_evt_LiquidityAddedToBuffer + + - name: Vault_evt_LiquidityRemovedFromBuffer + + - name: Vault_evt_SwapFeePercentageChanged \ No newline at end of file diff --git a/sources/balancer/gnosis/balancer_gnosis_sources.yml b/sources/balancer/gnosis/balancer_gnosis_sources.yml index f640f31cb22..af033124ccb 100644 --- a/sources/balancer/gnosis/balancer_gnosis_sources.yml +++ b/sources/balancer/gnosis/balancer_gnosis_sources.yml @@ -245,3 +245,36 @@ sources: - name: assetManager description: 'Address of the contract which managed the asset from the vault' + - name: balancer_v3_gnosis + description: > + Decoded tables related to Balancer V2, an automated portfolio manager and trading platform, on Gnosis. + tables: + - name: Vault_evt_PoolRegistered + + - name: WeightedPoolFactory_evt_PoolCreated + + - name: WeightedPoolFactory_call_create + + - name: Vault_evt_LiquidityAdded + + - name: Vault_evt_LiquidityRemoved + + - name: StablePoolFactory_evt_PoolCreated + + - name: StablePoolFactory_call_create + + - name: Vault_evt_Swap + + - name: ProtocolFeeController_evt_ProtocolSwapFeeCollected + + - name: ProtocolFeeController_evt_ProtocolYieldFeeCollected + + - name: Vault_evt_Wrap + + - name: Vault_evt_Unwrap + + - name: Vault_evt_LiquidityAddedToBuffer + + - name: Vault_evt_LiquidityRemovedFromBuffer + + - name: Vault_evt_SwapFeePercentageChanged \ No newline at end of file diff --git a/sources/morpho/ethereum/morpho_ethereum_sources.yml b/sources/morpho/ethereum/morpho_ethereum_sources.yml new file mode 100644 index 00000000000..112164e10bb --- /dev/null +++ b/sources/morpho/ethereum/morpho_ethereum_sources.yml @@ -0,0 +1,8 @@ +version: 2 + +sources: + - name: metamorpho_factory_ethereum + description: "Decoded contracts for Morpho on Ethereum" + + tables: + - name: MetaMorphoFactory_evt_CreateMetaMorpho \ No newline at end of file From 994e4d8f9388e71d974b193384a019d86f215528 Mon Sep 17 00:00:00 2001 From: jeff-dude <102681548+jeff-dude@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:35:20 -0500 Subject: [PATCH 16/59] update ronin token metadata (#7281) --- .../tokens/models/prices/ronin/prices_ronin_tokens.sql | 3 +-- .../tokens/models/tokens/ronin/tokens_ronin_erc20.sql | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/tokens/models/prices/ronin/prices_ronin_tokens.sql b/dbt_subprojects/tokens/models/prices/ronin/prices_ronin_tokens.sql index 00c77acfc2a..5b4b180f5bb 100644 --- a/dbt_subprojects/tokens/models/prices/ronin/prices_ronin_tokens.sql +++ b/dbt_subprojects/tokens/models/prices/ronin/prices_ronin_tokens.sql @@ -25,6 +25,5 @@ FROM , ('slp-smooth-love-potion', 'SLP', 0xa8754b9fa15fc18bb59458815510e40a12cd2014, 0) , ('pixel-pixels', 'PIXEL', 0x7eae20d11ef8c779433eb24503def900b9d28ad7, 18) , ('ygg-yield-guild-games', 'YGG', 0x1c306872bc82525d72bf3562e8f0aa3f8f26e857, 18) - , ('lua-lumi-finance', 'LUA', 0x18d2bdef572c67127e218c425f546fe64430a92c, 18) - + , ('lua-lumi-finance', 'LUA', 0xd61bbbb8369c46c15868ad9263a2710aced156c4, 18) ) as temp (token_id, symbol, contract_address, decimals) diff --git a/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql b/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql index 69567991704..52bcf26685e 100644 --- a/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql @@ -17,4 +17,7 @@ FROM (VALUES , (0x0b7007c13325c48911f73a2dad5fa5dcbf808adc, 'USDC', 6) , (0x97a9107c1793bc407d6f527b77e7fff4d812bece, 'AXS', 18) , (0xa8754b9fa15fc18bb59458815510e40a12cd2014, 'SLP', 0) + , (0x18d2bdef572c67127e218c425f546fe64430a92c, 'LUAUSD', 18) + , (0x7eae20d11ef8c779433eb24503def900b9d28ad7, 'PIXEL', 18) + , (0xd61bbbb8369c46c15868ad9263a2710aced156c4, 'LUA', 18) ) AS temp_table (contract_address, symbol, decimals) \ No newline at end of file From ce7f0969d60c99cbc15eff74ef0993fa1caa693c Mon Sep 17 00:00:00 2001 From: maybeyonas Date: Tue, 10 Dec 2024 19:10:02 +0530 Subject: [PATCH 17/59] adding swell ercs (#7288) --- .../tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql index 79c8d74e04e..d80493f4063 100644 --- a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql @@ -4725,4 +4725,7 @@ FROM (VALUES , ( 0x9994E35Db50125E0DF82e4c2dde62496CE330999, 'legacyMORPHO', 18) , ( 0xc824a08db624942c5e5f330d56530cd1598859fd, 'hgETH', 18) , ( 0x6e6B7ADfC7Db9fEeB8896418aC3422966f65D0A5, 'NET', 18) + , ( 0x358d94b5b2F147D741088803d932Acb566acB7B6, 'rSWELL', 18) + , ( 0x9Ed15383940CC380fAEF0a75edacE507cC775f22, 'earnETH', 18) + , ( 0x66E47E6957B85Cf62564610B76dD206BB04d831a, 'earnBTC', 8) ) AS temp_table (contract_address, symbol, decimals) From eff1412796ec85b5ef28d643c37b852860d6e70b Mon Sep 17 00:00:00 2001 From: synthquest <112836792+Synthquest@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:54:54 -0500 Subject: [PATCH 18/59] init transfers for lifinity v2 (#7266) --- .../models/_sector/dex/lifinity/lifinity_v2_base_trades.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v2_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v2_base_trades.sql index 49c39380a2a..8586e303b62 100644 --- a/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v2_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v2_base_trades.sql @@ -45,7 +45,7 @@ WITH order by COALESCE(tr_2.inner_instruction_index, 0) asc) as first_transfer_out FROM {{ source('lifinity_amm_v2_solana', 'lifinity_amm_v2_call_swap') }} sp INNER JOIN {{ ref('tokens_solana_transfers') }} tr_1 - ON tr_1.tx_id = sp.call_tx_id + ON tr_1.tx_id = sp.call_tx_id AND tr_1.action = 'transfer' AND tr_1.outer_instruction_index = sp.call_outer_instruction_index AND ((sp.call_is_inner = false AND tr_1.inner_instruction_index = 1) OR (sp.call_is_inner = true AND tr_1.inner_instruction_index = sp.call_inner_instruction_index + 1)) @@ -57,7 +57,7 @@ WITH {% endif %} --swap out can be either 2nd or 3rd transfer. INNER JOIN {{ ref('tokens_solana_transfers') }} tr_2 - ON tr_2.tx_id = sp.call_tx_id + ON tr_2.tx_id = sp.call_tx_id AND tr_2.action = 'transfer' AND tr_2.outer_instruction_index = sp.call_outer_instruction_index AND ((sp.call_is_inner = false AND (tr_2.inner_instruction_index = 2 OR tr_2.inner_instruction_index = 3)) OR (sp.call_is_inner = true AND (tr_2.inner_instruction_index = sp.call_inner_instruction_index + 2 OR tr_2.inner_instruction_index = sp.call_inner_instruction_index + 3)) From 7b4dbe5dd8e446a3ad3a346d3768a358ad94527a Mon Sep 17 00:00:00 2001 From: hildobby Date: Tue, 10 Dec 2024 20:55:34 +0100 Subject: [PATCH 19/59] Add Taiko and PandaSea blobs (#7278) * add taiko and pandasea blobs * fix --- .../_sector/blobs/ethereum/blobs_based_submitters.sql | 9 ++++++++- .../models/_sector/blobs/ethereum/blobs_submitters.sql | 1 + sources/_sector/blobs/_sources.yml | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_based_submitters.sql b/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_based_submitters.sql index c9f31f4afab..ee1d13174de 100644 --- a/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_based_submitters.sql +++ b/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_based_submitters.sql @@ -3,4 +3,11 @@ alias = 'based_submitters' )}} -SELECT evt_tx_hash AS tx_hash, evt_block_number AS block_number, 'Taiko' AS entity FROM {{ source('taikoxyz_ethereum', 'TaikoL1_evt_BlockProposed')}} \ No newline at end of file +SELECT DISTINCT tx_hash, block_number, entity +FROM ( + SELECT evt_tx_hash AS tx_hash, evt_block_number AS block_number, 'Taiko' AS entity FROM {{ source('taikoxyz_ethereum', 'TaikoL1_evt_BlockProposed')}} + + UNION ALL + + SELECT evt_tx_hash AS tx_hash, evt_block_number AS block_number, 'Taiko' AS entity FROM {{ source('taikoxyz_ethereum', 'TaikoL1_evt_BlockProposedV2')}} + ) \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql b/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql index 1a97acf2768..765e3e7e0c0 100644 --- a/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql +++ b/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql @@ -57,6 +57,7 @@ SELECT address , (0x8cda8351236199af7532bad53d683ddd9b275d89, 'RACE', 'RACEecosystem') , (0x7f9d9c1bce1062e1077845ea39a0303429600a06, 'Binary', 'thebinaryhldgs') , (0x4d875acfd836eb3d8a2f25ba03de16c97ec9fc0f, 'PandaSea', 'pandaseaweb3') + , (0xf4559766b24402f0fff8fc5a6835d5f6e837aed3, 'PandaSea', 'pandaseaweb3') , (0xf7ca543d652e38692fd12f989eb55b5327ec9a20, 'Shape', 'Shape_L2') , (0xde794bec196832474f2f218135bfd0f7ca7fb038, 'Swan Chain', 'swan_chain') , (0x67a44ce38627f46f20b1293960559ed85dd194f1, 'Polynomial', 'PolynomialFi') diff --git a/sources/_sector/blobs/_sources.yml b/sources/_sector/blobs/_sources.yml index 8a6dfec62f6..a988a85d5e4 100644 --- a/sources/_sector/blobs/_sources.yml +++ b/sources/_sector/blobs/_sources.yml @@ -4,4 +4,5 @@ sources: - name: taikoxyz_ethereum description: "Taiko smart contracts" tables: - - name: TaikoL1_evt_BlockProposed \ No newline at end of file + - name: TaikoL1_evt_BlockProposed + - name: TaikoL1_evt_BlockProposedV2 \ No newline at end of file From cd9372c09a60be18f6e6d9e1f7d430dffe772b4e Mon Sep 17 00:00:00 2001 From: blanchemaria6 <130497866+blanchemaria6@users.noreply.github.com> Date: Wed, 11 Dec 2024 02:55:45 +0700 Subject: [PATCH 20/59] Update dex.trades with Elkfinance (#7289) * Update _schema.yml * Update dex_gnosis_base_trades.sql * Create elk_finance_gnosis_base_trades.sql * Update _schema.yml * Create elk_finance_gnosis_base_trades_seed.csv * Update _sources.yml --------- Co-authored-by: Huang Geyang --- .../dex/models/trades/gnosis/_schema.yml | 18 ++++++++++++++++ .../trades/gnosis/dex_gnosis_base_trades.sql | 3 ++- .../elk_finance_gnosis_base_trades.sql | 21 +++++++++++++++++++ dbt_subprojects/dex/seeds/trades/_schema.yml | 15 +++++++++++++ .../elk_finance_gnosis_base_trades_seed.csv | 2 ++ .../_sector/dex/trades/gnosis/_sources.yml | 4 ++++ 6 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/dex/models/trades/gnosis/platforms/elk_finance_gnosis_base_trades.sql create mode 100644 dbt_subprojects/dex/seeds/trades/elk_finance_gnosis_base_trades_seed.csv diff --git a/dbt_subprojects/dex/models/trades/gnosis/_schema.yml b/dbt_subprojects/dex/models/trades/gnosis/_schema.yml index 71d0b3551f1..09081786a68 100644 --- a/dbt_subprojects/dex/models/trades/gnosis/_schema.yml +++ b/dbt_subprojects/dex/models/trades/gnosis/_schema.yml @@ -80,3 +80,21 @@ models: seed_file: ref('honeyswap_gnosis_base_trades_seed') filter: version: 2 + + - name: elk_finance_gnosis_base_trades + meta: + blockchain: gnosis + sector: dex + project: elk_finance + contributors: blanchemaria6 + config: + tags: ['gnosis', 'dex', 'trades', 'elk_finance'] + description: "elk finance gnosis base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('elk_finance_gnosis_base_trades_seed') + diff --git a/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql b/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql index 1803ccbcca8..a8196517cd8 100644 --- a/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql @@ -10,6 +10,7 @@ , ref('sushiswap_v2_gnosis_base_trades') , ref('balancer_v2_gnosis_base_trades') , ref('honeyswap_v2_gnosis_base_trades') + , ref('elk_finance_gnosis_base_trades') ] %} WITH base_union AS ( @@ -48,4 +49,4 @@ WITH base_union AS ( , blockchain = 'gnosis' , columns = ['from', 'to', 'index'] ) -}} \ No newline at end of file +}} diff --git a/dbt_subprojects/dex/models/trades/gnosis/platforms/elk_finance_gnosis_base_trades.sql b/dbt_subprojects/dex/models/trades/gnosis/platforms/elk_finance_gnosis_base_trades.sql new file mode 100644 index 00000000000..be4724a9984 --- /dev/null +++ b/dbt_subprojects/dex/models/trades/gnosis/platforms/elk_finance_gnosis_base_trades.sql @@ -0,0 +1,21 @@ +{{ + config( + schema = 'elk_finance_gnosis', + alias = 'base_trades', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{{ + uniswap_compatible_v2_trades( + blockchain = 'gnosis', + project = 'elk_finance', + version = '1', + Pair_evt_Swap = source('elk_finance_gnosis', 'ElkPair_evt_Swap'), + Factory_evt_PairCreated = source('elk_finance_gnosis', 'ElkFactory_evt_PairCreated') + ) +}} diff --git a/dbt_subprojects/dex/seeds/trades/_schema.yml b/dbt_subprojects/dex/seeds/trades/_schema.yml index fb61bd5db94..ad55b74ac25 100644 --- a/dbt_subprojects/dex/seeds/trades/_schema.yml +++ b/dbt_subprojects/dex/seeds/trades/_schema.yml @@ -4514,3 +4514,18 @@ seeds: token_bought_amount_raw: uint256 token_sold_amount_raw: uint256 block_date: timestamp + + - name: elk_finance_gnosis_base_trades_seed + config: + column_types: + blockchain: varchar + project: varchar + version: varchar + tx_hash: varbinary + evt_index: uint256 + block_number: uint256 + token_bought_address: varbinary + token_sold_address: varbinary + token_bought_amount_raw: uint256 + token_sold_amount_raw: uint256 + block_date: timestamp diff --git a/dbt_subprojects/dex/seeds/trades/elk_finance_gnosis_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/elk_finance_gnosis_base_trades_seed.csv new file mode 100644 index 00000000000..e71a7ae9f61 --- /dev/null +++ b/dbt_subprojects/dex/seeds/trades/elk_finance_gnosis_base_trades_seed.csv @@ -0,0 +1,2 @@ +blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw +gnosis,elk_finance,1,2024-12-10,0xbed63f2d1d8fa78c1f6d03570b91e9f34cad5f053c1a9d3543f8677e4558e4be,101,0xe91d153e0b41518a2ce8dd3d7944fa863463a97d,0x9c58bacc331c9aa871afd802db6379a98e80cedb,37454567,87123664537934436,324953295387159 diff --git a/sources/_sector/dex/trades/gnosis/_sources.yml b/sources/_sector/dex/trades/gnosis/_sources.yml index 6755ea39db5..821fa073bb9 100644 --- a/sources/_sector/dex/trades/gnosis/_sources.yml +++ b/sources/_sector/dex/trades/gnosis/_sources.yml @@ -16,3 +16,7 @@ sources: tables: - name: UniswapV2Pair_evt_Swap - name: UniswapV2Factory_evt_PairCreated + - name: elk_finance_gnosis + tables: + - name: ElkPair_evt_Swap + - name: ElkFactory_evt_PairCreated From 1516bdd0f3aa01c18ccdf6b9bcb82d7ab71283d2 Mon Sep 17 00:00:00 2001 From: intensodefi Date: Wed, 11 Dec 2024 02:56:40 +0700 Subject: [PATCH 21/59] Add Jetswap on polygon to dex.trades (#7287) * Update dex_info.sql * Update _schema.yml * Update dex_polygon_base_trades.sql * Create jetswap_polygon_base_trades.sql * Update _schema.yml * Create jetswap_polygon_base_trades_seed * Update _sources.yml * Delete dbt_subprojects/dex/seeds/trades/jetswap_polygon_base_trades_seed * Create jetswap_polygon_base_trades_seed.csv * Update jetswap_polygon_base_trades.sql --------- Co-authored-by: Huang Geyang Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- dbt_subprojects/dex/models/dex_info.sql | 1 + .../dex/models/trades/polygon/_schema.yml | 18 +++++++++++++++++ .../polygon/dex_polygon_base_trades.sql | 1 + .../platforms/jetswap_polygon_base_trades.sql | 20 +++++++++++++++++++ dbt_subprojects/dex/seeds/trades/_schema.yml | 15 ++++++++++++++ .../jetswap_polygon_base_trades_seed.csv | 2 ++ .../_sector/dex/trades/polygon/_sources.yml | 4 ++++ 7 files changed, 61 insertions(+) create mode 100644 dbt_subprojects/dex/models/trades/polygon/platforms/jetswap_polygon_base_trades.sql create mode 100644 dbt_subprojects/dex/seeds/trades/jetswap_polygon_base_trades_seed.csv diff --git a/dbt_subprojects/dex/models/dex_info.sql b/dbt_subprojects/dex/models/dex_info.sql index ba4dd6586d7..6fb1a33d621 100644 --- a/dbt_subprojects/dex/models/dex_info.sql +++ b/dbt_subprojects/dex/models/dex_info.sql @@ -189,4 +189,5 @@ FROM (VALUES , ('stablebase', 'StableBase', 'Direct', 'stablebasefi') , ('flashliquidity', 'Flashliquidity', 'Direct', 'flashliquidity') , ('akronswap', 'Akronswap', 'Direct', 'AkronFinance') + , ('jetswap', 'JetSwap', 'Direct', 'Jetfuelfinance') ) AS temp_table (project, name, marketplace_type, x_username) diff --git a/dbt_subprojects/dex/models/trades/polygon/_schema.yml b/dbt_subprojects/dex/models/trades/polygon/_schema.yml index a267955034e..ede80f4ae2f 100644 --- a/dbt_subprojects/dex/models/trades/polygon/_schema.yml +++ b/dbt_subprojects/dex/models/trades/polygon/_schema.yml @@ -396,3 +396,21 @@ models: - evt_index - check_dex_base_trades_seed: seed_file: ref('dfyn_polygon_base_trades_seed') + + - name: jetswap_polygon_base_trades + meta: + blockchain: polygon + sector: dex + project: jetswap + contributors: intensodefi + config: + tags: [ 'polygon', 'dex', 'trades', 'jetswap' ] + description: "jetswap polygon base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('jetswap_polygon_base_trades_seed') + diff --git a/dbt_subprojects/dex/models/trades/polygon/dex_polygon_base_trades.sql b/dbt_subprojects/dex/models/trades/polygon/dex_polygon_base_trades.sql index 7ffdae04a9b..2a324993760 100644 --- a/dbt_subprojects/dex/models/trades/polygon/dex_polygon_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/polygon/dex_polygon_base_trades.sql @@ -26,6 +26,7 @@ , ref('gridex_polygon_base_trades') , ref('swaap_v2_polygon_base_trades') , ref('dfyn_polygon_base_trades') + , ref('jetswap_polygon_base_trades') ] %} WITH base_union AS ( diff --git a/dbt_subprojects/dex/models/trades/polygon/platforms/jetswap_polygon_base_trades.sql b/dbt_subprojects/dex/models/trades/polygon/platforms/jetswap_polygon_base_trades.sql new file mode 100644 index 00000000000..2591a61614d --- /dev/null +++ b/dbt_subprojects/dex/models/trades/polygon/platforms/jetswap_polygon_base_trades.sql @@ -0,0 +1,20 @@ +{{ config( + schema = 'jetswap_polygon' + , alias = 'base_trades' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['tx_hash', 'evt_index'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{{ + uniswap_compatible_v2_trades( + blockchain = 'polygon', + project = 'jetswap', + version = '1', + Pair_evt_Swap = source('jetswap_polygon', 'JetswapPair_evt_Swap'), + Factory_evt_PairCreated = source('jetswap_polygon', 'JetswapFactory_evt_PairCreated') + ) +}} diff --git a/dbt_subprojects/dex/seeds/trades/_schema.yml b/dbt_subprojects/dex/seeds/trades/_schema.yml index ad55b74ac25..b38b6aae96a 100644 --- a/dbt_subprojects/dex/seeds/trades/_schema.yml +++ b/dbt_subprojects/dex/seeds/trades/_schema.yml @@ -4515,6 +4515,21 @@ seeds: token_sold_amount_raw: uint256 block_date: timestamp + - name: jetswap_polygon_base_trades_seed + config: + column_types: + blockchain: varchar + project: varchar + version: varchar + tx_hash: varbinary + evt_index: uint256 + block_number: uint256 + token_bought_address: varbinary + token_sold_address: varbinary + token_bought_amount_raw: uint256 + token_sold_amount_raw: uint256 + block_date: timestamp + - name: elk_finance_gnosis_base_trades_seed config: column_types: diff --git a/dbt_subprojects/dex/seeds/trades/jetswap_polygon_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/jetswap_polygon_base_trades_seed.csv new file mode 100644 index 00000000000..74b5155a843 --- /dev/null +++ b/dbt_subprojects/dex/seeds/trades/jetswap_polygon_base_trades_seed.csv @@ -0,0 +1,2 @@ +blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw +polygon,jetswap,1,2024-12-05,0x0aa7125eef5aefd2ca7df0e8e9161bea1c33f61b62cd3974d7be6a3726fcebba,365,0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270,0xe188edda9acc0166dd7030023bd33a8c6be7a0ff,65113490,175557149854780174,573564828367 diff --git a/sources/_sector/dex/trades/polygon/_sources.yml b/sources/_sector/dex/trades/polygon/_sources.yml index 81959c6337b..1c1a1ddd394 100644 --- a/sources/_sector/dex/trades/polygon/_sources.yml +++ b/sources/_sector/dex/trades/polygon/_sources.yml @@ -75,3 +75,7 @@ sources: tables: - name: UniswapV2Pair_evt_Swap - name: UniswapV2Factory_evt_PairCreated + - name: jetswap_polygon + tables: + - name: JetswapPair_evt_Swap + - name: JetswapFactory_evt_PairCreated From 39d38e92f849dab70549162c1a43dbe67cd84a5d Mon Sep 17 00:00:00 2001 From: jeff-dude <102681548+jeff-dude@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:58:32 -0500 Subject: [PATCH 22/59] DUX-798 ronin: initiate katana dex (#7280) * initiate katana dex * shorten CI timeframe * Revert "shorten CI timeframe" This reverts commit db5ba999c89ddbf7e8931173a98f76805bdafd7e. --- dbt_subprojects/dex/models/dex_info.sql | 1 + dbt_subprojects/dex/models/trades/_schema.yml | 4 +- .../dex/models/trades/dex_base_trades.sql | 1 + .../dex/models/trades/dex_trades.sql | 1 + .../dex/models/trades/ronin/_schema.yml | 23 ++++++++ .../trades/ronin/dex_ronin_base_trades.sql | 48 ++++++++++++++++ .../platforms/katana_ronin_base_trades.sql | 56 +++++++++++++++++++ dbt_subprojects/dex/seeds/trades/_schema.yml | 15 +++++ .../trades/katana_ronin_base_trades_seed.csv | 3 + sources/_sector/dex/trades/ronin/_sources.yml | 7 +++ 10 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 dbt_subprojects/dex/models/trades/ronin/_schema.yml create mode 100644 dbt_subprojects/dex/models/trades/ronin/dex_ronin_base_trades.sql create mode 100644 dbt_subprojects/dex/models/trades/ronin/platforms/katana_ronin_base_trades.sql create mode 100644 dbt_subprojects/dex/seeds/trades/katana_ronin_base_trades_seed.csv create mode 100644 sources/_sector/dex/trades/ronin/_sources.yml diff --git a/dbt_subprojects/dex/models/dex_info.sql b/dbt_subprojects/dex/models/dex_info.sql index 6fb1a33d621..f5ffa82a7d7 100644 --- a/dbt_subprojects/dex/models/dex_info.sql +++ b/dbt_subprojects/dex/models/dex_info.sql @@ -189,5 +189,6 @@ FROM (VALUES , ('stablebase', 'StableBase', 'Direct', 'stablebasefi') , ('flashliquidity', 'Flashliquidity', 'Direct', 'flashliquidity') , ('akronswap', 'Akronswap', 'Direct', 'AkronFinance') + , ('katana', 'Katana', 'Direct', 'AxieInfinity') , ('jetswap', 'JetSwap', 'Direct', 'Jetfuelfinance') ) AS temp_table (project, name, marketplace_type, x_username) diff --git a/dbt_subprojects/dex/models/trades/_schema.yml b/dbt_subprojects/dex/models/trades/_schema.yml index af7849351df..c913109f5c5 100644 --- a/dbt_subprojects/dex/models/trades/_schema.yml +++ b/dbt_subprojects/dex/models/trades/_schema.yml @@ -4,7 +4,7 @@ models: - name: dex_trades meta: docs_slug: /curated/trading/DEX/dex-trades - blockchain: arbitrum, avalanche_c, base, bnb, celo, ethereum, fantom, gnosis, kaia, optimism, polygon, scroll, zksync, linea, blast, sei + blockchain: arbitrum, avalanche_c, base, bnb, celo, ethereum, fantom, gnosis, kaia, optimism, polygon, scroll, zksync, linea, blast, sei, ronin sector: dex short_description: The `dex.trades` table captures detailed data on trades executed via decentralized exchanges (DEXs). This table contains a detailed breakdown of trade execution containing one or many trades per transaction. contributors: 0xRob, hosuke, jeff-dude, tomfutago @@ -102,7 +102,7 @@ models: - name: dex_base_trades meta: - blockchain: arbitrum, avalanche_c, base, bnb, celo, ethereum, fantom, gnosis, kaia,optimism, polygon, scroll, zksync, linea, blast, sei + blockchain: arbitrum, avalanche_c, base, bnb, celo, ethereum, fantom, gnosis, kaia,optimism, polygon, scroll, zksync, linea, blast, sei, ronin sector: dex contributors: 0xRob, hosuke, jeff-dude, tomfutago, viniabussafi config: diff --git a/dbt_subprojects/dex/models/trades/dex_base_trades.sql b/dbt_subprojects/dex/models/trades/dex_base_trades.sql index 944f36da985..6a5a68bb734 100644 --- a/dbt_subprojects/dex/models/trades/dex_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_base_trades.sql @@ -26,6 +26,7 @@ , ref('dex_nova_base_trades') , ref('dex_optimism_base_trades') , ref('dex_polygon_base_trades') + , ref('dex_ronin_base_trades') , ref('dex_scroll_base_trades') , ref('dex_sei_base_trades') , ref('dex_worldchain_base_trades') diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index 0452bb2d1db..654e2eb5e4a 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -23,6 +23,7 @@ , "nova" , "optimism" , "polygon" + , "ronin" , "scroll" , "sei" , "zkevm" diff --git a/dbt_subprojects/dex/models/trades/ronin/_schema.yml b/dbt_subprojects/dex/models/trades/ronin/_schema.yml new file mode 100644 index 00000000000..9d9035dda64 --- /dev/null +++ b/dbt_subprojects/dex/models/trades/ronin/_schema.yml @@ -0,0 +1,23 @@ +version: 2 + +models: + - name: dex_ronin_base_trades + data_tests: + - check_dex_info_relationship + + - name: katana_ronin_base_trades + meta: + blockchain: ronin + sector: dex + project: katana + contributors: jeff-dude + config: + tags: [ 'ronin', 'dex', 'trades', 'katana' ] + description: "Katana ronin base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('katana_ronin_base_trades_seed') \ No newline at end of file diff --git a/dbt_subprojects/dex/models/trades/ronin/dex_ronin_base_trades.sql b/dbt_subprojects/dex/models/trades/ronin/dex_ronin_base_trades.sql new file mode 100644 index 00000000000..1cd7eaa3262 --- /dev/null +++ b/dbt_subprojects/dex/models/trades/ronin/dex_ronin_base_trades.sql @@ -0,0 +1,48 @@ +{{ config( + schema = 'dex_ronin' + , alias = 'base_trades' + , materialized = 'view' + ) +}} + +{% set base_models = [ + ref('katana_ronin_base_trades') +] %} + +WITH base_union AS ( + SELECT * + FROM ( + {% for base_model in base_models %} + SELECT + blockchain + , project + , version + , block_month + , block_date + , block_time + , block_number + , token_bought_amount_raw + , token_sold_amount_raw + , token_bought_address + , token_sold_address + , taker + , maker + , project_contract_address + , tx_hash + , evt_index + FROM + {{ base_model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} + ) +) + +{{ + add_tx_columns( + model_cte = 'base_union' + , blockchain = 'ronin' + , columns = ['from', 'to', 'index'] + ) +}} diff --git a/dbt_subprojects/dex/models/trades/ronin/platforms/katana_ronin_base_trades.sql b/dbt_subprojects/dex/models/trades/ronin/platforms/katana_ronin_base_trades.sql new file mode 100644 index 00000000000..c2f1d62f8a6 --- /dev/null +++ b/dbt_subprojects/dex/models/trades/ronin/platforms/katana_ronin_base_trades.sql @@ -0,0 +1,56 @@ +{{ config( + schema = 'katana_ronin' + , alias = 'base_trades' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['tx_hash', 'evt_index'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +WITH dexs AS +( + SELECT + t.evt_block_number AS block_number + , t.evt_block_time AS block_time + , t._to AS taker + , t.contract_address AS maker + , CASE WHEN _amount0Out = UINT256 '0' THEN _amount1Out ELSE _amount0Out END AS token_bought_amount_raw + , CASE WHEN _amount0In = UINT256 '0' OR _amount1Out = UINT256 '0' THEN _amount1In ELSE _amount0In END AS token_sold_amount_raw + , CASE WHEN _amount0Out = UINT256 '0' THEN f._token1 ELSE f._token0 END AS token_bought_address + , CASE WHEN _amount0In = UINT256 '0' OR _amount1Out = UINT256 '0' THEN f._token1 ELSE f._token0 END AS token_sold_address + , t.contract_address AS project_contract_address + , t.evt_tx_hash AS tx_hash + , t.evt_index AS evt_index + FROM + {{ source('katana_dex_ronin', 'KatanaPair_evt_Swap') }} t + INNER JOIN + {{ source('katana_dex_ronin', 'KatanaFactory_evt_PairCreated') }} f + ON f._pair = t.contract_address + {% if is_incremental() %} + WHERE + {{ incremental_predicate('t.evt_block_time') }} + {% endif %} + +) + +SELECT + 'ronin' AS blockchain + , 'katana' AS project + , '2' AS version + , CAST(date_trunc('month', dexs.block_time) AS date) AS block_month + , CAST(date_trunc('day', dexs.block_time) AS date) AS block_date + , dexs.block_time + , dexs.block_number + , dexs.token_bought_amount_raw + , dexs.token_sold_amount_raw + , dexs.token_bought_address + , dexs.token_sold_address + , dexs.taker + , dexs.maker + , dexs.project_contract_address + , dexs.tx_hash + , dexs.evt_index +FROM + dexs \ No newline at end of file diff --git a/dbt_subprojects/dex/seeds/trades/_schema.yml b/dbt_subprojects/dex/seeds/trades/_schema.yml index b38b6aae96a..5164c88beac 100644 --- a/dbt_subprojects/dex/seeds/trades/_schema.yml +++ b/dbt_subprojects/dex/seeds/trades/_schema.yml @@ -4514,6 +4514,21 @@ seeds: token_bought_amount_raw: uint256 token_sold_amount_raw: uint256 block_date: timestamp + + - name: katana_ronin_base_trades_seed + config: + column_types: + blockchain: varchar + project: varchar + version: varchar + tx_hash: varbinary + evt_index: uint256 + block_number: uint256 + token_bought_address: varbinary + token_sold_address: varbinary + token_bought_amount_raw: uint256 + token_sold_amount_raw: uint256 + block_date: timestamp - name: jetswap_polygon_base_trades_seed config: diff --git a/dbt_subprojects/dex/seeds/trades/katana_ronin_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/katana_ronin_base_trades_seed.csv new file mode 100644 index 00000000000..d8a0d2f8ea0 --- /dev/null +++ b/dbt_subprojects/dex/seeds/trades/katana_ronin_base_trades_seed.csv @@ -0,0 +1,3 @@ +blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw +ronin,katana,2,2024-01-17,0xca2c3b5d6aec890d5bfe1fb8616036a01b702948107fdb4a243453cfe5a6d86a,10,0x1b918543b518e34902e1e8dd76052bee43c762ff,0xe514d9deb7966c8be0ca922de8a064264ea6bcd4,31230074,12492104858888467232497,5109345881300390132 +ronin,katana,2,2023-03-01,0xcab674e07e3c5b0b6dd53b52971edabc929a2fd907b85d781f70bac74767c6e3,10,0xc99a6a985ed2cac1ef41640596c5a5f9f4e19ef5,0xe514d9deb7966c8be0ca922de8a064264ea6bcd4,22465731,1700000000000000,3070678253641756888 \ No newline at end of file diff --git a/sources/_sector/dex/trades/ronin/_sources.yml b/sources/_sector/dex/trades/ronin/_sources.yml new file mode 100644 index 00000000000..8c61432710f --- /dev/null +++ b/sources/_sector/dex/trades/ronin/_sources.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: katana_dex_ronin + tables: + - name: KatanaPair_evt_Swap + - name: KatanaFactory_evt_PairCreated \ No newline at end of file From 4bf03f7f2bfafa07e9770f789296c66ebda47dbd Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Tue, 10 Dec 2024 20:59:03 +0100 Subject: [PATCH 23/59] [cowprotocol] Fix trades, batches and solver models (#7253) * [cowprotocol] Fix trades, batches and solver models * add ETH flow schema * fix arbitrum solver ref * fix arbitrum solver ref * fix arbitrum solver ref * fix arbitrum solver ref * use ref * fix other chains * remove BASE reference * fix gnosis table * put back arbitrum batches --- .../arbitrum/cow_protocol_arbitrum_trades.sql | 84 +++++++++++++------ .../gnosis/cow_protocol_gnosis_trades.sql | 69 +++++++++------ .../cow_protocol_arbitrum_batches.sql | 39 ++------- .../cow_protocol_arbitrum_solvers.sql | 12 +-- .../cow_protocol_ethereum_batches.sql | 12 +-- .../cow_protocol_ethereum_solvers.sql | 10 +-- .../gnosis/cow_protocol_gnosis_batches.sql | 2 +- .../cow_protocol_arbitrum_sources.yml | 36 ++++++++ .../gnosis/cow_protocol_gnosis_sources.yml | 36 ++++++++ 9 files changed, 200 insertions(+), 100 deletions(-) diff --git a/dbt_subprojects/dex/models/_projects/cow_protocol/arbitrum/cow_protocol_arbitrum_trades.sql b/dbt_subprojects/dex/models/_projects/cow_protocol/arbitrum/cow_protocol_arbitrum_trades.sql index f8d70b4fbd5..71743c9ac54 100644 --- a/dbt_subprojects/dex/models/_projects/cow_protocol/arbitrum/cow_protocol_arbitrum_trades.sql +++ b/dbt_subprojects/dex/models/_projects/cow_protocol/arbitrum/cow_protocol_arbitrum_trades.sql @@ -1,17 +1,17 @@ {{ config( - schema = 'cow_protocol_arbitrum', - alias='trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['tx_hash', 'order_uid', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - post_hook='{{ expose_spells(\'["arbitrum"]\', - "project", - "cow_protocol", - \'["olgafetisova"]\') }}' + schema='cow_protocol_arbitrum', + alias='trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['tx_hash', 'order_uid', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + post_hook='{{ expose_spells(\'["arbitrum"]\', + "project", + "cow_protocol", + \'["olgafetisova"]\') }}' ) }} @@ -26,17 +26,17 @@ trades_with_prices AS ( evt_block_number as block_number, evt_tx_hash as tx_hash, evt_index, - settlement.contract_address as project_contract_address, + trade.contract_address as project_contract_address, owner as trader, orderUid as order_uid, sellToken as sell_token, buyToken as buy_token, - (sellAmount - feeAmount) as sell_amount, + sellAmount - feeAmount as sell_amount, buyAmount as buy_amount, feeAmount as fee_amount, ps.price as sell_price, pb.price as buy_price - FROM {{ source('gnosis_protocol_v2_arbitrum', 'GPv2Settlement_evt_Trade') }} settlement + FROM {{ source('gnosis_protocol_v2_arbitrum', 'GPv2Settlement_evt_Trade') }} trade LEFT OUTER JOIN {{ source('prices', 'usd') }} as ps ON sellToken = ps.contract_address AND ps.minute = date_trunc('minute', evt_block_time) @@ -45,7 +45,12 @@ trades_with_prices AS ( AND {{ incremental_predicate('ps.minute') }} {% endif %} LEFT OUTER JOIN {{ source('prices', 'usd') }} as pb - ON pb.contract_address = buyToken + ON pb.contract_address = ( + CASE + WHEN buyToken = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + THEN 0x82af49447d8a07e3bd95bd0d56f35241523fbab1 + ELSE buyToken + END) AND pb.minute = date_trunc('minute', evt_block_time) AND pb.blockchain = 'arbitrum' {% if is_incremental() %} @@ -73,7 +78,8 @@ trades_with_token_units as ( END) as sell_token, buy_token as buy_token_address, (CASE - WHEN tb.symbol IS NULL THEN cast(buy_token AS varchar) + WHEN tb.symbol IS NULL THEN cast(buy_token as varchar) + WHEN buy_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 'ETH' ELSE tb.symbol END) as buy_token, sell_amount / pow(10, ts.decimals) as units_sold, @@ -89,7 +95,12 @@ trades_with_token_units as ( LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ts ON ts.blockchain='arbitrum' AND ts.contract_address = sell_token LEFT OUTER JOIN {{ source('tokens', 'erc20') }} tb - ON tb.blockchain='arbitrum' AND tb.contract_address = buy_token + ON tb.blockchain='arbitrum' AND tb.contract_address = + (CASE + WHEN buy_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + THEN 0x82af49447d8a07e3bd95bd0d56f35241523fbab1 + ELSE buy_token + END) ), sorted_orders as ( select @@ -146,6 +157,25 @@ uid_to_app_id as ( i = j ), +eth_flow_senders as ( + select + sender, + bytearray_concat( + bytearray_concat( + output_orderHash, + bytearray_substring(event.contract_address, 1, 20) + ), + 0xffffffff + ) AS order_uid + from {{ source('cow_protocol_arbitrum', 'CoWSwapEthFlow_evt_OrderPlacement') }} event + inner join {{ source('cow_protocol_arbitrum', 'CoWSwapEthFlow_call_createOrder') }} call + on call_block_number = evt_block_number + and call_tx_hash = evt_tx_hash + {% if is_incremental() %} + where {{ incremental_predicate('evt_block_time') }} + {% endif %} +), + valued_trades as ( SELECT block_date, block_month, @@ -156,9 +186,10 @@ valued_trades as ( ARRAY[-1] as trace_address, project_contract_address, trades.order_uid, - trader, + -- ETH Flow orders have trader = sender of orderCreation. + case when sender is not null then sender else trader end as trader, sell_token_address, - sell_token, + case when sender is not null then 'ETH' else sell_token end as sell_token, buy_token_address, buy_token, case @@ -216,11 +247,12 @@ valued_trades as ( JOIN uid_to_app_id ON uid = trades.order_uid AND hash=tx_hash + LEFT OUTER JOIN eth_flow_senders efs + ON trades.order_uid = efs.order_uid ) -select - *, - -- Relative surplus (in %) is the difference between limit price and executed price as a ratio of the limit price. - -- Absolute surplus (in USD) is relative surplus multiplied with the value of the trade - usd_value * (atoms_bought * limit_sell_amount - atoms_sold * limit_buy_amount) / (atoms_bought * limit_sell_amount) as surplus_usd +select *, + -- Relative surplus (in %) is the difference between limit price and executed price as a ratio of the limit price. + -- Absolute surplus (in USD) is relative surplus multiplied with the value of the trade + usd_value * (atoms_bought * limit_sell_amount - atoms_sold * limit_buy_amount) / (atoms_bought * limit_sell_amount) as surplus_usd from valued_trades diff --git a/dbt_subprojects/dex/models/_projects/cow_protocol/gnosis/cow_protocol_gnosis_trades.sql b/dbt_subprojects/dex/models/_projects/cow_protocol/gnosis/cow_protocol_gnosis_trades.sql index efc7923ced8..5155a6364d4 100644 --- a/dbt_subprojects/dex/models/_projects/cow_protocol/gnosis/cow_protocol_gnosis_trades.sql +++ b/dbt_subprojects/dex/models/_projects/cow_protocol/gnosis/cow_protocol_gnosis_trades.sql @@ -1,17 +1,17 @@ {{ config( - schema = 'cow_protocol_gnosis', - alias='trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['tx_hash', 'order_uid', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - post_hook='{{ expose_spells(\'["gnosis"]\', - "project", - "cow_protocol", - \'["bh2smith", "gentrexha", "olgafetisova"]\') }}' + schema='cow_protocol_gnosis', + alias='trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['tx_hash', 'order_uid', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + post_hook='{{ expose_spells(\'["gnosis"]\', + "project", + "cow_protocol", + \'["bh2smith", "gentrexha", "olgafetisova"]\') }}' ) }} @@ -26,17 +26,17 @@ trades_with_prices AS ( evt_block_number as block_number, evt_tx_hash as tx_hash, evt_index, - settlement.contract_address as project_contract_address, + trade.contract_address as project_contract_address, owner as trader, orderUid as order_uid, sellToken as sell_token, buyToken as buy_token, - (sellAmount - feeAmount) as sell_amount, + sellAmount - feeAmount as sell_amount, buyAmount as buy_amount, feeAmount as fee_amount, ps.price as sell_price, pb.price as buy_price - FROM {{ source('gnosis_protocol_v2_gnosis', 'GPv2Settlement_evt_Trade') }} settlement + FROM {{ source('gnosis_protocol_v2_gnosis', 'GPv2Settlement_evt_Trade') }} trade LEFT OUTER JOIN {{ source('prices', 'usd') }} as ps ON sellToken = ps.contract_address AND ps.minute = date_trunc('minute', evt_block_time) @@ -78,7 +78,7 @@ trades_with_token_units as ( END) as sell_token, buy_token as buy_token_address, (CASE - WHEN tb.symbol IS NULL THEN cast(buy_token AS varchar) + WHEN tb.symbol IS NULL THEN cast(buy_token as varchar) WHEN buy_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 'xDAI' ELSE tb.symbol END) as buy_token, @@ -157,6 +157,25 @@ uid_to_app_id as ( i = j ), +eth_flow_senders as ( + select + sender, + bytearray_concat( + bytearray_concat( + output_orderHash, + bytearray_substring(event.contract_address, 1, 20) + ), + 0xffffffff + ) AS order_uid + from {{ source('cow_protocol_gnosis', 'CoWSwapEthFlow_evt_OrderPlacement') }} event + inner join {{ source('cow_protocol_gnosis', 'CoWSwapEthFlow_call_createOrder') }} call + on call_block_number = evt_block_number + and call_tx_hash = evt_tx_hash + {% if is_incremental() %} + where {{ incremental_predicate('evt_block_time') }} + {% endif %} +), + valued_trades as ( SELECT block_date, block_month, @@ -167,9 +186,10 @@ valued_trades as ( ARRAY[-1] as trace_address, project_contract_address, trades.order_uid, - trader, + -- ETH Flow orders have trader = sender of orderCreation. + case when sender is not null then sender else trader end as trader, sell_token_address, - sell_token, + case when sender is not null then 'xDAI' else sell_token end as sell_token, buy_token_address, buy_token, case @@ -227,11 +247,12 @@ valued_trades as ( JOIN uid_to_app_id ON uid = trades.order_uid AND hash=tx_hash + LEFT OUTER JOIN eth_flow_senders efs + ON trades.order_uid = efs.order_uid ) -select - *, - -- Relative surplus (in %) is the difference between limit price and executed price as a ratio of the limit price. - -- Absolute surplus (in USD) is relative surplus multiplied with the value of the trade - usd_value * (atoms_bought * limit_sell_amount - atoms_sold * limit_buy_amount) / (atoms_bought * limit_sell_amount) as surplus_usd +select *, + -- Relative surplus (in %) is the difference between limit price and executed price as a ratio of the limit price. + -- Absolute surplus (in USD) is relative surplus multiplied with the value of the trade + usd_value * (atoms_bought * limit_sell_amount - atoms_sold * limit_buy_amount) / (atoms_bought * limit_sell_amount) as surplus_usd from valued_trades diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql index 24679ceaacb..46c51e364c8 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql @@ -16,41 +16,13 @@ }} WITH --- Find the PoC Query here: https://dune.com/queries/3840594 -solver_activation_events AS ( - SELECT - solver, - evt_block_number, - evt_index, - TRUE AS activated - FROM {{ source('gnosis_protocol_v2_arbitrum', 'GPv2AllowListAuthentication_evt_SolverAdded') }} - UNION - SELECT - solver, - evt_block_number, - evt_index, - FALSE AS activated - FROM {{ source('gnosis_protocol_v2_arbitrum', 'GPv2AllowListAuthentication_evt_SolverRemoved') }} - ), - ranked_solver_events as ( - select - rank() over (partition by solver order by evt_block_number desc, evt_index desc) as rk, - solver, - evt_block_number, - evt_index, - activated as active - from solver_activation_events -), -registered_solvers as ( - select solver as address, active - from ranked_solver_events - where rk = 1 -), batch_counts as ( select try_cast(date_trunc('day', s.evt_block_time) as date) as block_date, + s.evt_block_number, s.evt_block_time, s.evt_tx_hash, solver, + name, sum( case when selector != 0x2e1a7d4d -- unwrap @@ -66,12 +38,12 @@ batch_counts as ( {% if is_incremental() %} AND {{ incremental_predicate('i.evt_block_time') }} {% endif %} - join registered_solvers + join {{ ref('cow_protocol_arbitrum_solvers') }} on solver = address {% if is_incremental() %} WHERE {{ incremental_predicate('s.evt_block_time') }} {% endif %} - group by s.evt_tx_hash, solver, s.evt_block_time + group by s.evt_block_number, s.evt_block_time, s.evt_tx_hash, solver, name ), batch_values as ( @@ -98,6 +70,7 @@ batch_values as ( combined_batch_info as ( select b.block_date, + evt_block_number as block_number, evt_block_time as block_time, num_trades, dex_swaps, @@ -117,7 +90,7 @@ combined_batch_info as ( inner join {{ source('arbitrum', 'transactions') }} tx on evt_tx_hash = hash {% if is_incremental() %} - AND {{ incremental_predicate('tx.block_time') }} + AND {{ incremental_predicate('block_time') }} {% endif %} where num_trades > 0 --! Exclude Withdraw Batches ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql index 4b73365d687..22feb72c40a 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql @@ -36,7 +36,8 @@ registered_solvers as ( -- Manually inserting environment and name for each "known" solver known_solver_metadata (address, environment, name) as ( SELECT * - FROM (VALUES + FROM ( + VALUES (0xA70892d1Af41aBD2F648FEC74Ea2c17e56Ac3B9A, 'prod', 'Naive'), (0xba36CEfb45d1CdD2Ae30a899C432c5081E095Ff8, 'prod', 'Baseline'), (0xF017C6F66D68d11AF00FD243494E3fa0EBf44C66, 'prod', 'Quasimodo'), @@ -69,12 +70,13 @@ known_solver_metadata (address, environment, name) as ( (0xc8371B2898FBaEeAe658f9FaeE8ddeDA24e37012, 'barn', 'OpenOcean_Aggregator'), (0x2aeC288B42C99D2e8e984c5C324FB069f7705186, 'barn', 'Rizzolver'), (0x26B5e3bF135D3Dd05A220508dD61f25BF1A47cBD, 'barn', 'Rizzolver') - ) as _ + ) as _ ) -- Combining the metadata with current activation status for final table select solver as address, - case when environment is not null then environment else 'new' end as environment, - case when name is not null then name else 'Uncatalogued' end as name, + case when environment is not null then environment else 'new' end as environment, + case when name is not null then name else 'Uncatalogued' end as name, active from registered_solvers - left outer join known_solver_metadata on solver = address; +left outer join known_solver_metadata on solver = address + diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql index 743499aea18..27849f7bd64 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql @@ -1,6 +1,6 @@ {{ config( + schema = 'cow_protocol_ethereum', alias = 'batches', - materialized='incremental', partition_by = ['block_date'], unique_key = ['tx_hash'], @@ -8,10 +8,10 @@ file_format ='delta', incremental_strategy='merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - post_hook='{{ expose_spells(\'["ethereum"]\', - "project", - "cow_protocol", - \'["bh2smith", "gentrexha"]\') }}' + post_hook='{{ expose_spells(blockchains = \'["ethereum"]\', + spell_type = "project", + spell_name = "cow_protocol", + contributors = \'["bh2smith", "gentrexha"]\') }}' ) }} @@ -39,7 +39,7 @@ batch_counts as ( {% if is_incremental() %} AND {{ incremental_predicate('i.evt_block_time') }} {% endif %} - join cow_protocol_ethereum.solvers + join {{ ref('cow_protocol_ethereum_solvers') }} on solver = address {% if is_incremental() %} WHERE {{ incremental_predicate('s.evt_block_time') }} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql index 489038303a2..1b598260e61 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql @@ -1,10 +1,10 @@ {{ config( + schema = 'cow_protocol_ethereum', alias='solvers', - - post_hook='{{ expose_spells(\'["ethereum"]\', - "project", - "cow_protocol", - \'["bh2smith", "gentrexha"]\') }}' + post_hook='{{ expose_spells(blockchains = \'["ethereum"]\', + spell_type = "project", + spell_name = "cow_protocol", + contributors = \'["bh2smith", "gentrexha"]\') }}' )}} -- Find the PoC Query here: https://dune.com/queries/1276806 diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql index 4654c7053ab..dd63b3cafad 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql @@ -38,7 +38,7 @@ batch_counts as ( {% if is_incremental() %} AND {{ incremental_predicate('i.evt_block_time') }} {% endif %} - join cow_protocol_gnosis.solvers + join {{ ref('cow_protocol_gnosis_solvers') }} on solver = address {% if is_incremental() %} WHERE {{ incremental_predicate('s.evt_block_time') }} diff --git a/sources/cow_protocol/arbitrum/cow_protocol_arbitrum_sources.yml b/sources/cow_protocol/arbitrum/cow_protocol_arbitrum_sources.yml index e5ee24ab4da..3aed3e46180 100644 --- a/sources/cow_protocol/arbitrum/cow_protocol_arbitrum_sources.yml +++ b/sources/cow_protocol/arbitrum/cow_protocol_arbitrum_sources.yml @@ -122,3 +122,39 @@ sources: - &value name: value description: ETH amount (in WEI) being passed into contract interaction + - name: cow_protocol_arbitrum + description: "Arbitrum decoded tables related to CoW Protocol contract's" + tables: + - name: CoWSwapEthFlow_evt_OrderPlacement + description: "CoWSwapEthFlow Order Placement Events" + columns: + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - &order + name: order + description: "Contains json encoded data for an EthFlowOrder: https://github.com/cowprotocol/ethflowcontract/blob/main/src/libraries/EthFlowOrder.sol#L19-L45" + - &data + name: data + description: "Hex Encoded data containing quoteId and validTo: https://github.com/cowprotocol/ethflowcontract/blob/9c74c8ba36ff9ff3e255172b02454f831c066865/src/CoWSwapEthFlow.sol#L110-L113" + - &sender + name: sender + description: "User who placed the order" + - &signature + name: signature + description: "Onchain EIP1271 signature" + - name: CoWSwapEthFlow_call_createOrder + description: "CoWSwapEthFlow createOrder function calls with input parameters and return value" + columns: + - *call_block_number + - *call_block_time + - *call_success + - *call_trace_address + - *call_tx_hash + - *contract_address + - *order + - &output_orderHash + name: output_orderHash + description: "ETH Flow Order Hash corresponds directly to a GPv2 Order Uid by concatenation with owner = contract_address and validTo = `ffffffff`" diff --git a/sources/cow_protocol/gnosis/cow_protocol_gnosis_sources.yml b/sources/cow_protocol/gnosis/cow_protocol_gnosis_sources.yml index 29eb6a3ba98..76e8db8581d 100644 --- a/sources/cow_protocol/gnosis/cow_protocol_gnosis_sources.yml +++ b/sources/cow_protocol/gnosis/cow_protocol_gnosis_sources.yml @@ -122,3 +122,39 @@ sources: - &value name: value description: ETH amount (in WEI) being passed into contract interaction + - name: cow_protocol_gnosis + description: "Gnosis Chain decoded tables related to CoW Protocol contract's" + tables: + - name: CoWSwapEthFlow_evt_OrderPlacement + description: "CoWSwapEthFlow Order Placement Events" + columns: + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - &order + name: order + description: "Contains json encoded data for an EthFlowOrder: https://github.com/cowprotocol/ethflowcontract/blob/main/src/libraries/EthFlowOrder.sol#L19-L45" + - &data + name: data + description: "Hex Encoded data containing quoteId and validTo: https://github.com/cowprotocol/ethflowcontract/blob/9c74c8ba36ff9ff3e255172b02454f831c066865/src/CoWSwapEthFlow.sol#L110-L113" + - &sender + name: sender + description: "User who placed the order" + - &signature + name: signature + description: "Onchain EIP1271 signature" + - name: CoWSwapEthFlow_call_createOrder + description: "CoWSwapEthFlow createOrder function calls with input parameters and return value" + columns: + - *call_block_number + - *call_block_time + - *call_success + - *call_trace_address + - *call_tx_hash + - *contract_address + - *order + - &output_orderHash + name: output_orderHash + description: "ETH Flow Order Hash corresponds directly to a GPv2 Order Uid by concatenation with owner = contract_address and validTo = `ffffffff`" From cb9e353241b0a3dea0b55522d162610b3e1cff0e Mon Sep 17 00:00:00 2001 From: imimim-username <100382074+imimim-username@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:49:15 -0500 Subject: [PATCH 24/59] Added a few tokens (#7298) * Added jUSDC and farmdWETHV3 tokens * added apxETH to tokens --- .../tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql | 2 ++ .../tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql | 1 + 2 files changed, 3 insertions(+) diff --git a/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql b/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql index 286f6aadfcb..b185e8e88f4 100644 --- a/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql @@ -121,4 +121,6 @@ FROM (VALUES , (0xb59c8912c83157a955f9d715e556257f432c35d7, 'TRUF', 18) , (0xe1d3495717f9534db67a6a8d4940dd17435b6a9e, 'LOCUS', 18) , (0x764bfc309090e7f93edce53e5befa374cdcb7b8e, 'GRIMACE', 18) + , (0xB0BDE111812EAC913b392D80D51966eC977bE3A2, 'jUSDC', 18) + , (0xf3b7994e4dA53E04155057Fd61dc501599d57877, 'farmdWETHV3', 18) ) AS temp_table (contract_address, symbol, decimals) diff --git a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql index d80493f4063..3cdd312ce0e 100644 --- a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql @@ -4728,4 +4728,5 @@ FROM (VALUES , ( 0x358d94b5b2F147D741088803d932Acb566acB7B6, 'rSWELL', 18) , ( 0x9Ed15383940CC380fAEF0a75edacE507cC775f22, 'earnETH', 18) , ( 0x66E47E6957B85Cf62564610B76dD206BB04d831a, 'earnBTC', 8) + , (0x9Ba021B0a9b958B5E75cE9f6dff97C7eE52cb3E6, 'apxETH', 18) ) AS temp_table (contract_address, symbol, decimals) From f5aba923041e8bf0fa2b653991b49e5fba30e205 Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:50:03 +0000 Subject: [PATCH 25/59] Add USDL and USR to prices.usd (#7296) * Update prices_ethereum_tokens.sql * Update prices_ethereum_tokens.sql --- .../tokens/models/prices/ethereum/prices_ethereum_tokens.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql b/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql index e70afa9db2d..78c535ee76b 100644 --- a/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql +++ b/dbt_subprojects/tokens/models/prices/ethereum/prices_ethereum_tokens.sql @@ -1868,7 +1868,9 @@ FROM ('shrub-shrub', 'ethereum', 'SHRUB', 0x3b991130eae3cca364406d718da22fa1c3e7c256, 18), ('sbr-strategic-bitcoin-reserve', 'ethereum', 'SBR', 0xd6203889c22d9fe5e938a9200f50fdffe9dd8e02, 9), ('sigma2-sigma', 'ethereum', 'SIGMA', 0x18a8D75F70eAEad79B5A55903D036ce337F623A5, 18), - ('mstr-mstr2100', 'ethereum', 'MSTR', 0x42069026EAC8Eee0Fd9b5f7aDFa4f6E6D69a2B39, 9) + ('mstr-mstr2100', 'ethereum', 'MSTR', 0x42069026EAC8Eee0Fd9b5f7aDFa4f6E6D69a2B39, 9), + ('usdl-lift-dollar', 'ethereum', 'USDL', 0xbdC7c08592Ee4aa51D06C27Ee23D5087D65aDbcD, 18), + ('usr-resolv-usr', 'ethereum', 'USR', 0x66a1e37c9b0eaddca17d3662d6c05f4decf3e110, 18) ) as temp (token_id, blockchain, symbol, contract_address, decimals) where contract_address not in ( -- bad price feeds From fd8b1de206f81c050fc8f2e641a6a540be41e4e8 Mon Sep 17 00:00:00 2001 From: Dan Smith | Blockworks Research <119909738+smyyguy@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:51:39 -0500 Subject: [PATCH 26/59] Update lifinity_v1_base_trades.sql (#7292) same update to v2 --- .../models/_sector/dex/lifinity/lifinity_v1_base_trades.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v1_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v1_base_trades.sql index c5ac3d3a3ff..8c012e3a950 100644 --- a/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v1_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/lifinity/lifinity_v1_base_trades.sql @@ -45,7 +45,7 @@ WITH order by COALESCE(tr_2.inner_instruction_index, 0) asc) as first_transfer_out FROM {{ source('lifinity_amm_solana', 'lifinity_amm_call_swap') }} sp INNER JOIN {{ ref('tokens_solana_transfers') }} tr_1 - ON tr_1.tx_id = sp.call_tx_id + ON tr_1.tx_id = sp.call_tx_id AND tr_1.action = 'transfer' AND tr_1.outer_instruction_index = sp.call_outer_instruction_index AND ((sp.call_is_inner = false AND tr_1.inner_instruction_index = 1) OR (sp.call_is_inner = true AND tr_1.inner_instruction_index = sp.call_inner_instruction_index + 1)) @@ -57,7 +57,7 @@ WITH {% endif %} --swap out can be either 2nd or 3rd transfer. INNER JOIN {{ ref('tokens_solana_transfers') }} tr_2 - ON tr_2.tx_id = sp.call_tx_id + ON tr_2.tx_id = sp.call_tx_id AND tr_2.action = 'transfer' AND tr_2.outer_instruction_index = sp.call_outer_instruction_index AND ((sp.call_is_inner = false AND (tr_2.inner_instruction_index = 2 OR tr_2.inner_instruction_index = 3)) OR (sp.call_is_inner = true AND (tr_2.inner_instruction_index = sp.call_inner_instruction_index + 2 OR tr_2.inner_instruction_index = sp.call_inner_instruction_index + 3)) From ed7bfd7f2cb36ea240ec64b00a45cb03f91442d7 Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Wed, 11 Dec 2024 21:32:32 +0000 Subject: [PATCH 27/59] Init Balancer V3 spells and macros (#7274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * commit v3 macros (sans liquidity) * commit new spells * add pools_fees * ange poolbalancechanged to liquidityadded/removed * update sources and schemas * add aave token pricing * improve schemas and add aave tokens prices to macros * introduce incremental aave models * fix typo on gnosis trades * add new spells and schemas * remove potentially troublesome incremental filter from token_data * 1st round of review on macros * 1st round of review on dex subproject * update static tokens to include morpho * replace atokens for erc4626 tokens * update sources and schemas * update sources and schema * again * again * again * again2 * fix names * again * again * add pool labels to sources * fix macros * fix liquidity macro * fix protocol_fee * a * fix errors * again * fix new errors * fix copypasta * also fix on sources * fix references * fix pools_tokens_weights * update token_data for varbinary * fix erc4626 token prices * improvements for erc4626_token_prices * replicate changes on gnosis * again * fix labels * last fix erc4626 prices * fix * again * fix§ * again * fix group by * add gnosis seed * yay * fix bpt_supply_macro * fix liquidity and bpt_prices * final round (hopefully) * fix errors * fix errors * fix trades * fix incremental errors * update ethereum seed * last tweaks * gain * again * fix typo in schema name * add incremental filter to prices * add incremental filter to price join --------- Co-authored-by: Huang Geyang Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../balancer/balancer_bpt_prices_macro.sql | 275 +++++++++++++++ .../balancer/balancer_bpt_supply_macro.sql | 100 ++++++ .../balancer/balancer_liquidity_macro.sql | 330 ++++++++++++++++++ ...l_token_supply_changes_daily_agg_macro.sql | 45 ++- ...lancer_pool_token_supply_changes_macro.sql | 133 ++++++- .../balancer/balancer_protocol_fee_macro.sql | 204 ++++++++++- ..._token_balance_changes_daily_agg_macro.sql | 195 +++++++++++ .../balancer_token_balance_changes_macro.sql | 196 +++++++++++ .../balancer/balancer_transfers_bpt_macro.sql | 38 ++ .../_sector/labels/labels_addresses.sql | 2 +- .../_project/balancer_compatible_trades.sql | 75 ++++ .../_projects/balancer/labels/_schema.yml | 19 +- .../labels_balancer_v3_pools_ethereum.sql | 13 +- .../labels_balancer_v3_pools_gnosis.sql | 15 +- .../_projects/balancer/pools/_schema.yml | 56 ++- .../balancer/pools/balancer_pools_fees.sql | 4 +- .../balancer_v3_ethereum_pools_fees.sql | 28 ++ .../gnosis/balancer_v3_gnosis_pools_fees.sql | 28 ++ .../balancer/trades/ethereum/_schema.yml | 41 ++- .../ethereum/balancer_ethereum_trades.sql | 3 +- .../ethereum/balancer_v3_ethereum_trades.sql | 151 ++++++++ .../balancer/trades/gnosis/_schema.yml | 41 ++- .../trades/gnosis/balancer_gnosis_trades.sql | 3 +- .../gnosis/balancer_v3_gnosis_trades.sql | 151 ++++++++ .../dex/models/trades/ethereum/_schema.yml | 21 +- .../ethereum/dex_ethereum_base_trades.sql | 1 + .../balancer_v3_ethereum_base_trades.sql | 19 + .../dex/models/trades/gnosis/_schema.yml | 21 +- .../trades/gnosis/dex_gnosis_base_trades.sql | 1 + .../balancer_v3_gnosis_base_trades.sql | 19 + .../balancer_ethereum_base_trades_seed.csv | 1 + .../balancer_gnosis_base_trades_seed.csv | 1 + .../balancer_token_balance_changes.sql | 4 +- .../balancer_token_balance_changes_daily.sql | 4 +- .../balancer/balances/ethereum/_schema.yml | 64 +++- ...ncer_v3_ethereum_token_balance_changes.sql | 19 + ...3_ethereum_token_balance_changes_daily.sql | 19 + .../balancer/balances/gnosis/_schema.yml | 64 +++- ...lancer_v3_gnosis_token_balance_changes.sql | 19 + ..._v3_gnosis_token_balance_changes_daily.sql | 19 + .../balancer/bpt/balancer_bpt_prices.sql | 4 +- .../balancer/bpt/balancer_bpt_supply.sql | 4 +- .../bpt/balancer_bpt_supply_changes.sql | 4 +- .../bpt/balancer_bpt_supply_changes_daily.sql | 4 +- .../balancer/bpt/balancer_transfers_bpt.sql | 4 +- .../balancer/bpt/ethereum/_schema.yml | 132 ++++++- .../balancer_v3_ethereum_bpt_prices.sql | 21 ++ .../balancer_v3_ethereum_bpt_supply.sql | 21 ++ ...alancer_v3_ethereum_bpt_supply_changes.sql | 23 ++ ...r_v3_ethereum_bpt_supply_changes_daily.sql | 19 + .../balancer_v3_ethereum_transfers_bpt.sql | 22 ++ .../_project/balancer/bpt/gnosis/_schema.yml | 132 ++++++- .../gnosis/balancer_v3_gnosis_bpt_prices.sql | 21 ++ .../gnosis/balancer_v3_gnosis_bpt_supply.sql | 21 ++ .../balancer_v3_gnosis_bpt_supply_changes.sql | 23 ++ ...cer_v3_gnosis_bpt_supply_changes_daily.sql | 19 + .../balancer_v3_gnosis_transfers_bpt.sql | 22 ++ .../balancer/erc4626_tokens/_schema.yml | 65 ++++ .../balancer_v3_erc4626_token_mapping.sql | 32 ++ .../balancer_v3_erc4626_token_prices.sql | 34 ++ .../erc4626_tokens/ethereum/_schema.yml | 65 ++++ ...ncer_v3_ethereum_erc4626_token_mapping.sql | 69 ++++ ...ancer_v3_ethereum_erc4626_token_prices.sql | 65 ++++ .../erc4626_tokens/gnosis/_schema.yml | 65 ++++ ...lancer_v3_gnosis_erc4626_token_mapping.sql | 30 ++ ...alancer_v3_gnosis_erc4626_token_prices.sql | 65 ++++ .../balancer/liquidity/balancer_liquidity.sql | 2 + .../balancer/liquidity/ethereum/_schema.yml | 34 +- .../balancer_v3_ethereum_liquidity.sql | 22 ++ .../balancer/liquidity/gnosis/_schema.yml | 54 ++- .../gnosis/balancer_v3_gnosis_liquidity.sql | 21 ++ .../pools/balancer_pools_tokens_weights.sql | 16 +- .../balancer/pools/ethereum/_schema.yml | 23 +- ...balancer_ethereum_pools_tokens_weights.sql | 39 ++- ...ancer_v3_ethereum_pools_tokens_weights.sql | 61 ++++ .../balancer/pools/gnosis/_schema.yml | 23 +- .../balancer_gnosis_pools_tokens_weights.sql | 31 +- ...alancer_v3_gnosis_pools_tokens_weights.sql | 61 ++++ .../balancer/protocol_fee/_schema.yml | 5 +- .../protocol_fee/balancer_protocol_fee.sql | 9 +- .../protocol_fee/ethereum/_schema.yml | 40 ++- .../balancer_v3_ethereum_protocol_fee.sql | 20 ++ .../balancer/protocol_fee/gnosis/_schema.yml | 38 +- .../balancer_v3_gnosis_protocol_fee.sql | 20 ++ .../tokens/ethereum/tokens_ethereum_erc20.sql | 8 + .../tokens/gnosis/tokens_gnosis_erc20.sql | 4 + sources/_subprojects_outputs/dex/_sources.yml | 6 + .../spellbook/_sources.yml | 6 + .../aave/ethereum/aave_ethereum_sources.yml | 4 +- sources/aave/gnosis/aave_gnosis_sources.yml | 2 +- .../ethereum/balancer_ethereum_sources.yml | 4 +- .../gnosis/balancer_gnosis_sources.yml | 2 +- 92 files changed, 3865 insertions(+), 118 deletions(-) create mode 100644 dbt_subprojects/dex/models/_projects/balancer/pools/ethereum/balancer_v3_ethereum_pools_fees.sql create mode 100644 dbt_subprojects/dex/models/_projects/balancer/pools/gnosis/balancer_v3_gnosis_pools_fees.sql create mode 100644 dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql create mode 100644 dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql create mode 100644 dbt_subprojects/dex/models/trades/ethereum/platforms/balancer_v3_ethereum_base_trades.sql create mode 100644 dbt_subprojects/dex/models/trades/gnosis/platforms/balancer_v3_gnosis_base_trades.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes_daily.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes_daily.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_prices.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes_daily.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_transfers_bpt.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_prices.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes_daily.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_transfers_bpt.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_mapping.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/balancer_v3_ethereum_liquidity.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/balancer_v3_gnosis_liquidity.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_v3_ethereum_pools_tokens_weights.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_v3_gnosis_pools_tokens_weights.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/balancer_v3_ethereum_protocol_fee.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/balancer_v3_gnosis_protocol_fee.sql diff --git a/dbt_macros/shared/balancer/balancer_bpt_prices_macro.sql b/dbt_macros/shared/balancer/balancer_bpt_prices_macro.sql index 4e71e626d50..92fae3fa1b5 100644 --- a/dbt_macros/shared/balancer/balancer_bpt_prices_macro.sql +++ b/dbt_macros/shared/balancer/balancer_bpt_prices_macro.sql @@ -349,3 +349,278 @@ WITH pool_labels AS ( WHERE supply > 0 {% endmacro %} + + {# ######################################################################### #} + + {% macro + balancer_v3_compatible_bpt_prices_macro( + blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell + ) +%} + +WITH pool_labels AS ( + SELECT + address AS pool_id, + name AS pool_symbol, + pool_type + FROM {{ pool_labels_spell }} + WHERE blockchain = '{{blockchain}}' + ), + + token_data AS ( + SELECT + pool, + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + +-- liquidity formulation, with a few simplifications, compared to liquidity spell + + prices AS ( + SELECT + date_trunc('day', minute) AS day, + contract_address AS token, + decimals, + AVG(price) AS price + FROM {{ source('prices', 'usd') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3 + ), + + erc4626_prices AS( + SELECT + date_trunc('day', minute) AS day, + wrapped_token AS token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + DATE_TRUNC ('day', next_change) AS next_change + FROM {{ ref('balancer_v3_erc4626_token_prices') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3, 5 + ), + + swaps_changes AS ( + SELECT + day, + pool_id, + token, + SUM(COALESCE(delta, INT256 '0')) AS delta + FROM + ( + SELECT + date_trunc('day', evt_block_time) AS day, + pool AS pool_id, + tokenIn AS token, + CAST(amountIn AS INT256) AS delta + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + + UNION ALL + + SELECT + date_trunc('day', evt_block_time) AS day, + pool AS pool_id, + tokenOut AS token, + -CAST(amountOut AS INT256) AS delta + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + ) swaps + GROUP BY 1, 2, 3 + ), + + balance_changes AS( + SELECT + evt_block_time, + pool_id, + category, + deltas, + swapFeeAmountsRaw + FROM + ( + SELECT + evt_block_time, + pool AS pool_id, + 'add' AS category, + amountsAddedRaw AS deltas, + swapFeeAmountsRaw + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityAdded') }} + + UNION ALL + + SELECT + evt_block_time, + pool AS pool_id, + 'remove' AS category, + amountsRemovedRaw AS deltas, + swapFeeAmountsRaw + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityRemoved') }} + ) adds_and_removes + ), + + zipped_balance_changes AS ( + SELECT + date_trunc('day', evt_block_time) AS day, + pool_id, + t.tokens, + CASE WHEN b.category = 'add' + THEN d.deltas + WHEN b.category = 'remove' + THEN -d.deltas + END AS deltas, + p.swapFeeAmountsRaw + FROM balance_changes b + JOIN token_data td ON b.pool_id = td.pool + CROSS JOIN UNNEST (td.tokens) WITH ORDINALITY as t(tokens,i) + CROSS JOIN UNNEST (b.deltas) WITH ORDINALITY as d(deltas,i) + CROSS JOIN UNNEST (b.swapFeeAmountsRaw) WITH ORDINALITY as p(swapFeeAmountsRaw,i) + WHERE t.i = d.i + AND d.i = p.i + ORDER BY 1,2,3 + ), + + balances_changes AS ( + SELECT + day, + pool_id, + tokens AS token, + deltas - CAST(swapFeeAmountsRaw as int256) AS delta + FROM zipped_balance_changes + ORDER BY 1, 2, 3 + ), + + daily_delta_balance AS ( + SELECT + day, + pool_id, + token, + SUM(COALESCE(amount, INT256 '0')) AS amount + FROM + ( + SELECT + day, + pool_id, + token, + SUM(COALESCE(delta, INT256 '0')) AS amount + FROM balances_changes + GROUP BY 1, 2, 3 + + UNION ALL + + SELECT + day, + pool_id, + token, + delta AS amount + FROM + swaps_changes + ) balance + GROUP BY 1, 2, 3 + ), + + + cumulative_balance AS ( + SELECT + DAY, + pool_id, + token, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token, pool_id ORDER BY DAY) AS day_of_next_change, + SUM(amount) OVER (PARTITION BY pool_id, token ORDER BY DAY ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_amount + FROM daily_delta_balance + ), + + calendar AS ( + SELECT date_sequence AS day + FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence) + ), + + cumulative_usd_balance AS ( + SELECT + c.day, + '{{blockchain}}' as blockchain, + b.pool_id, + b.token, + cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p4.decimals)) * COALESCE(p1.price, p4.price, 0) AS protocol_liquidity_usd + FROM calendar c + LEFT JOIN cumulative_balance b ON b.day <= c.day + AND c.day < b.day_of_next_change + LEFT JOIN {{ source('tokens', 'erc20') }} t ON t.contract_address = b.token + AND blockchain = '{{blockchain}}' + LEFT JOIN prices p1 ON p1.day = b.day + AND p1.token = b.token + LEFT JOIN erc4626_prices p4 ON p4.day <= c.day + AND c.day < p4.next_change + AND p4.token = b.token + WHERE b.token != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) + ), + + weighted_pool_liquidity_estimates AS ( + SELECT + b.day, + b.pool_id, + q.name, + pool_type, + ROW_NUMBER() OVER (partition by b.day, b.pool_id ORDER BY SUM(b.protocol_liquidity_usd) ASC) AS pricing_count, --to avoid double count in pools with multiple pricing assets + SUM(b.protocol_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS protocol_liquidity + FROM cumulative_usd_balance b + LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id + AND b.token = w.token_address + AND b.protocol_liquidity_usd > 0 + LEFT JOIN {{ source('balancer','token_whitelist') }} q ON b.token = q.address + AND b.blockchain = q.chain + LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) + WHERE q.name IS NOT NULL + AND p.pool_type IN ('weighted') -- filters for weighted pools with pricing assets + AND w.blockchain = '{{blockchain}}' + AND w.version = '{{version}}' + GROUP BY 1, 2, 3, 4 + ), + + weighted_pool_liquidity_estimates_2 AS( + SELECT e.day, + e.pool_id, + SUM(e.protocol_liquidity) / MAX(e.pricing_count) AS protocol_liquidity + FROM weighted_pool_liquidity_estimates e + GROUP BY 1,2 + ), + + tvl AS( + SELECT + c.day, + BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) AS pool_address, + '{{version}}' AS version, + '{{blockchain}}' AS blockchain, + SUM(COALESCE(b.protocol_liquidity * w.normalized_weight, c.protocol_liquidity_usd)) AS liquidity + FROM cumulative_usd_balance c + FULL OUTER JOIN weighted_pool_liquidity_estimates_2 b ON c.day = b.day + AND c.pool_id = b.pool_id + LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id + AND w.blockchain = '{{blockchain}}' + AND w.version = '{{version}}' + AND w.token_address = c.token + LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) + GROUP BY 1, 2, 3, 4 + ) + + SELECT + l.day, + l.blockchain, + l.version, + 18 AS decimals, + l.pool_address AS contract_address, + pl.pool_type, + l.liquidity / s.supply AS bpt_price + FROM tvl l + LEFT JOIN {{ ref(base_spells_namespace + '_bpt_supply') }} s ON l.pool_address = s.token_address + AND l.blockchain = s.blockchain + AND l.version = s.version + AND l.day = s.day + LEFT JOIN pool_labels pl ON pl.pool_id = l.pool_address + WHERE supply > 0 + + {% endmacro %} diff --git a/dbt_macros/shared/balancer/balancer_bpt_supply_macro.sql b/dbt_macros/shared/balancer/balancer_bpt_supply_macro.sql index 9d128983005..d9a5f75abdc 100644 --- a/dbt_macros/shared/balancer/balancer_bpt_supply_macro.sql +++ b/dbt_macros/shared/balancer/balancer_bpt_supply_macro.sql @@ -138,3 +138,103 @@ WITH pool_labels AS ( HAVING SUM(b.supply - COALESCE(preminted_bpts, 0) + COALESCE(adelta, 0)) >= 0 --simple filter to remove outliers {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_bpt_supply_macro( + blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell + ) +%} + + WITH pool_labels AS ( + SELECT * FROM ( + SELECT + address, + name, + pool_type, + ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num + FROM {{ pool_labels_spell }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3) + WHERE num = 1 + ), + + -- Extract mints and burns from transfers + transfers AS ( + SELECT + block_date AS day, + contract_address AS token, + COALESCE(SUM(CASE WHEN t."from" = 0x0000000000000000000000000000000000000000 THEN value / POWER(10, 18) ELSE 0 END), 0) AS mints, + COALESCE(SUM(CASE WHEN t.to = 0x0000000000000000000000000000000000000000 THEN value / POWER(10, 18) ELSE 0 END), 0) AS burns + FROM {{ ref(base_spells_namespace + '_transfers_bpt') }} t + WHERE blockchain = '{{blockchain}}' + AND version = '{{version}}' + GROUP BY 1, 2 + ), + + -- Calculate token balances over time + balances AS ( + SELECT + day, + token, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change, + SUM(COALESCE(mints, 0) - COALESCE(burns, 0)) OVER (PARTITION BY token ORDER BY DAY ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS supply + FROM transfers + ), + + -- Calculating Joins(mint) and Exits(burn) via Swap + joins AS ( + SELECT + DATE_TRUNC('day', evt_block_time) AS block_date, + tokenOut, + pool_type, + SUM(amountOut / POWER(10, 18)) AS ajoins + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + LEFT JOIN pool_labels ON pool = address + WHERE tokenOut = pool + GROUP BY 1, 2, 3 + ), + + exits AS ( + SELECT + DATE_TRUNC('day', evt_block_time) AS block_date, + tokenIn, + pool_type, + SUM(amountIn / POWER(10, 18)) AS aexits + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + LEFT JOIN pool_labels ON pool = address + WHERE tokenIn = pool + GROUP BY 1, 2, 3 + ), + + joins_and_exits AS ( + SELECT + j.block_date, + j.tokenOut AS bpt, + SUM(COALESCE(ajoins, 0) - COALESCE(aexits, 0)) OVER (PARTITION BY j.tokenOut ORDER BY j.block_date ASC) AS adelta + FROM joins j + FULL OUTER JOIN exits e ON j.block_date = e.block_date AND e.tokenIn = j.tokenOut + ), + + calendar AS ( + SELECT + date_sequence AS day + FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence) + ) + + SELECT + c.day, + l.pool_type, + '{{version}}' as version, + '{{blockchain}}' as blockchain, + b.token AS token_address, + COALESCE(SUM(b.supply + COALESCE(adelta, 0)),0) AS supply + FROM calendar c + LEFT JOIN balances b ON b.day <= c.day AND c.day < b.day_of_next_change + LEFT JOIN joins_and_exits j ON c.day = j.block_date AND b.token = j.bpt + LEFT JOIN pool_labels l ON b.token = l.address + GROUP BY 1, 2, 3, 4, 5 + HAVING SUM(b.supply + COALESCE(adelta, 0)) >= 0 --simple filter to remove outliers + +{% endmacro %} \ No newline at end of file diff --git a/dbt_macros/shared/balancer/balancer_liquidity_macro.sql b/dbt_macros/shared/balancer/balancer_liquidity_macro.sql index f4d4704028c..0f893a0f561 100644 --- a/dbt_macros/shared/balancer/balancer_liquidity_macro.sql +++ b/dbt_macros/shared/balancer/balancer_liquidity_macro.sql @@ -288,3 +288,333 @@ WITH pool_labels AS ( LEFT JOIN eth_prices e ON e.day = c.day LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_liquidity_macro( + blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell + ) +%} + +WITH pool_labels AS ( + SELECT + address AS pool_id, + name AS pool_symbol, + pool_type + FROM {{ pool_labels_spell }} + WHERE blockchain = '{{blockchain}}' + ), + + token_data AS ( + SELECT + pool, + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + + prices AS ( + SELECT + date_trunc('day', minute) AS day, + contract_address AS token, + decimals, + APPROX_PERCENTILE(price, 0.5) AS price + FROM {{ source('prices', 'usd') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3 + ), + + dex_prices_1 AS ( + SELECT + date_trunc('day', HOUR) AS DAY, + contract_address AS token, + approx_percentile(median_price, 0.5) AS price, + sum(sample_size) AS sample_size + FROM {{ source('dex', 'prices') }} + WHERE + ('{{blockchain}}' != 'fantom' OR + ('{{blockchain}}' = 'fantom' AND contract_address NOT IN (0xde1e704dae0b4051e80dabb26ab6ad6c12262da0, 0x5ddb92a5340fd0ead3987d3661afcd6104c3b757))) --broken price feeds + AND blockchain = '{{blockchain}}' + GROUP BY 1, 2 + HAVING sum(sample_size) > 3 + ), + + dex_prices_2 AS( + SELECT + day, + token, + price, + lag(price) OVER(PARTITION BY token ORDER BY day) AS previous_price + FROM dex_prices_1 + ), + + dex_prices AS ( + SELECT + day, + token, + price, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change + FROM dex_prices_2 + WHERE (price < previous_price * 1e4 AND price > previous_price / 1e4) + ), + + bpt_prices AS( + SELECT DISTINCT + day, + contract_address AS token, + decimals, + bpt_price + FROM {{ ref(base_spells_namespace + '_bpt_prices') }} + WHERE blockchain = '{{blockchain}}' + AND version = '{{version}}' + ), + + eth_prices AS ( + SELECT + DATE_TRUNC('day', minute) as day, + AVG(price) as eth_price + FROM {{ source('prices', 'usd') }} + WHERE blockchain = 'ethereum' + AND contract_address = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + GROUP BY 1 + ), + + erc4626_prices AS( + SELECT + date_trunc('day', minute) AS day, + wrapped_token AS token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + DATE_TRUNC ('day', next_change) AS next_change + FROM {{ ref('balancer_v3_erc4626_token_prices') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3, 5 + ), + + swaps_changes AS ( + SELECT + day, + pool_id, + token, + SUM(COALESCE(delta, INT256 '0')) AS delta + FROM + ( + SELECT + date_trunc('day', evt_block_time) AS day, + pool AS pool_id, + tokenIn AS token, + CAST(amountIn AS INT256) AS delta + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + + UNION ALL + + SELECT + date_trunc('day', evt_block_time) AS day, + pool AS pool_id, + tokenOut AS token, + -CAST(amountOut AS INT256) AS delta + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + ) swaps + GROUP BY 1, 2, 3 + ), + + balance_changes AS( + SELECT + evt_block_time, + pool_id, + category, + deltas, + swapFeeAmountsRaw + FROM + ( + SELECT + evt_block_time, + pool AS pool_id, + 'add' AS category, + amountsAddedRaw AS deltas, + swapFeeAmountsRaw + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityAdded') }} + + UNION ALL + + SELECT + evt_block_time, + pool AS pool_id, + 'remove' AS category, + amountsRemovedRaw AS deltas, + swapFeeAmountsRaw + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityRemoved') }} + ) adds_and_removes + ), + + zipped_balance_changes AS ( + SELECT + date_trunc('day', evt_block_time) AS day, + pool_id, + t.tokens, + CASE WHEN b.category = 'add' + THEN d.deltas + WHEN b.category = 'remove' + THEN -d.deltas + END AS deltas, + p.swapFeeAmountsRaw + FROM balance_changes b + JOIN token_data td ON b.pool_id = td.pool + CROSS JOIN UNNEST (td.tokens) WITH ORDINALITY as t(tokens,i) + CROSS JOIN UNNEST (b.deltas) WITH ORDINALITY as d(deltas,i) + CROSS JOIN UNNEST (b.swapFeeAmountsRaw) WITH ORDINALITY as p(swapFeeAmountsRaw,i) + WHERE t.i = d.i + AND d.i = p.i + ORDER BY 1,2,3 + ), + + balances_changes AS ( + SELECT + day, + pool_id, + tokens AS token, + deltas - CAST(swapFeeAmountsRaw as int256) AS delta + FROM zipped_balance_changes + ORDER BY 1, 2, 3 + ), + + daily_delta_balance AS ( + SELECT + day, + pool_id, + token, + SUM(COALESCE(amount, INT256 '0')) AS amount + FROM + ( + SELECT + day, + pool_id, + token, + SUM(COALESCE(delta, INT256 '0')) AS amount + FROM balances_changes + GROUP BY 1, 2, 3 + + UNION ALL + + SELECT + day, + pool_id, + token, + delta AS amount + FROM + swaps_changes + ) balance + GROUP BY 1, 2, 3 + ), + + cumulative_balance AS ( + SELECT + DAY, + pool_id, + token, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token, pool_id ORDER BY DAY) AS day_of_next_change, + SUM(amount) OVER (PARTITION BY pool_id, token ORDER BY DAY ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_amount + FROM daily_delta_balance + ), + + calendar AS ( + SELECT date_sequence AS day + FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence) + ), + + cumulative_usd_balance AS ( + SELECT + c.day, + '{{blockchain}}' as blockchain, + b.pool_id, + b.token, + symbol AS token_symbol, + cumulative_amount as token_balance_raw, + cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p3.decimals, p4.decimals)) AS token_balance, + cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p3.decimals, p4.decimals)) * COALESCE(p1.price, p2.price, p4.price, 0) AS protocol_liquidity_usd, + cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p3.decimals, p4.decimals)) * COALESCE(p1.price, p2.price, p3.bpt_price, p4.price, 0) AS pool_liquidity_usd + FROM calendar c + LEFT JOIN cumulative_balance b ON b.day <= c.day + AND c.day < b.day_of_next_change + LEFT JOIN {{ source('tokens', 'erc20') }} t ON t.contract_address = b.token + AND blockchain = '{{blockchain}}' + LEFT JOIN prices p1 ON p1.day = b.day + AND p1.token = b.token + LEFT JOIN dex_prices p2 ON p2.day <= c.day + AND c.day < p2.day_of_next_change + AND p2.token = b.token + LEFT JOIN bpt_prices p3 ON p3.day = b.day + AND p3.token = b.token + LEFT JOIN erc4626_prices p4 ON p4.day <= c.day + AND c.day < p4.next_change + AND p4.token = b.token + WHERE b.token != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) + ), + + weighted_pool_liquidity_estimates AS ( + SELECT + b.day, + b.pool_id, + q.name, + pool_type, + ROW_NUMBER() OVER (partition by b.day, b.pool_id ORDER BY SUM(b.pool_liquidity_usd) ASC) AS pricing_count, --to avoid double count in pools with multiple pricing assets + SUM(b.protocol_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS protocol_liquidity, + SUM(b.pool_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS pool_liquidity + FROM cumulative_usd_balance b + LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id + AND b.token = w.token_address + AND b.pool_liquidity_usd > 0 + LEFT JOIN {{ source('balancer','token_whitelist') }} q ON b.token = q.address + AND b.blockchain = q.chain + LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) + WHERE q.name IS NOT NULL + AND p.pool_type IN ('weighted') -- filters for weighted pools with pricing assets + AND w.blockchain = '{{blockchain}}' + AND w.version = '{{version}}' + GROUP BY 1, 2, 3, 4 + ), + + weighted_pool_liquidity_estimates_2 AS( + SELECT e.day, + e.pool_id, + SUM(e.pool_liquidity) / MAX(e.pricing_count) AS pool_liquidity, + SUM(e.protocol_liquidity) / MAX(e.pricing_count) AS protocol_liquidity + FROM weighted_pool_liquidity_estimates e + GROUP BY 1,2 + ) + + SELECT + c.day, + c.pool_id, + BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) AS pool_address, + p.pool_symbol, + '{{version}}' AS version, + '{{blockchain}}' AS blockchain, + p.pool_type, + c.token AS token_address, + c.token_symbol, + c.token_balance_raw, + c.token_balance, + COALESCE(b.protocol_liquidity * w.normalized_weight, c.protocol_liquidity_usd) AS protocol_liquidity_usd, + COALESCE(b.protocol_liquidity * w.normalized_weight, c.protocol_liquidity_usd)/e.eth_price AS protocol_liquidity_eth, + COALESCE(b.pool_liquidity * w.normalized_weight, c.pool_liquidity_usd) AS pool_liquidity_usd, + COALESCE(b.pool_liquidity * w.normalized_weight, c.pool_liquidity_usd)/e.eth_price AS pool_liquidity_eth + FROM cumulative_usd_balance c + FULL OUTER JOIN weighted_pool_liquidity_estimates_2 b ON c.day = b.day + AND c.pool_id = b.pool_id + LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id + AND w.blockchain = '{{blockchain}}' + AND w.version = '{{version}}' + AND w.token_address = c.token + LEFT JOIN eth_prices e ON e.day = c.day + LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) + + {% endmacro %} \ No newline at end of file diff --git a/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_daily_agg_macro.sql b/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_daily_agg_macro.sql index 510ebbe007e..35f850e3397 100644 --- a/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_daily_agg_macro.sql +++ b/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_daily_agg_macro.sql @@ -35,4 +35,47 @@ WITH LEFT JOIN daily_balance b ON b.block_date = c.day WHERE b.token_address IS NOT NULL AND b.pool_type IS NOT NULL - {% endmacro %} \ No newline at end of file + {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_bpt_supply_changes_daily_agg_macro( + blockchain, version, project_decoded_as, base_spells_namespace + ) +%} +WITH + daily_balance AS ( + SELECT + block_date, + blockchain, + pool_type, + pool_symbol, + token_address, + LEAD(block_date, 1, NOW()) OVER (PARTITION BY token_address ORDER BY block_date) AS day_of_next_change, + SUM(delta_amount) AS daily_amount + FROM {{ ref(base_spells_namespace + '_bpt_supply_changes') }} + WHERE blockchain = '{{blockchain}}' + AND version = '{{version}}' + GROUP BY 1, 2, 3, 4, 5 + ), + + calendar AS ( + SELECT date_sequence AS day + FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence) + ) + + SELECT + c.day AS block_date, + '{{blockchain}}' as blockchain, + '{{version}}' AS version, + b.pool_type, + b.pool_symbol, + b.token_address, + b.daily_amount AS daily_delta + FROM calendar c + LEFT JOIN daily_balance b ON b.block_date = c.day + WHERE b.token_address IS NOT NULL + AND b.pool_type IS NOT NULL + + {% endmacro %} \ No newline at end of file diff --git a/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_macro.sql b/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_macro.sql index 71d9b78053e..90df02e7851 100644 --- a/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_macro.sql +++ b/dbt_macros/shared/balancer/balancer_pool_token_supply_changes_macro.sql @@ -63,7 +63,7 @@ WITH pool_labels AS ( LEFT JOIN pool_labels l ON BYTEARRAY_SUBSTRING(s.poolId, 1, 20) = l.address WHERE tokenOut = BYTEARRAY_SUBSTRING(s.poolId, 1, 20) {% if is_incremental() %} - AND {{ incremental_predicate('s. evt_block_time') }} + AND {{ incremental_predicate('s.evt_block_time') }} {% endif %} ), @@ -86,7 +86,7 @@ WITH pool_labels AS ( LEFT JOIN pool_labels l ON BYTEARRAY_SUBSTRING(s.poolId, 1, 20) = l.address WHERE tokenIn = BYTEARRAY_SUBSTRING(s.poolId, 1, 20) {% if is_incremental() %} - AND {{ incremental_predicate('s. evt_block_time') }} + AND {{ incremental_predicate('s.evt_block_time') }} {% endif %} ) @@ -127,4 +127,131 @@ WITH pool_labels AS ( WHERE {{ incremental_predicate('evt_block_time') }} {% endif %} - {% endmacro %} \ No newline at end of file + {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_bpt_supply_changes_macro( + blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell + ) +%} +WITH pool_labels AS ( + SELECT + address, + name, + pool_type + FROM {{ pool_labels_spell }} + WHERE blockchain = '{{blockchain}}' + ), + + -- Extract mints and burns from transfers + transfers AS ( + SELECT + t.evt_block_time, + t.evt_block_number, + t.evt_tx_hash, + t.evt_index, + t.contract_address AS token, + CASE + WHEN t."from" = 0x0000000000000000000000000000000000000000 + THEN 'mint' + WHEN t.to = 0x0000000000000000000000000000000000000000 + THEN 'burn' + END AS label, + l.pool_type, + l.name, + CASE + WHEN t."from" = 0x0000000000000000000000000000000000000000 + THEN value + WHEN t.to = 0x0000000000000000000000000000000000000000 + THEN - value + ELSE 0 + END AS amount + FROM {{ ref(base_spells_namespace + '_transfers_bpt') }} t + LEFT JOIN pool_labels l ON t.contract_address = l.address + WHERE t.blockchain = '{{blockchain}}' + AND t.version = '{{version}}' + {% if is_incremental() %} + AND {{ incremental_predicate('t.evt_block_time') }} + {% endif %} + ), + + -- Calculating Joins(mint) and Exits(burn) via Swap + joins AS ( + SELECT + s.evt_block_time, + s.evt_block_number, + s.evt_tx_hash, + s.evt_index, + s.tokenOut AS token, + 'join' AS label, + l.pool_type, + l.name, + s.amountOut AS amount + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} s + LEFT JOIN pool_labels l ON pool = l.address + WHERE tokenOut = pool + {% if is_incremental() %} + AND {{ incremental_predicate('s.evt_block_time') }} + {% endif %} + + ), + + exits AS ( + SELECT + s.evt_block_time, + s.evt_block_number, + s.evt_tx_hash, + s.evt_index, + s.tokenIn AS token, + 'exit' AS label, + l.pool_type, + l.name, + - s.amountIn AS amount + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} s + LEFT JOIN pool_labels l ON pool = l.address + WHERE tokenIn = pool + {% if is_incremental() %} + AND {{ incremental_predicate('s.evt_block_time') }} + {% endif %} + ) + + SELECT + date_trunc('day', evt_block_time) AS block_date, + evt_block_time, + evt_block_number, + '{{blockchain}}' AS blockchain, + evt_tx_hash, + evt_index, + pool_type, + name AS pool_symbol, + '{{version}}' AS version, + label, + token AS token_address, + amount AS delta_amount_raw, + amount / POWER (10, 18) AS delta_amount --18 decimals standard for BPTs + FROM + ( + SELECT + * + FROM joins + + UNION ALL + + SELECT + * + FROM exits + + UNION ALL + + SELECT + * + FROM transfers + WHERE label IS NOT NULL + ) + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + + {% endmacro %} \ No newline at end of file diff --git a/dbt_macros/shared/balancer/balancer_protocol_fee_macro.sql b/dbt_macros/shared/balancer/balancer_protocol_fee_macro.sql index 725c2cb76b8..eb5f72e13fb 100644 --- a/dbt_macros/shared/balancer/balancer_protocol_fee_macro.sql +++ b/dbt_macros/shared/balancer/balancer_protocol_fee_macro.sql @@ -120,7 +120,7 @@ WITH pool_labels AS ( d.token_address, t.symbol AS token_symbol, SUM(d.protocol_fee_amount_raw) AS token_amount_raw, - SUM(d.protocol_fee_amount_raw / power(10, COALESCE(t.decimals,p1.decimals, p3.decimals))) AS token_amount, + SUM(d.protocol_fee_amount_raw / POWER(10, COALESCE(t.decimals,p1.decimals, p3.decimals))) AS token_amount, SUM(COALESCE(p1.price, p2.price, p3.price) * protocol_fee_amount_raw / POWER(10, COALESCE(t.decimals,p1.decimals, p3.decimals))) AS protocol_fee_collected_usd FROM daily_protocol_fee_collected d LEFT JOIN prices p1 @@ -160,6 +160,7 @@ WITH pool_labels AS ( '{{version}}' as version, '{{blockchain}}' as blockchain, l.pool_type, + 'v2' AS fee_type, f.token_address, f.token_symbol, SUM(f.token_amount_raw) as token_amount_raw, @@ -172,6 +173,205 @@ WITH pool_labels AS ( ON r.day = f.day LEFT JOIN pool_labels l ON BYTEARRAY_SUBSTRING(f.pool_id,1,20) = l.address - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 13 + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14 {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_protocol_fee_macro( + blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell + ) +%} + +WITH pool_labels AS ( + SELECT * FROM ( + SELECT + address, + name, + pool_type, + ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num + FROM {{ pool_labels_spell }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3) + WHERE num = 1 + ), + + prices AS ( + SELECT + date_trunc('day', minute) AS day, + contract_address AS token, + decimals, + AVG(price) AS price + FROM {{ source('prices', 'usd') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3 + + ), + + dex_prices_1 AS ( + SELECT + date_trunc('day', hour) AS DAY, + contract_address AS token, + approx_percentile(median_price, 0.5) AS price, + sum(sample_size) AS sample_size + FROM {{ source('dex', 'prices') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2 + HAVING sum(sample_size) > 3 + ), + + dex_prices_2 AS( + SELECT + day, + token, + price, + lag(price) OVER(PARTITION BY token ORDER BY day) AS previous_price + FROM dex_prices_1 + ), + + dex_prices AS ( + SELECT + day, + token, + price, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change + FROM dex_prices_2 + WHERE (price < previous_price * 1e4 AND price > previous_price / 1e4) + ), + + bpt_prices_1 AS ( --special calculation for this spell, in order to achieve completeness without relying on prices.usd + SELECT + l.day, + s.token_address AS token, + 18 AS decimals, + SUM(protocol_liquidity_usd / supply) AS price + FROM {{ ref(base_spells_namespace + '_liquidity') }} l + LEFT JOIN {{ ref(base_spells_namespace + '_bpt_supply') }} s ON s.token_address = l.pool_address + AND l.blockchain = s.blockchain AND s.day = l.day AND s.supply > 1e-4 + WHERE l.blockchain = '{{blockchain}}' + AND l.version = '{{version}}' + GROUP BY 1, 2, 3 + ), + + bpt_prices AS ( + SELECT + day, + token, + decimals, + price, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change + FROM bpt_prices_1 + ), + + erc4626_prices AS( + SELECT + date_trunc('day', minute) AS day, + wrapped_token AS token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + DATE_TRUNC ('day', next_change) AS next_change + FROM {{ ref('balancer_v3_erc4626_token_prices') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3, 5 + ), + + daily_protocol_fee_collected AS ( + SELECT + d.day, + d.pool_id, + d.token_address, + d.fee_type, + SUM(d.protocol_fee_amount_raw) AS token_amount_raw + FROM ( + SELECT + date_trunc('day', evt_block_time) AS day, + pool AS pool_id, + token AS token_address, + 'swap_fee' AS fee_type, + SUM(amount) AS protocol_fee_amount_raw + FROM {{ source(project_decoded_as + '_' + blockchain, 'ProtocolFeeController_evt_ProtocolSwapFeeCollected') }} + GROUP BY 1, 2, 3, 4 + + UNION ALL + + SELECT + date_trunc('day', evt_block_time) AS day, + pool AS pool_id, + token AS token_address, + 'yield_fee' AS fee_type, + SUM(amount) AS protocol_fee_amount_raw + FROM {{ source(project_decoded_as + '_' + blockchain, 'ProtocolFeeController_evt_ProtocolYieldFeeCollected') }} + GROUP BY 1, 2, 3, 4 + ) d + GROUP BY 1, 2, 3, 4 + ), + + decorated_protocol_fee AS ( + SELECT + d.day, + d.pool_id, + d.token_address, + t.symbol AS token_symbol, + d.fee_type, + SUM(d.token_amount_raw) AS token_amount_raw, + SUM(d.token_amount_raw / POWER(10, COALESCE(t.decimals,p1.decimals, p3.decimals, p4.decimals))) AS token_amount, + SUM(COALESCE(p1.price, p2.price, p3.price, p4.price) * token_amount_raw / POWER(10, COALESCE(t.decimals,p1.decimals, p3.decimals, p4.decimals))) AS protocol_fee_collected_usd + FROM daily_protocol_fee_collected d + LEFT JOIN prices p1 + ON p1.token = d.token_address + AND p1.day = d.day + LEFT JOIN dex_prices p2 + ON p2.token = d.token_address + AND p2.day = d.day + LEFT JOIN bpt_prices p3 + ON p3.token = d.token_address + AND p3.day <= d.day + AND d.day < p3.day_of_next_change + LEFT JOIN erc4626_prices p4 ON p4.day <= d.day + AND d.day < p4.next_change + AND p4.token = d.token_address + LEFT JOIN {{ source('tokens', 'erc20') }} t + ON t.contract_address = d.token_address + AND t.blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3, 4, 5 + ), + + revenue_share as( + SELECT + day, + CASE + WHEN day < DATE '2022-07-03' THEN 0.25 -- veBAL release + WHEN day >= DATE '2022-07-03' AND day < DATE '2023-01-23' THEN 0.25 -- BIP 19 + WHEN day >= DATE '2023-01-23' AND day < DATE '2023-07-24' THEN 0.35 -- BIP 161 + WHEN day >= DATE '2023-07-24' THEN 0.175 -- BIP 371 + END AS treasury_share + FROM UNNEST(SEQUENCE(DATE '2022-03-01', CURRENT_DATE, INTERVAL '1' DAY)) AS date(day) + ) + + + SELECT + f.day, + f.pool_id, + BYTEARRAY_SUBSTRING(f.pool_id,1,20) as pool_address, + l.name AS pool_symbol, + '{{version}}' as version, + '{{blockchain}}' as blockchain, + l.pool_type, + f.fee_type, + f.token_address, + f.token_symbol, + SUM(f.token_amount_raw) as token_amount_raw, + SUM(f.token_amount) as token_amount, + SUM(f.protocol_fee_collected_usd) as protocol_fee_collected_usd, + r.treasury_share, + SUM(f.protocol_fee_collected_usd) * r.treasury_share as treasury_revenue_usd + FROM decorated_protocol_fee f + INNER JOIN revenue_share r + ON r.day = f.day + LEFT JOIN pool_labels l + ON BYTEARRAY_SUBSTRING(f.pool_id,1,20) = l.address + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 14 + +{% endmacro %} \ No newline at end of file diff --git a/dbt_macros/shared/balancer/balancer_token_balance_changes_daily_agg_macro.sql b/dbt_macros/shared/balancer/balancer_token_balance_changes_daily_agg_macro.sql index 8930036313b..2d177e5d182 100644 --- a/dbt_macros/shared/balancer/balancer_token_balance_changes_daily_agg_macro.sql +++ b/dbt_macros/shared/balancer/balancer_token_balance_changes_daily_agg_macro.sql @@ -182,3 +182,198 @@ WITH AND w.token_address = c.token_address LEFT JOIN eth_prices e ON e.day = c.block_date {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_token_balance_changes_daily_agg_macro( + blockchain, version, project_decoded_as, base_spells_namespace + ) +%} +WITH + prices AS ( + SELECT + date_trunc('day', minute) AS day, + contract_address AS token, + decimals, + AVG(price) AS price + FROM {{ source('prices', 'usd') }} + WHERE blockchain = '{{blockchain}}' + {% if is_incremental() %} + WHERE {{ incremental_predicate('minute') }} + {% endif %} + GROUP BY 1, 2, 3 + ), + + dex_prices_1 AS ( + SELECT + date_trunc('day', HOUR) AS DAY, + contract_address AS token, + approx_percentile(median_price, 0.5) AS price, + sum(sample_size) AS sample_size + FROM {{ source('dex', 'prices') }} + {% if is_incremental() %} + WHERE {{ incremental_predicate('hour') }} + {% endif %} + GROUP BY 1, 2 + HAVING sum(sample_size) > 3 + ), + + dex_prices_2 AS( + SELECT + day, + token, + price, + lag(price) OVER(PARTITION BY token ORDER BY day) AS previous_price + FROM dex_prices_1 + ), + + dex_prices AS ( + SELECT + day, + token, + price, + LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change + FROM dex_prices_2 + WHERE (price < previous_price * 1e4 AND price > previous_price / 1e4) + ), + + bpt_prices AS( + SELECT DISTINCT + day, + contract_address AS token, + decimals, + bpt_price + FROM {{ ref(base_spells_namespace + '_bpt_prices') }} + WHERE blockchain = '{{blockchain}}' + {% if is_incremental() %} + AND {{ incremental_predicate('day') }} + {% endif %} + AND version = '{{version}}' + ), + + eth_prices AS ( + SELECT + DATE_TRUNC('day', minute) as day, + AVG(price) as eth_price + FROM {{ source('prices', 'usd') }} + WHERE symbol = 'ETH' + {% if is_incremental() %} + WHERE {{ incremental_predicate('minute') }} + {% endif %} + GROUP BY 1 + ), + + erc4626_prices AS( + SELECT + date_trunc('day', minute) AS day, + wrapped_token AS token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + DATE_TRUNC ('day', next_change) AS next_change + FROM {{ ref('balancer_v3_erc4626_token_prices') }} + WHERE blockchain = '{{blockchain}}' + GROUP BY 1, 2, 3, 5 + ), + + daily_balance AS ( + SELECT + block_date, + pool_id, + pool_symbol, + pool_type, + token_address, + token_symbol, + LEAD(block_date, 1, NOW()) OVER (PARTITION BY token_address, pool_id ORDER BY block_date) AS day_of_next_change, + SUM(delta_amount) AS daily_amount + FROM {{ ref(base_spells_namespace + '_token_balance_changes') }} + WHERE blockchain = '{{blockchain}}' + AND version = '{{version}}' + GROUP BY 1, 2, 3, 4, 5, 6 + ), + + calendar AS ( + SELECT date_sequence AS day + FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence) + ), + + daily_usd_balance AS ( + SELECT + c.day AS block_date, + '{{blockchain}}"' as blockchain, + b.pool_id, + b.pool_symbol, + b.pool_type, + b.token_address, + b.token_symbol, + daily_amount, + daily_amount * COALESCE(p1.price, p2.price, p3.bpt_price, p4.price, 0) AS daily_amount_usd + FROM calendar c + LEFT JOIN daily_balance b ON b.block_date <= c.day + AND c.day < b.day_of_next_change + LEFT JOIN prices p1 ON p1.day = b.block_date + AND p1.token = b.token_address + LEFT JOIN dex_prices p2 ON p2.day <= c.day + AND c.day < p2.day_of_next_change + AND p2.token = b.token_address + LEFT JOIN bpt_prices p3 ON p3.day = b.block_date + AND p3.token = b.token_address + LEFT JOIN erc4626_prices p4 ON p4.day <= c.day + AND c.day < p4.next_change + AND p4.token = b.token_address + WHERE b.token_address != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) + ), + + weighted_pool_amount_estimates AS ( + SELECT + b.block_date, + b.pool_id, + q.name, + pool_type, + ROW_NUMBER() OVER (PARTITION BY b.block_date, b.pool_id ORDER BY SUM(b.daily_amount_usd) ASC) AS pricing_count, --to avoid double count in pools with multiple pricing assets + SUM(b.daily_amount_usd) / COALESCE(SUM(w.normalized_weight), 1) AS weighted_daily_amount_usd + FROM daily_usd_balance b + LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id + AND b.token_address = w.token_address + AND b.daily_amount_usd > 0 + LEFT JOIN {{ source('balancer','token_whitelist') }} q ON b.token_address = q.address + AND b.blockchain = q.chain + WHERE q.name IS NOT NULL + AND b.pool_type = 'weighted' -- filters for weighted pools with pricing assets + AND w.blockchain = '{{blockchain}}' + AND w.version = '2' + GROUP BY 1, 2, 3, 4 + ), + + weighted_pool_amount_estimates_2 AS( + SELECT e.block_date, + e.pool_id, + SUM(e.weighted_daily_amount_usd) / MAX(e.pricing_count) AS weighted_daily_amount_usd + FROM weighted_pool_amount_estimates e + GROUP BY 1,2 + ) + + SELECT + c.block_date, + c.pool_id, + BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) AS pool_address, + c.pool_symbol, + '2' AS version, + '{{blockchain}}' AS blockchain, + c.pool_type, + c.token_address, + c.token_symbol, + c.daily_amount AS daily_delta, + COALESCE(b.weighted_daily_amount_usd * w.normalized_weight, c.daily_amount_usd) AS daily_delta_usd, + COALESCE(b.weighted_daily_amount_usd * w.normalized_weight, c.daily_amount_usd)/e.eth_price AS daily_delta_eth + FROM daily_usd_balance c + FULL OUTER JOIN weighted_pool_amount_estimates_2 b ON c.block_date = b.block_date + AND c.pool_id = b.pool_id + LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id + AND w.blockchain = '{{blockchain}}' + AND w.version = '2' + AND w.token_address = c.token_address + LEFT JOIN eth_prices e ON e.day = c.block_date + {% endmacro %} + +{# ######################################################################### #} diff --git a/dbt_macros/shared/balancer/balancer_token_balance_changes_macro.sql b/dbt_macros/shared/balancer/balancer_token_balance_changes_macro.sql index 4e606ed955b..d22a7da1cb8 100644 --- a/dbt_macros/shared/balancer/balancer_token_balance_changes_macro.sql +++ b/dbt_macros/shared/balancer/balancer_token_balance_changes_macro.sql @@ -166,4 +166,200 @@ WITH pool_labels AS ( WHERE {{ incremental_predicate('evt_block_time') }} {% endif %} + {% endmacro %} + +{# ######################################################################### #} + +{% macro + balancer_v3_compatible_token_balance_changes_macro( + blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell + ) +%} +WITH pool_labels AS ( + SELECT + address AS pool_id, + name AS pool_symbol, + pool_type + FROM {{ pool_labels_spell }} + WHERE blockchain = '{{blockchain}}' + ), + + token_data AS ( + SELECT + pool, + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + + swaps_changes AS ( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool_id, + token, + SUM(COALESCE(delta, INT256 '0')) AS delta + FROM + ( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool AS pool_id, + tokenIn AS token, + CAST(amountIn as int256) AS delta + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + + UNION ALL + + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool AS pool_id, + tokenOut AS token, + -CAST(amountOut AS int256) AS delta + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} + ) swaps + GROUP BY 1, 2, 3, 4, 5, 6 + ), + + balance_changes AS( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool_id, + category, + deltas, + swapFeeAmountsRaw + FROM + ( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool AS pool_id, + 'add' AS category, + amountsAddedRaw AS deltas, + swapFeeAmountsRaw + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityAdded') }} + + UNION ALL + + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool AS pool_id, + 'remove' AS category, + amountsRemovedRaw AS deltas, + swapFeeAmountsRaw + FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityRemoved') }} + ) adds_and_removes + ), + + zipped_balance_changes AS ( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool_id, + t.tokens, + CASE WHEN b.category = 'add' + THEN d.deltas + WHEN b.category = 'remove' + THEN -d.deltas + END AS deltas, + p.swapFeeAmountsRaw + FROM balance_changes b + JOIN token_data td ON b.pool_id = td.pool + CROSS JOIN UNNEST (td.tokens) WITH ORDINALITY as t(tokens,i) + CROSS JOIN UNNEST (b.deltas) WITH ORDINALITY as d(deltas,i) + CROSS JOIN UNNEST (b.swapFeeAmountsRaw) WITH ORDINALITY as p(swapFeeAmountsRaw,i) + WHERE t.i = d.i + AND d.i = p.i + ORDER BY 1,2,3 + ), + + balances_changes AS ( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool_id, + tokens AS token, + deltas - CAST(swapFeeAmountsRaw as int256) AS delta + FROM zipped_balance_changes + ORDER BY 1, 2, 3 + ) + + + SELECT + date_trunc('day', b.evt_block_time) AS block_date, + b.evt_block_time, + b.evt_block_number, + '{{blockchain}}' AS blockchain, + b.evt_tx_hash, + b.evt_index, + b.pool_id, + BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) AS pool_address, + p.pool_symbol, + p.pool_type, + '{{version}}' AS version, + b.token AS token_address, + t.symbol AS token_symbol, + b.amount AS delta_amount_raw, + CASE WHEN BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) = b.token + THEN amount / POWER (10, 18) --for Balancer Pool Tokens + ELSE amount / POWER (10, COALESCE(t.decimals, 0)) + END AS delta_amount + FROM + ( + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool_id, + token, + COALESCE(delta, INT256 '0') AS amount + FROM balances_changes + + UNION ALL + + SELECT + evt_block_time, + evt_block_number, + evt_tx_hash, + evt_index, + pool_id, + token, + delta AS amount + FROM + swaps_changes + ) b + LEFT JOIN {{ source('tokens', 'erc20') }} t ON t.contract_address = b.token + AND blockchain = '{{blockchain}}' + LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20) + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% endmacro %} \ No newline at end of file diff --git a/dbt_macros/shared/balancer/balancer_transfers_bpt_macro.sql b/dbt_macros/shared/balancer/balancer_transfers_bpt_macro.sql index e7137266c70..bfe582be148 100644 --- a/dbt_macros/shared/balancer/balancer_transfers_bpt_macro.sql +++ b/dbt_macros/shared/balancer/balancer_transfers_bpt_macro.sql @@ -33,6 +33,44 @@ {% if is_incremental() %} WHERE {{ incremental_predicate('evt_block_time') }} {% endif %} ) transfers + {% endmacro %} + {# ######################################################################### #} + {% macro + balancer_v3_compatible_transfers_bpt_macro( + blockchain, version, project_decoded_as + ) +%} + + WITH registered_pools AS ( + SELECT + DISTINCT pool AS pool_address + FROM + {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }} + ) + + SELECT DISTINCT * FROM ( + SELECT + '{{blockchain}}' AS blockchain, + '{{version}}' AS version, + transfer.contract_address, + transfer.evt_tx_hash, + transfer.evt_index, + transfer.evt_block_time, + TRY_CAST(date_trunc('DAY', transfer.evt_block_time) AS date) AS block_date, + TRY_CAST(date_trunc('MONTH', transfer.evt_block_time) AS date) AS block_month, + transfer.evt_block_number, + transfer."from", + transfer.to, + transfer.value + FROM {{ source('erc20_' + blockchain, 'evt_transfer') }} transfer + INNER JOIN registered_pools p ON p.pool_address = transfer.contract_address + {% if not is_incremental() %} + WHERE transfer.evt_block_time >= TIMESTAMP '2024-12-01' + {% endif %} + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} ) transfers + {% endmacro %} \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_sector/labels/labels_addresses.sql b/dbt_subprojects/daily_spellbook/models/_sector/labels/labels_addresses.sql index e0afc67d250..53e6085ec6f 100644 --- a/dbt_subprojects/daily_spellbook/models/_sector/labels/labels_addresses.sql +++ b/dbt_subprojects/daily_spellbook/models/_sector/labels/labels_addresses.sql @@ -6,7 +6,7 @@ post_hook='{{ expose_spells(\'["bitcoin", "ethereum", "solana", "arbitrum", "gnosis", "optimism", "bnb", "avalanche_c", "fantom"]\', "sector", "labels", - \'["soispoke", "hildobby", "ilemi", "hosuke", "kaiblade"]\') }}') + \'["soispoke", "hildobby", "ilemi", "hosuke", "kaiblade", "viniabussafi"]\') }}') }} -- single category labels (no subsets), needs label_type and model_name added still. diff --git a/dbt_subprojects/dex/macros/models/_project/balancer_compatible_trades.sql b/dbt_subprojects/dex/macros/models/_project/balancer_compatible_trades.sql index 792c211fee1..13aefc608b6 100644 --- a/dbt_subprojects/dex/macros/models/_project/balancer_compatible_trades.sql +++ b/dbt_subprojects/dex/macros/models/_project/balancer_compatible_trades.sql @@ -179,3 +179,78 @@ SELECT FROM dexs {% endmacro %} + +{# ######################################################################### #} + +{% macro balancer_compatible_v3_trades( + blockchain = '', + project = '', + version = '', + project_decoded_as = 'balancer_v3', + Vault_evt_Swap = 'Vault_evt_Swap' + ) +%} + +WITH + +pool_labels AS ( + SELECT + blockchain, + address AS pool_address, + name AS pool_symbol, + pool_type + FROM {{ ref('labels_balancer_v3_pools') }} +), + +dexs AS ( + SELECT + swap.evt_block_number AS block_number, + swap.evt_block_time AS block_time, + CAST(NULL AS VARBINARY) AS taker, + CAST(NULL AS VARBINARY) AS maker, + swap.amountOut AS token_bought_amount_raw, + swap.amountIn AS token_sold_amount_raw, + swap.tokenOut AS token_bought_address, + swap.tokenIn AS token_sold_address, + swap.pool AS project_contract_address, + swap.pool AS pool_id, + l.pool_symbol, + l.pool_type, + swap.SwapFeePercentage / POWER(10, 18) AS swap_fee, + swap.evt_tx_hash AS tx_hash, + swap.evt_index + FROM {{ source(project_decoded_as ~ '_' ~ blockchain, Vault_evt_Swap) }} swap + LEFT JOIN pool_labels l + ON l.blockchain = '{{ blockchain }}' + AND l.pool_address = swap.pool + WHERE swap.tokenIn <> swap.pool + AND swap.tokenOut <> swap.pool + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} +) + +SELECT + '{{ blockchain }}' AS blockchain, + '{{ project }}' AS project, + '{{ version }}' AS version, + CAST(date_trunc('month', dexs.block_time) AS date) AS block_month, + CAST(date_trunc('day', dexs.block_time) AS date) AS block_date, + dexs.block_time, + dexs.block_number, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + dexs.token_bought_address, + dexs.token_sold_address, + dexs.taker, + dexs.maker, + dexs.project_contract_address, + dexs.tx_hash, + dexs.evt_index, + dexs.pool_id, + dexs.swap_fee, + dexs.pool_symbol, + dexs.pool_type +FROM dexs + +{% endmacro %} \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml b/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml index 86bc33222c3..c9b3bc91415 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/_schema.yml @@ -6,7 +6,7 @@ models: blockchain: ethereum sector: labels project: balancer_v1 - contributors: balancerlabs + contributors: balancerlabs, viniabussafi config: tags: ['labels', 'ethereum', 'balancer', 'pools'] description: "Balancer V1 liquidity pools created on Ethereum. " @@ -51,7 +51,7 @@ models: blockchain: ethereum sector: labels project: balancer_v1 - contributors: balancerlabs + contributors: balancerlabs, viniabussafi config: tags: ['labels', 'ethereum', 'balancer', 'pools'] description: "Balancer V1 liquidity pools created across blockchains." @@ -364,20 +364,7 @@ models: project: balancer_v3 contributors: balancerlabs, viniabussafi config: - tags: - [ - 'labels', - 'ethereum', - 'polygon', - 'arbitrum', - 'optimism', - 'avalanche_c', - 'base', - 'gnosis', - 'zkevm', - 'balancer', - 'pools', - ] + tags: ['labels', 'ethereum', 'gnosis', 'balancer', 'pools'] description: 'Balancer V3 liquidity pools created across blockchains.' columns: - *blockchain diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql index 8e28104d6ff..55a9624bfe3 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/ethereum/labels_balancer_v3_pools_ethereum.sql @@ -10,7 +10,7 @@ WITH token_data AS ( SELECT pool, - ARRAY_AGG(json_extract_scalar(token, '$.token') ORDER BY token_index) AS tokens + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens FROM ( SELECT pool, @@ -42,8 +42,6 @@ WITH token_data AS ( CROSS JOIN UNNEST(c.tokens) WITH ORDINALITY t(tokens, pos) CROSS JOIN UNNEST(cc.normalizedWeights) WITH ORDINALITY w(weights, pos) WHERE t.pos = w.pos - ) zip - UNION ALL @@ -53,11 +51,12 @@ WITH token_data AS ( 0 AS weights, cc.symbol, 'stable' AS pool_type - FROM {{ source('balancer_v3_ethereum', 'Vault_evt_PoolRegistered') }} c + FROM token_data c INNER JOIN {{ source('balancer_v3_ethereum', 'StablePoolFactory_call_create') }} cc ON c.pool = cc.output_pool - CROSS JOIN UNNEST(cc.tokens) AS t(tokens) - ), + CROSS JOIN UNNEST(c.tokens) AS t(tokens) + ) zip + ), settings AS ( SELECT @@ -67,7 +66,7 @@ WITH token_data AS ( p.symbol AS pool_symbol, p.pool_type FROM pools p - LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = CAST(t.contract_address AS VARCHAR) + LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address AND t.blockchain = 'ethereum' ) diff --git a/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql b/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql index 5997e2be864..8333126451b 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/labels/gnosis/labels_balancer_v3_pools_gnosis.sql @@ -10,7 +10,7 @@ WITH token_data AS ( SELECT pool, - ARRAY_AGG(json_extract_scalar(token, '$.token') ORDER BY token_index) AS tokens + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens FROM ( SELECT pool, @@ -42,8 +42,6 @@ WITH token_data AS ( CROSS JOIN UNNEST(c.tokens) WITH ORDINALITY t(tokens, pos) CROSS JOIN UNNEST(cc.normalizedWeights) WITH ORDINALITY w(weights, pos) WHERE t.pos = w.pos - ) zip - UNION ALL @@ -53,11 +51,12 @@ WITH token_data AS ( 0 AS weights, cc.symbol, 'stable' AS pool_type - FROM {{ source('balancer_v3_gnosis', 'Vault_evt_PoolRegistered') }} c + FROM token_data c INNER JOIN {{ source('balancer_v3_gnosis', 'StablePoolFactory_call_create') }} cc ON c.pool = cc.output_pool - CROSS JOIN UNNEST(cc.tokens) AS t(tokens) - ), + CROSS JOIN UNNEST(c.tokens) AS t(tokens) + ) zip + ), settings AS ( SELECT @@ -67,7 +66,7 @@ WITH token_data AS ( p.symbol AS pool_symbol, p.pool_type FROM pools p - LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = CAST(t.contract_address AS VARCHAR) + LEFT JOIN {{ source('tokens', 'erc20') }} t ON p.token_address = t.contract_address AND t.blockchain = 'gnosis' ) @@ -98,4 +97,4 @@ FROM ( GROUP BY s1.pool_id, token_symbol, pool_symbol, normalized_weight, pool_type ) s GROUP BY pool_id, pool_symbol, pool_type -ORDER BY 1 \ No newline at end of file +ORDER BY 1 diff --git a/dbt_subprojects/dex/models/_projects/balancer/pools/_schema.yml b/dbt_subprojects/dex/models/_projects/balancer/pools/_schema.yml index 79a17fe7cdf..c62279b3d91 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/pools/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/balancer/pools/_schema.yml @@ -8,7 +8,7 @@ models: config: tags: ['arbitrum', 'avalanche_c', 'base', 'ethereum', 'gnosis', 'optimism', 'polygon', 'zkevm', 'pools', 'fees'] description: > - Balancer v2 swap fees stored at the pool level. + Balancer v2 and v3 swap fees stored at the pool level. data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -265,4 +265,56 @@ models: - *tx_index - *block_time - *block_number - - *swap_fee_percentage \ No newline at end of file + - *swap_fee_percentage + + - name: balancer_v3_ethereum_pools_fees + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'balancer', 'pools_fees'] + description: > + Balancer v3 swap fees stored at the pool level. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_number + - tx_hash + - index + columns: + - *blockchain + - *version + - *contract_address + - *tx_hash + - *index + - *tx_index + - *block_time + - *block_number + - *swap_fee_percentage + + - name: balancer_v3_gnosis_pools_fees + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'balancer', 'pools_fees'] + description: > + Balancer v3 swap fees stored at the pool level. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_number + - tx_hash + - index + columns: + - *blockchain + - *version + - *contract_address + - *tx_hash + - *index + - *tx_index + - *block_time + - *block_number + - *swap_fee_percentage \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/pools/balancer_pools_fees.sql b/dbt_subprojects/dex/models/_projects/balancer/pools/balancer_pools_fees.sql index 2a54b518ec1..a733d20db6a 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/pools/balancer_pools_fees.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/pools/balancer_pools_fees.sql @@ -16,7 +16,9 @@ ref('balancer_v2_gnosis_pools_fees'), ref('balancer_v2_optimism_pools_fees'), ref('balancer_v2_polygon_pools_fees'), - ref('balancer_v2_zkevm_pools_fees') + ref('balancer_v2_zkevm_pools_fees'), + ref('balancer_v3_ethereum_pools_fees'), + ref('balancer_v3_gnosis_pools_fees') ] %} diff --git a/dbt_subprojects/dex/models/_projects/balancer/pools/ethereum/balancer_v3_ethereum_pools_fees.sql b/dbt_subprojects/dex/models/_projects/balancer/pools/ethereum/balancer_v3_ethereum_pools_fees.sql new file mode 100644 index 00000000000..69fdd2eb826 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/pools/ethereum/balancer_v3_ethereum_pools_fees.sql @@ -0,0 +1,28 @@ +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'pools_fees', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['block_number', 'tx_hash', 'index'] + ) +}} + +SELECT + 'ethereum' AS blockchain, + '3' AS version, + pool AS contract_address, + evt_tx_hash AS tx_hash, + evt_index AS tx_index, + evt_index AS index, + evt_block_time AS block_time, + evt_block_number AS block_number, + swapFeePercentage AS swap_fee_percentage +FROM {{ source ('balancer_v3_ethereum', 'Vault_evt_SwapFeePercentageChanged') }} +WHERE 1 = 1 +{% if is_incremental() %} +AND {{ incremental_predicate('evt_block_time') }} +{% endif %} + diff --git a/dbt_subprojects/dex/models/_projects/balancer/pools/gnosis/balancer_v3_gnosis_pools_fees.sql b/dbt_subprojects/dex/models/_projects/balancer/pools/gnosis/balancer_v3_gnosis_pools_fees.sql new file mode 100644 index 00000000000..88743f46eb9 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/pools/gnosis/balancer_v3_gnosis_pools_fees.sql @@ -0,0 +1,28 @@ +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'pools_fees', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['block_number', 'tx_hash', 'index'] + ) +}} + +SELECT + 'gnosis' AS blockchain, + '3' AS version, + pool AS contract_address, + evt_tx_hash AS tx_hash, + evt_index AS tx_index, + evt_index AS index, + evt_block_time AS block_time, + evt_block_number AS block_number, + swapFeePercentage AS swap_fee_percentage +FROM {{ source ('balancer_v3_gnosis', 'Vault_evt_SwapFeePercentageChanged') }} +WHERE 1 = 1 +{% if is_incremental() %} +AND {{ incremental_predicate('evt_block_time') }} +{% endif %} + diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/_schema.yml b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/_schema.yml index 14737f94d4a..c46f5ff8483 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/_schema.yml @@ -138,4 +138,43 @@ models: - *tx_hash - *tx_from - *tx_to - - *evt_index \ No newline at end of file + - *evt_index + + - name: balancer_v3_ethereum_trades + meta: + blockchain: ethereum + sector: dex + contributors: viniabussafi + config: + tags: [ 'balancer', 'ethereum', 'amm', 'trades', 'dex', 'beta' ] + description: > + DEX trades on balancer v3 on ethereum + columns: + - *blockchain + - *project + - *version + - *block_month + - *block_date + - *block_time + - *block_number + - *token_bought_symbol + - *token_sold_symbol + - *token_pair + - *token_bought_amount + - *token_sold_amount + - *token_bought_amount_raw + - *token_sold_amount_raw + - *amount_usd + - *token_bought_address + - *token_sold_address + - *taker + - *maker + - *project_contract_address + - *pool_symbol + - *pool_type + - *pool_id + - *swap_fee + - *tx_hash + - *tx_from + - *tx_to + - *evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_ethereum_trades.sql b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_ethereum_trades.sql index 965b86cdf54..d6657d18d4f 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_ethereum_trades.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_ethereum_trades.sql @@ -8,7 +8,8 @@ {% set balancer_models = [ ref('balancer_v1_ethereum_trades'), ref('balancer_v2_ethereum_trades'), - ref('balancer_cowswap_amm_ethereum_trades') + ref('balancer_cowswap_amm_ethereum_trades'), + ref('balancer_v3_ethereum_trades') ] %} SELECT * diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql new file mode 100644 index 00000000000..71d7257a356 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql @@ -0,0 +1,151 @@ +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'trades', + materialized = 'view', + post_hook = '{{ expose_spells(\'["ethereum"]\', + spell_type = "project", + spell_name = "balancer", + contributors = \'["viniabussafi"]\') }}' + ) +}} + +WITH + dexs_base AS ( + SELECT + tx_hash, + evt_index, + pool_id, + swap_fee, + pool_symbol, + pool_type + FROM {{ ref('balancer_v3_ethereum_base_trades') }} + ), + + dexs AS ( + SELECT + dexs.blockchain, + dexs.project, + dexs.version, + dexs.block_month, + dexs.block_date, + dexs.block_time, + dexs.block_number, + dexs.token_bought_symbol, + dexs.token_sold_symbol, + dexs.token_pair, + dexs.token_bought_amount, + dexs.token_sold_amount, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + dexs.amount_usd, + dexs.token_bought_address, + dexs.token_sold_address, + dexs.taker, + dexs.maker, + dexs.project_contract_address, + dexs.tx_hash, + dexs.tx_from, + dexs.tx_to, + dexs.evt_index, + dexs_base.pool_id, + dexs_base.swap_fee, + dexs_base.pool_symbol, + dexs_base.pool_type + FROM {{ ref('dex_trades') }} dexs + INNER JOIN dexs_base + ON dexs.tx_hash = dexs_base.tx_hash + AND dexs.evt_index = dexs_base.evt_index + WHERE dexs.blockchain = 'ethereum' + AND dexs.project = 'balancer' + AND dexs.version = '3' + ), + + bpa AS ( + SELECT + dexs.block_number, + dexs.tx_hash, + dexs.evt_index, + bpt_prices.contract_address, + dexs.block_time, + MAX(bpt_prices.day) AS bpa_max_block_date + FROM dexs + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpt_prices + ON bpt_prices.contract_address = dexs.token_bought_address + AND bpt_prices.day <= DATE_TRUNC('day', dexs.block_time) + AND bpt_prices.blockchain = 'ethereum' + GROUP BY 1, 2, 3, 4, 5 + ), + + bpb AS ( + SELECT + dexs.block_number, + dexs.tx_hash, + dexs.evt_index, + bpt_prices.contract_address, + dexs.block_time, + MAX(bpt_prices.day) AS bpb_max_block_date + FROM dexs + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpt_prices + ON bpt_prices.contract_address = dexs.token_sold_address + AND bpt_prices.day <= DATE_TRUNC('day', dexs.block_time) + AND bpt_prices.blockchain = 'ethereum' + GROUP BY 1, 2, 3, 4, 5 + ) + +SELECT + dexs.blockchain, + dexs.project, + dexs.version, + dexs.block_date, + dexs.block_number, + dexs.block_month, + dexs.block_time, + dexs.token_bought_symbol, + dexs.token_sold_symbol, + dexs.token_pair, + dexs.token_bought_amount, + dexs.token_sold_amount, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + COALESCE( + dexs.amount_usd, + dexs.token_bought_amount_raw / POWER(10, COALESCE(erc20a.decimals, 18)) * bpa_bpt_prices.bpt_price, + dexs.token_sold_amount_raw / POWER(10, COALESCE(erc20b.decimals, 18)) * bpb_bpt_prices.bpt_price + ) AS amount_usd, + dexs.token_bought_address, + dexs.token_sold_address, + dexs.taker, + dexs.maker, + dexs.project_contract_address, + dexs.pool_id, + dexs.swap_fee, + dexs.pool_symbol, + dexs.pool_type, + dexs.tx_hash, + dexs.tx_from, + dexs.tx_to, + dexs.evt_index +FROM dexs + LEFT JOIN {{ source('tokens', 'erc20') }} erc20a + ON erc20a.contract_address = dexs.token_bought_address + AND erc20a.blockchain = dexs.blockchain + LEFT JOIN {{ source('tokens', 'erc20') }} erc20b + ON erc20b.contract_address = dexs.token_sold_address + AND erc20b.blockchain = dexs.blockchain + INNER JOIN bpa + ON bpa.block_number = dexs.block_number + AND bpa.tx_hash = dexs.tx_hash + AND bpa.evt_index = dexs.evt_index + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpa_bpt_prices + ON bpa_bpt_prices.contract_address = bpa.contract_address + AND bpa_bpt_prices.day = bpa.bpa_max_block_date + AND bpa_bpt_prices.blockchain = 'ethereum' + INNER JOIN bpb + ON bpb.block_number = dexs.block_number + AND bpb.tx_hash = dexs.tx_hash + AND bpb.evt_index = dexs.evt_index + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpb_bpt_prices + ON bpb_bpt_prices.contract_address = bpb.contract_address + AND bpb_bpt_prices.day = bpb.bpb_max_block_date + AND bpb_bpt_prices.blockchain = 'ethereum' \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/_schema.yml b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/_schema.yml index 7da8f746429..2ba4aa24f59 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/_schema.yml @@ -142,4 +142,43 @@ models: - *tx_hash - *tx_from - *tx_to - - *evt_index \ No newline at end of file + - *evt_index + + - name: balancer_v3_gnosis_trades + meta: + blockchain: gnosis + sector: dex + contributors: viniabussafi + config: + tags: [ 'balancer', 'gnosis', 'amm', 'trades', 'dex', 'beta' ] + description: > + DEX trades on balancer v3 on gnosis + columns: + - *blockchain + - *project + - *version + - *block_month + - *block_date + - *block_time + - *block_number + - *token_bought_symbol + - *token_sold_symbol + - *token_pair + - *token_bought_amount + - *token_sold_amount + - *token_bought_amount_raw + - *token_sold_amount_raw + - *amount_usd + - *token_bought_address + - *token_sold_address + - *taker + - *maker + - *project_contract_address + - *pool_symbol + - *pool_type + - *pool_id + - *swap_fee + - *tx_hash + - *tx_from + - *tx_to + - *evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_gnosis_trades.sql b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_gnosis_trades.sql index abc1b89aa99..4b8d2711d4d 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_gnosis_trades.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_gnosis_trades.sql @@ -7,7 +7,8 @@ {% set balancer_models = [ ref('balancer_v2_gnosis_trades'), - ref('balancer_cowswap_amm_gnosis_trades') + ref('balancer_cowswap_amm_gnosis_trades'), + ref('balancer_v3_gnosis_trades') ] %} SELECT * diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql new file mode 100644 index 00000000000..71b73f5fa60 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql @@ -0,0 +1,151 @@ +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'trades', + materialized = 'view', + post_hook = '{{ expose_spells(\'["gnosis"]\', + spell_type = "project", + spell_name = "balancer", + contributors = \'["viniabussafi"]\') }}' + ) +}} + +WITH + dexs_base AS ( + SELECT + tx_hash, + evt_index, + pool_id, + swap_fee, + pool_symbol, + pool_type + FROM {{ ref('balancer_v3_gnosis_base_trades') }} + ), + + dexs AS ( + SELECT + dexs.blockchain, + dexs.project, + dexs.version, + dexs.block_month, + dexs.block_date, + dexs.block_time, + dexs.block_number, + dexs.token_bought_symbol, + dexs.token_sold_symbol, + dexs.token_pair, + dexs.token_bought_amount, + dexs.token_sold_amount, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + dexs.amount_usd, + dexs.token_bought_address, + dexs.token_sold_address, + dexs.taker, + dexs.maker, + dexs.project_contract_address, + dexs.tx_hash, + dexs.tx_from, + dexs.tx_to, + dexs.evt_index, + dexs_base.pool_id, + dexs_base.swap_fee, + dexs_base.pool_symbol, + dexs_base.pool_type + FROM {{ ref('dex_trades') }} dexs + INNER JOIN dexs_base + ON dexs.tx_hash = dexs_base.tx_hash + AND dexs.evt_index = dexs_base.evt_index + WHERE dexs.blockchain = 'gnosis' + AND dexs.project = 'balancer' + AND dexs.version = '3' + ), + + bpa AS ( + SELECT + dexs.block_number, + dexs.tx_hash, + dexs.evt_index, + bpt_prices.contract_address, + dexs.block_time, + MAX(bpt_prices.day) AS bpa_max_block_date + FROM dexs + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpt_prices + ON bpt_prices.contract_address = dexs.token_bought_address + AND bpt_prices.day <= DATE_TRUNC('day', dexs.block_time) + AND bpt_prices.blockchain = 'gnosis' + GROUP BY 1, 2, 3, 4, 5 + ), + + bpb AS ( + SELECT + dexs.block_number, + dexs.tx_hash, + dexs.evt_index, + bpt_prices.contract_address, + dexs.block_time, + MAX(bpt_prices.day) AS bpb_max_block_date + FROM dexs + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpt_prices + ON bpt_prices.contract_address = dexs.token_sold_address + AND bpt_prices.day <= DATE_TRUNC('day', dexs.block_time) + AND bpt_prices.blockchain = 'gnosis' + GROUP BY 1, 2, 3, 4, 5 + ) + +SELECT + dexs.blockchain, + dexs.project, + dexs.version, + dexs.block_date, + dexs.block_number, + dexs.block_month, + dexs.block_time, + dexs.token_bought_symbol, + dexs.token_sold_symbol, + dexs.token_pair, + dexs.token_bought_amount, + dexs.token_sold_amount, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + COALESCE( + dexs.amount_usd, + dexs.token_bought_amount_raw / POWER(10, COALESCE(erc20a.decimals, 18)) * bpa_bpt_prices.bpt_price, + dexs.token_sold_amount_raw / POWER(10, COALESCE(erc20b.decimals, 18)) * bpb_bpt_prices.bpt_price + ) AS amount_usd, + dexs.token_bought_address, + dexs.token_sold_address, + dexs.taker, + dexs.maker, + dexs.project_contract_address, + dexs.pool_id, + dexs.swap_fee, + dexs.pool_symbol, + dexs.pool_type, + dexs.tx_hash, + dexs.tx_from, + dexs.tx_to, + dexs.evt_index +FROM dexs + LEFT JOIN {{ source('tokens', 'erc20') }} erc20a + ON erc20a.contract_address = dexs.token_bought_address + AND erc20a.blockchain = dexs.blockchain + LEFT JOIN {{ source('tokens', 'erc20') }} erc20b + ON erc20b.contract_address = dexs.token_sold_address + AND erc20b.blockchain = dexs.blockchain + INNER JOIN bpa + ON bpa.block_number = dexs.block_number + AND bpa.tx_hash = dexs.tx_hash + AND bpa.evt_index = dexs.evt_index + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpa_bpt_prices + ON bpa_bpt_prices.contract_address = bpa.contract_address + AND bpa_bpt_prices.day = bpa.bpa_max_block_date + AND bpa_bpt_prices.blockchain = 'gnosis' + INNER JOIN bpb + ON bpb.block_number = dexs.block_number + AND bpb.tx_hash = dexs.tx_hash + AND bpb.evt_index = dexs.evt_index + LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpb_bpt_prices + ON bpb_bpt_prices.contract_address = bpb.contract_address + AND bpb_bpt_prices.day = bpb.bpb_max_block_date + AND bpb_bpt_prices.blockchain = 'gnosis' \ No newline at end of file diff --git a/dbt_subprojects/dex/models/trades/ethereum/_schema.yml b/dbt_subprojects/dex/models/trades/ethereum/_schema.yml index aa463001637..9290c868bf6 100644 --- a/dbt_subprojects/dex/models/trades/ethereum/_schema.yml +++ b/dbt_subprojects/dex/models/trades/ethereum/_schema.yml @@ -251,7 +251,7 @@ models: blockchain: ethereum sector: dex project: balancer - contributors: bizzyvinci, thetroyharris, tomfutago + contributors: bizzyvinci, thetroyharris, tomfutago, viniabussafi config: tags: ["ethereum", "dex", "trades", "balancer"] description: "Balancer v2 ethereum base trades" @@ -265,6 +265,25 @@ models: filter: version: 2 + - name: balancer_v3_ethereum_base_trades + meta: + blockchain: ethereum + sector: dex + project: balancer + contributors: viniabussafi + config: + tags: ["ethereum", "dex", "trades", "balancer"] + description: "Balancer v3 ethereum base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('balancer_ethereum_base_trades_seed') + filter: + version: 3 + - name: fraxswap_ethereum_base_trades meta: blockchain: ethereum diff --git a/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql b/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql index 3a121900231..25e4ae0e1b8 100644 --- a/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql @@ -20,6 +20,7 @@ , ref('shibaswap_v1_ethereum_base_trades') , ref('balancer_v1_ethereum_base_trades') , ref('balancer_v2_ethereum_base_trades') + , ref('balancer_v3_ethereum_base_trades') , ref('fraxswap_ethereum_base_trades') , ref('bancor_ethereum_base_trades') , ref('verse_dex_ethereum_base_trades') diff --git a/dbt_subprojects/dex/models/trades/ethereum/platforms/balancer_v3_ethereum_base_trades.sql b/dbt_subprojects/dex/models/trades/ethereum/platforms/balancer_v3_ethereum_base_trades.sql new file mode 100644 index 00000000000..46aff728ebd --- /dev/null +++ b/dbt_subprojects/dex/models/trades/ethereum/platforms/balancer_v3_ethereum_base_trades.sql @@ -0,0 +1,19 @@ +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'base_trades', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{{ + balancer_compatible_v3_trades( + blockchain = 'ethereum', + project = 'balancer', + version = '3' + ) +}} diff --git a/dbt_subprojects/dex/models/trades/gnosis/_schema.yml b/dbt_subprojects/dex/models/trades/gnosis/_schema.yml index 09081786a68..1ac8909ff33 100644 --- a/dbt_subprojects/dex/models/trades/gnosis/_schema.yml +++ b/dbt_subprojects/dex/models/trades/gnosis/_schema.yml @@ -48,7 +48,7 @@ models: blockchain: gnosis sector: dex project: balancer - contributors: bizzyvinci, thetroyharris, tomfutago + contributors: bizzyvinci, thetroyharris, tomfutago, viniabussafi config: tags: [ 'gnosis', 'dex', 'trades', 'balancer' ] description: "Balancer v2 gnosis base trades" @@ -62,6 +62,25 @@ models: filter: version: 2 + - name: balancer_v3_gnosis_base_trades + meta: + blockchain: gnosis + sector: dex + project: balancer + contributors: viniabussafi + config: + tags: [ 'gnosis', 'dex', 'trades', 'balancer' ] + description: "Balancer v3 gnosis base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('balancer_gnosis_base_trades_seed') + filter: + version: 3 + - name: honeyswap_v2_gnosis_base_trades meta: blockchain: gnosis diff --git a/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql b/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql index a8196517cd8..70883d551bb 100644 --- a/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql @@ -9,6 +9,7 @@ ref('sushiswap_v1_gnosis_base_trades') , ref('sushiswap_v2_gnosis_base_trades') , ref('balancer_v2_gnosis_base_trades') + , ref('balancer_v3_gnosis_base_trades') , ref('honeyswap_v2_gnosis_base_trades') , ref('elk_finance_gnosis_base_trades') ] %} diff --git a/dbt_subprojects/dex/models/trades/gnosis/platforms/balancer_v3_gnosis_base_trades.sql b/dbt_subprojects/dex/models/trades/gnosis/platforms/balancer_v3_gnosis_base_trades.sql new file mode 100644 index 00000000000..9e78c0be294 --- /dev/null +++ b/dbt_subprojects/dex/models/trades/gnosis/platforms/balancer_v3_gnosis_base_trades.sql @@ -0,0 +1,19 @@ +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'base_trades', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{{ + balancer_compatible_v3_trades( + blockchain = 'gnosis', + project = 'balancer', + version = '3' + ) +}} diff --git a/dbt_subprojects/dex/seeds/trades/balancer_ethereum_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/balancer_ethereum_base_trades_seed.csv index b08148c48b3..52089a72c77 100644 --- a/dbt_subprojects/dex/seeds/trades/balancer_ethereum_base_trades_seed.csv +++ b/dbt_subprojects/dex/seeds/trades/balancer_ethereum_base_trades_seed.csv @@ -1,3 +1,4 @@ blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw ethereum,balancer,1,2020-08-02,0xac80f41cdfc018bfdc6d37f881fe3def377aa9fd3d0b794c3d7e7ee1e6501ce4,99,0x6b175474e89094c44da98b954eedeac495271d0f,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,10577678,25347466807281742558,65815288571622598 ethereum,balancer,2,2022-05-06,0xb6bba47d296682416f32fb4264cfe3fe2994284183081beafc8c36cc1e76b1f2,3,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xba100000625a3754423978a60c9317c58a424e3d,14725355,3755554997108104989,758723679950740258816 +ethereum,balancer,3,2024-12-11,0xa4909d720f14c0c4a89720150982a60768dc16bd30265cba6c660ab0cb0dbdb5,140,0xd4fa2d31b7968e448877f69a96de69f5de8cd23e,0x7bc3485026ac48b6cf9baf0a377477fff5703af8,21378232,452946,454359 \ No newline at end of file diff --git a/dbt_subprojects/dex/seeds/trades/balancer_gnosis_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/balancer_gnosis_base_trades_seed.csv index f9f3f1084b5..c3aa2a57691 100644 --- a/dbt_subprojects/dex/seeds/trades/balancer_gnosis_base_trades_seed.csv +++ b/dbt_subprojects/dex/seeds/trades/balancer_gnosis_base_trades_seed.csv @@ -1,2 +1,3 @@ blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw gnosis,balancer,2,2023-01-20,0x70a7380d73bd8dc96aa2a2ec338f6a660df54e797bab507b5ee04c27e6433844,20,0xe91d153e0b41518a2ce8dd3d7944fa863463a97d,0xddafbb505ad214d7b80b1f830fccc89b60fb7a83,26059328,2811201702694397942,2811563 +gnosis,balancer,3,2024-12-09,0xde406d9adbc7ec5c0157990cbcb77a6eccb053321786b91de53a4c5a848bcfab,53,0xaf204776c7245bf4147c2612bf6e5972ee483701,0x7c16f0185a26db0ae7a9377f23bc18ea7ce5d644,37444463,4106199510945472,18921089727982 \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes.sql index 7f531a769f3..b2955c5f4a7 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes.sql @@ -16,7 +16,9 @@ ref('balancer_v2_gnosis_token_balance_changes'), ref('balancer_v2_optimism_token_balance_changes'), ref('balancer_v2_polygon_token_balance_changes'), - ref('balancer_v2_zkevm_token_balance_changes') + ref('balancer_v2_zkevm_token_balance_changes'), + ref('balancer_v3_ethereum_token_balance_changes'), + ref('balancer_v3_gnosis_token_balance_changes') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes_daily.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes_daily.sql index 8e199a25708..4ab6c62937e 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes_daily.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/balancer_token_balance_changes_daily.sql @@ -16,7 +16,9 @@ ref('balancer_v2_gnosis_token_balance_changes_daily'), ref('balancer_v2_optimism_token_balance_changes_daily'), ref('balancer_v2_polygon_token_balance_changes_daily'), - ref('balancer_v2_zkevm_token_balance_changes_daily') + ref('balancer_v2_zkevm_token_balance_changes_daily'), + ref('balancer_v3_ethereum_token_balance_changes_daily'), + ref('balancer_v3_gnosis_token_balance_changes_daily') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/_schema.yml index ac563c49277..9fe9fe54fe6 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/_schema.yml @@ -128,4 +128,66 @@ models: description: "Daily total impact on token balance, in USD" - &daily_delta_eth name: daily_delta_eth - description: "Daily total impact on token balance, in ETH" \ No newline at end of file + description: "Daily total impact on token balance, in ETH" + + - name: balancer_v3_ethereum_token_balance_changes + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'supply', 'changes'] + description: > + Token Balance Changes on Balancer Pools. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - evt_tx_hash + - evt_index + - token_address + columns: + - *block_date + - *evt_block_time + - *evt_block_number + - *blockchain + - *evt_tx_hash + - *evt_index + - *pool_id + - *pool_address + - *pool_symbol + - *pool_type + - *version + - *token_address + - *token_symbol + - *delta_amount_raw + - *delta_amount + + - name: balancer_v3_ethereum_token_balance_changes_daily + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'supply', 'changes'] + description: > + Token Balance Changes on Balancer Pools, grouped by day. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - pool_id + - token_address + columns: + - *block_date + - *pool_id + - *pool_address + - *pool_symbol + - *version + - *blockchain + - *pool_type + - *token_address + - *token_symbol + - *daily_delta + - *daily_delta_usd + - *daily_delta_eth \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes.sql new file mode 100644 index 00000000000..83ee62ca4af --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes.sql @@ -0,0 +1,19 @@ +{% set blockchain = 'ethereum' %} + +{{ config( + schema = 'balancer_v3_ethereum', + alias = 'token_balance_changes', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_token_balance_changes_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes_daily.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes_daily.sql new file mode 100644 index 00000000000..7ad4d5fc051 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/ethereum/balancer_v3_ethereum_token_balance_changes_daily.sql @@ -0,0 +1,19 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'token_balance_changes_daily', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_token_balance_changes_daily_agg_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer' + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/_schema.yml index 42b0c73b982..790b9fa25ec 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/_schema.yml @@ -97,4 +97,66 @@ models: description: "Daily total impact on token balance, in USD" - &daily_delta_eth name: daily_delta_eth - description: "Daily total impact on token balance, in eth" \ No newline at end of file + description: "Daily total impact on token balance, in eth" + + - name: balancer_v3_gnosis_token_balance_changes + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'supply', 'changes'] + description: > + Token Balance Changes on Balancer Pools. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - evt_tx_hash + - evt_index + - token_address + columns: + - *block_date + - *evt_block_time + - *evt_block_number + - *blockchain + - *evt_tx_hash + - *evt_index + - *pool_id + - *pool_address + - *pool_symbol + - *pool_type + - *version + - *token_address + - *token_symbol + - *delta_amount_raw + - *delta_amount + + - name: balancer_v3_gnosis_token_balance_changes_daily + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'supply', 'changes'] + description: > + Token Balance Changes on Balancer Pools, grouped by day. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - pool_id + - token_address + columns: + - *block_date + - *pool_id + - *pool_address + - *pool_symbol + - *version + - *blockchain + - *pool_type + - *token_address + - *token_symbol + - *daily_delta + - *daily_delta_usd + - *daily_delta_eth \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes.sql new file mode 100644 index 00000000000..17833ba59b0 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes.sql @@ -0,0 +1,19 @@ +{% set blockchain = 'gnosis' %} + +{{ config( + schema = 'balancer_v3_gnosis', + alias = 'token_balance_changes', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_token_balance_changes_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes_daily.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes_daily.sql new file mode 100644 index 00000000000..09b28214e75 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/balances/gnosis/balancer_v3_gnosis_token_balance_changes_daily.sql @@ -0,0 +1,19 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'token_balance_changes_daily', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_token_balance_changes_daily_agg_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer' + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_prices.sql index 0a1180a8013..1f59d4db7bb 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_prices.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_prices.sql @@ -16,7 +16,9 @@ ref('balancer_v2_optimism_bpt_prices'), ref('balancer_v2_avalanche_c_bpt_prices'), ref('balancer_v2_base_bpt_prices'), - ref('balancer_v2_zkevm_bpt_prices') + ref('balancer_v2_zkevm_bpt_prices'), + ref('balancer_v3_ethereum_bpt_prices'), + ref('balancer_v3_gnosis_bpt_prices') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply.sql index e43ef846b55..3acb22256f3 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply.sql @@ -16,7 +16,9 @@ ref('balancer_v2_optimism_bpt_supply'), ref('balancer_v2_avalanche_c_bpt_supply'), ref('balancer_v2_base_bpt_supply'), - ref('balancer_v2_zkevm_bpt_supply') + ref('balancer_v2_zkevm_bpt_supply'), + ref('balancer_v3_gnosis_bpt_supply'), + ref('balancer_v3_ethereum_bpt_supply') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes.sql index e46a587b3b0..108a0053a89 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes.sql @@ -16,7 +16,9 @@ ref('balancer_v2_gnosis_bpt_supply_changes'), ref('balancer_v2_optimism_bpt_supply_changes'), ref('balancer_v2_polygon_bpt_supply_changes'), - ref('balancer_v2_zkevm_bpt_supply_changes') + ref('balancer_v2_zkevm_bpt_supply_changes'), + ref('balancer_v3_ethereum_bpt_supply_changes'), + ref('balancer_v3_gnosis_bpt_supply_changes') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes_daily.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes_daily.sql index 5b593cef48a..84146e8a633 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes_daily.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_bpt_supply_changes_daily.sql @@ -16,7 +16,9 @@ ref('balancer_v2_gnosis_bpt_supply_changes_daily'), ref('balancer_v2_optimism_bpt_supply_changes_daily'), ref('balancer_v2_polygon_bpt_supply_changes_daily'), - ref('balancer_v2_zkevm_bpt_supply_changes_daily') + ref('balancer_v2_zkevm_bpt_supply_changes_daily'), + ref('balancer_v3_ethereum_bpt_supply_changes_daily'), + ref('balancer_v3_gnosis_bpt_supply_changes_daily') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_transfers_bpt.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_transfers_bpt.sql index dda9c32ecb1..a629dbcfe88 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_transfers_bpt.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/balancer_transfers_bpt.sql @@ -16,7 +16,9 @@ ref('balancer_v2_gnosis_transfers_bpt'), ref('balancer_v2_optimism_transfers_bpt'), ref('balancer_v2_polygon_transfers_bpt'), - ref('balancer_v2_zkevm_transfers_bpt') + ref('balancer_v2_zkevm_transfers_bpt'), + ref('balancer_v3_ethereum_transfers_bpt'), + ref('balancer_v3_gnosis_transfers_bpt') ] %} SELECT * diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/_schema.yml index e9a905070ff..e113be8d1ee 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/_schema.yml @@ -175,4 +175,134 @@ models: - *token_address - &daily_delta name: daily_delta - description: "Daily total impact on BPT supply" \ No newline at end of file + description: "Daily total impact on BPT supply" + + - name: balancer_v3_ethereum_transfers_bpt + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'transfers'] + description: > + Balancer Pool Token (BPT) transfer logs on Balancer, an automated portfolio manager and trading platform, on ethereum. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - evt_tx_hash + - evt_index + - block_date + columns: + - *blockchain + - *version + - *contract_address + - *block_date + - *block_month + - *evt_tx_hash + - *evt_index + - *evt_block_time + - *evt_block_number + - *from + - *to + - *value + + - name: balancer_v3_ethereum_bpt_prices + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'prices'] + description: > + Balancer Pool Token (BPT) daily price by pool on Balancer, an automated portfolio manager and trading platform. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - day + - contract_address + columns: + - *blockchain + - *day + - *version + - *decimals + - *contract_address + - *bpt_price + + - name: balancer_v3_ethereum_bpt_supply + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'supply'] + description: > + Balancer Pool Token (BPT) supply by pool on Balancer, an automated portfolio manager and trading platform. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - blockchain + - token_address + columns: + - *day + - *pool_type + - *version + - *blockchain + - *token_address + - *supply + + - name: balancer_v3_ethereum_bpt_supply_changes + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'supply', 'changes'] + description: > + Balancer Pool Token (BPT) supply change events. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - evt_tx_hash + - evt_index + - label + columns: + - *block_date + - *evt_block_time + - *evt_block_number + - *blockchain + - *evt_tx_hash + - *evt_index + - *pool_type + - *pool_symbol + - *version + - *label + - *token_address + - *delta_amount_raw + - *delta_amount + + - name: balancer_v3_ethereum_bpt_supply_changes_daily + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'bpt', 'supply', 'changes'] + description: > + Balancer Pool Token (BPT) supply change events. grouped by day + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - blockchain + - token_address + columns: + - *block_date + - *blockchain + - *pool_type + - *pool_symbol + - *version + - *token_address + - *daily_delta \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_prices.sql new file mode 100644 index 00000000000..4e44eedf294 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_prices.sql @@ -0,0 +1,21 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'bpt_prices', + materialized = 'table', + file_format = 'delta' + ) +}} + + +{{ + balancer_v3_compatible_bpt_prices_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply.sql new file mode 100644 index 00000000000..a484e1725b6 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply.sql @@ -0,0 +1,21 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema='balancer_v3_ethereum', + alias = 'bpt_supply', + materialized = 'table', + file_format = 'delta' + + ) +}} + +{{ + balancer_v3_compatible_bpt_supply_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes.sql new file mode 100644 index 00000000000..fa93184fdde --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes.sql @@ -0,0 +1,23 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'bpt_supply_changes', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_date', 'evt_tx_hash', 'evt_index', 'label'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +{{ + balancer_v3_compatible_bpt_supply_changes_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes_daily.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes_daily.sql new file mode 100644 index 00000000000..dbc197a5b48 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_bpt_supply_changes_daily.sql @@ -0,0 +1,19 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'bpt_supply_changes_daily', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_bpt_supply_changes_daily_agg_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer' + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_transfers_bpt.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_transfers_bpt.sql new file mode 100644 index 00000000000..81c95ef01cd --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/ethereum/balancer_v3_ethereum_transfers_bpt.sql @@ -0,0 +1,22 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'transfers_bpt', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_date', 'evt_tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +{{ + balancer_v3_compatible_transfers_bpt_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3' + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/_schema.yml index 0ce3d1ccc86..bf6e344d674 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/_schema.yml @@ -175,4 +175,134 @@ models: - *token_address - &daily_delta name: daily_delta - description: "Daily total impact on BPT supply" \ No newline at end of file + description: "Daily total impact on BPT supply" + + - name: balancer_v3_gnosis_transfers_bpt + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'transfers'] + description: > + Balancer Pool Token (BPT) transfer logs on Balancer, an automated portfolio manager and trading platform, on gnosis. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - evt_tx_hash + - evt_index + - block_date + columns: + - *blockchain + - *version + - *contract_address + - *block_date + - *block_month + - *evt_tx_hash + - *evt_index + - *evt_block_time + - *evt_block_number + - *from + - *to + - *value + + - name: balancer_v3_gnosis_bpt_prices + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'prices'] + description: > + Balancer Pool Token (BPT) daily price by pool on Balancer, an automated portfolio manager and trading platform. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - day + - contract_address + columns: + - *blockchain + - *day + - *version + - *decimals + - *contract_address + - *bpt_price + + - name: balancer_v3_gnosis_bpt_supply + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'supply'] + description: > + Balancer Pool Token (BPT) supply by pool on Balancer, an automated portfolio manager and trading platform. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - blockchain + - token_address + columns: + - *day + - *pool_type + - *version + - *blockchain + - *token_address + - *supply + + - name: balancer_v3_gnosis_bpt_supply_changes + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'supply', 'changes'] + description: > + Balancer Pool Token (BPT) supply change events. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - evt_tx_hash + - evt_index + - label + columns: + - *block_date + - *evt_block_time + - *evt_block_number + - *blockchain + - *evt_tx_hash + - *evt_index + - *pool_type + - *pool_symbol + - *version + - *label + - *token_address + - *delta_amount_raw + - *delta_amount + + - name: balancer_v3_gnosis_bpt_supply_changes_daily + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'bpt', 'supply', 'changes'] + description: > + Balancer Pool Token (BPT) supply change events. grouped by day + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - blockchain + - token_address + columns: + - *block_date + - *blockchain + - *pool_type + - *pool_symbol + - *version + - *token_address + - *daily_delta \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_prices.sql new file mode 100644 index 00000000000..d35d5d9b55e --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_prices.sql @@ -0,0 +1,21 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'bpt_prices', + materialized = 'table', + file_format = 'delta' + ) +}} + + +{{ + balancer_v3_compatible_bpt_prices_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply.sql new file mode 100644 index 00000000000..612c3fce8c3 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply.sql @@ -0,0 +1,21 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema='balancer_v3_gnosis', + alias = 'bpt_supply', + materialized = 'table', + file_format = 'delta' + + ) +}} + +{{ + balancer_v3_compatible_bpt_supply_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes.sql new file mode 100644 index 00000000000..d95ee336821 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes.sql @@ -0,0 +1,23 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'bpt_supply_changes', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_date', 'evt_tx_hash', 'evt_index', 'label'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +{{ + balancer_v3_compatible_bpt_supply_changes_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes_daily.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes_daily.sql new file mode 100644 index 00000000000..7458210fe1f --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_bpt_supply_changes_daily.sql @@ -0,0 +1,19 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'bpt_supply_changes_daily', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_bpt_supply_changes_daily_agg_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer' + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_transfers_bpt.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_transfers_bpt.sql new file mode 100644 index 00000000000..b61c8780b0f --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/bpt/gnosis/balancer_v3_gnosis_transfers_bpt.sql @@ -0,0 +1,22 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'transfers_bpt', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_date', 'evt_tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +{{ + balancer_v3_compatible_transfers_bpt_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3' + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml new file mode 100644 index 00000000000..6192f2b94b8 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml @@ -0,0 +1,65 @@ +version: 2 + +models: + - name: balancer_v3_erc4626_token_mapping + meta: + blockchain: ethereum, gnosis + project: balancer + contributors: viniabussafi + config: + tags: ['ethereum', 'gnosis', 'static', 'token', 'mapping'] + description: > + Mapping of ERC4626 tokens, including aTokens and MetaMorpho tokens, and their corresponding underlying tokens. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - erc4626_token + columns: + - name: blockchain + description: "Blockchain." + - name: erc4626_token + description: "Contract address of the erc4626 token." + - name: erc4626_token_name + description: "Name of the erc4626 token." + - name: erc4626_token_symbol + description: "Symbol of the erc4626 token." + - name: underlying_token + description: "Contract address of the underlying token." + - name: underlying_token_symbol + description: "Symbol of the underlying token." + - name: token_decimals + description: "Decimal precision of the underlying token." + + - name: balancer_v3_erc4626_token_prices + meta: + blockchain: ethereum, gnosis + project: balancer + contributors: viniabussafi + config: + tags: ['ethereum', 'gnosis', 'static', 'token', 'pricing'] + description: > + Pricing information for ERC4626 tokens. Prices are derived from the ratio of underlying tokens deposited or withdrawn to shares issued or burned, multiplied by the underlying token price. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - minute + - wrapped_token + columns: + - name: minute + description: "Timestamp of the event, truncated to the nearest minute." + - name: blockchain + description: "Blockchain." + - name: wrapped_token + description: "Contract address of the wrapped token." + - name: underlying_token + description: "Contract address of the underlying token." + - name: erc4626_token_symbol + description: "Symbol of the erc4626 token." + - name: underlying_token_symbol + description: "Symbol of the underlying token." + - name: decimals + description: "Decimal precision for the token." + - name: median_price + description: "Median price of the static token, computed over the dataset." + - name: next_change + description: "Timestamp of the next expected pricing change or event affecting the price." \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_mapping.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_mapping.sql new file mode 100644 index 00000000000..00f666a601e --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_mapping.sql @@ -0,0 +1,32 @@ +{{ config( + schema = 'balancer_v3', + alias = 'erc4626_token_mapping', + post_hook='{{ expose_spells(blockchains = \'["ethereum", "gnosis"]\', + spell_type = "project", + spell_name = "balancer", + contributors = \'["viniabussafi"]\') }}' + ) +}} + +{% set balancer_models = [ + ref('balancer_v3_ethereum_erc4626_token_mapping'), + ref('balancer_v3_gnosis_erc4626_token_mapping') +] %} + +SELECT * +FROM ( + {% for model in balancer_models %} + SELECT + blockchain, + erc4626_token, + erc4626_token_name, + erc4626_token_symbol, + underlying_token, + underlying_token_symbol, + decimals + FROM {{ model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql new file mode 100644 index 00000000000..d47484c2ad3 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql @@ -0,0 +1,34 @@ +{{ config( + schema = 'balancer_v3', + alias = 'erc4626_token_prices', + post_hook='{{ expose_spells(blockchains = \'["ethereum", "gnosis"]\', + spell_type = "project", + spell_name = "balancer", + contributors = \'["viniabussafi"]\') }}' + ) +}} + +{% set balancer_models = [ + ref('balancer_v3_ethereum_erc4626_token_prices'), + ref('balancer_v3_gnosis_erc4626_token_prices') +] %} + +SELECT * +FROM ( + {% for model in balancer_models %} + SELECT + minute, + blockchain, + wrapped_token, + underlying_token, + erc4626_token_symbol, + underlying_token_symbol, + decimals, + median_price, + next_change + FROM {{ model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml new file mode 100644 index 00000000000..ae19546932e --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml @@ -0,0 +1,65 @@ +version: 2 + +models: + - name: balancer_v3_ethereum_erc4626_token_mapping + meta: + blockchain: ethereum + project: balancer + contributors: viniabussafi + config: + tags: ['ethereum', 'static', 'token', 'mapping'] + description: > + Mapping of ERC4626 tokens on Ethereum, including aTokens and MetaMorpho tokens, and their corresponding underlying tokens. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - erc4626_token + columns: + - name: blockchain + description: "Blockchain." + - name: erc4626_token + description: "Contract address of the erc4626 token." + - name: erc4626_token_name + description: "Name of the erc4626 token." + - name: erc4626_token_symbol + description: "Symbol of the erc4626 token." + - name: underlying_token + description: "Contract address of the underlying token." + - name: underlying_token_symbol + description: "Symbol of the underlying token." + - name: decimals + description: "Decimal precision of the ERC4626 token." + + - name: balancer_v3_ethereum_erc4626_token_prices + meta: + blockchain: ethereum + project: balancer + contributors: viniabussafi + config: + tags: ['ethereum', 'static', 'token', 'prices'] + description: > + Pricing information for ERC4626 tokens on Ethereum. Prices are derived from the ratio of underlying tokens deposited or withdrawn to shares issued or burned, multiplied by the underlying token price. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - minute + - wrapped_token + columns: + - name: minute + description: "Timestamp of the event, truncated to the nearest minute." + - name: blockchain + description: "Blockchain." + - name: wrapped_token + description: "Contract address of the wrapped token." + - name: underlying_token + description: "Contract address of the underlying token." + - name: erc4626_token_symbol + description: "Symbol of the erc4626 token." + - name: underlying_token_symbol + description: "Symbol of the underlying token." + - name: decimals + description: "Decimal precision for the token." + - name: median_price + description: "Median price of the static token, computed over the dataset." + - name: next_change + description: "Timestamp of the next expected pricing change or event affecting the price." \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql new file mode 100644 index 00000000000..dddfbb06938 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql @@ -0,0 +1,69 @@ +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'erc4626_tokens_mapping', + materialized = 'table', + file_format = 'delta' + ) +}} + +WITH aave_tokens AS( +SELECT + b.staticaToken AS erc4626_token, + a.staticaTokenName AS erc4626_token_name, + a.staticaTokenSymbol AS erc4626_token_symbol, + b.underlying AS underlying_token, + t.symbol AS underlying_token_symbol, + t.decimals AS decimals +FROM {{ source('aave_ethereum', 'StaticATokenLM_evt_Initialized') }} a +JOIN {{ source('aave_ethereum', 'StaticATokenFactory_evt_StaticTokenCreated') }} b +ON b.staticaToken = a.contract_address +JOIN {{ source('aave_v3_ethereum', 'VariableDebtToken_evt_Initialized') }} c +ON a.aToken = c.contract_address +JOIN {{ source('tokens', 'erc20') }} t +ON t.blockchain = 'ethereum' +AND b.underlying = t.contract_address + +UNION + +SELECT + erc4626_token, + erc4626_token_name, + erc4626_token_symbol, + underlying_token, + underlying_token_symbol, + decimals +FROM (VALUES + (0xd4fa2d31b7968e448877f69a96de69f5de8cd23e, 'Static Aave Ethereum USDC', 'WaEthUSDC', 0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48, 'USDC', 6), + (0x7bc3485026ac48b6cf9baf0a377477fff5703af8, 'Static Aave Ethereum USDT', 'WaEthUSDT', 0xdac17f958d2ee523a2206206994597c13d831ec7, 'USDT', 6), + (0x0bfc9d54fc184518a81162f8fb99c2eaca081202, 'Static Aave Ethereum WETH', 'WaEthWETH', 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2, 'WETH', 18), + (0x0fe906e030a44ef24ca8c7dc7b7c53a6c4f00ce9, 'Static Aave Ethereum Lido WETH', 'waEthLidoWETH', 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2, 'WETH', 18), + (0x775f661b0bd1739349b9a2a3ef60be277c5d2d29, 'Static Aave Ethereum Lido wstETH', 'waEthLidowstETH', 0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0, 'wstETH', 18) + ) AS temp_table (erc4626_token, erc4626_token_name, erc4626_token_symbol, underlying_token, underlying_token_symbol, decimals) +), + +morpho_tokens AS( +SELECT DISTINCT + a.metaMorpho AS erc4626_token, + a.name AS erc4626_token_name, + a.symbol AS erc4626_token_symbol, + a.asset AS underlying_token, + t.symbol AS underlying_token_symbol, + 18 AS decimals +FROM {{ source('metamorpho_factory_ethereum', 'MetaMorphoFactory_evt_CreateMetaMorpho') }} a +JOIN {{ source('tokens', 'erc20') }} t +ON t.blockchain = 'ethereum' +AND a.asset = t.contract_address +) + +SELECT + 'ethereum' AS blockchain, + * +FROM aave_tokens + +UNION + +SELECT + 'ethereum' AS blockchain, + * +FROM morpho_tokens \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql new file mode 100644 index 00000000000..99f9c1acac8 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql @@ -0,0 +1,65 @@ +{{ config( + schema = 'balancer_v3_ethereum', + alias = 'erc4626_token_prices', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['minute', 'wrapped_token'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.minute')] + ) +}} + +WITH wrap_unwrap AS( + SELECT + evt_block_time, + wrappedToken, + CAST(depositedUnderlying AS DOUBLE) / CAST(mintedShares AS DOUBLE) AS ratio + FROM {{ source('balancer_v3_ethereum', 'Vault_evt_Wrap') }} + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + + UNION ALL + + SELECT + evt_block_time, + wrappedToken, + CAST(withdrawnUnderlying AS DOUBLE) / CAST(burnedShares AS DOUBLE) AS ratio + FROM {{ source('balancer_v3_ethereum', 'Vault_evt_Unwrap') }} + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + ), + + + price_join AS( + SELECT + w.evt_block_time, + m.underlying_token, + w.wrappedToken, + m.erc4626_token_symbol, + m.underlying_token_symbol, + m.decimals, + ratio * price AS adjusted_price + FROM wrap_unwrap w + JOIN {{ref('balancer_v3_ethereum_erc4626_token_mapping')}} m ON m.erc4626_token = w.wrappedToken + JOIN {{ source('prices', 'usd') }} p ON m.underlying_token = p.contract_address + AND p.blockchain = 'ethereum' + AND DATE_TRUNC('minute', w.evt_block_time) = DATE_TRUNC('minute', p.minute) + {% if is_incremental() %} + AND {{ incremental_predicate('p.minute') }} + {% endif %} + ) + +SELECT + p.evt_block_time AS minute, + 'ethereum' AS blockchain, + wrappedToken AS wrapped_token, + underlying_token, + erc4626_token_symbol, + underlying_token_symbol, + decimals, + APPROX_PERCENTILE(adjusted_price, 0.5) AS median_price, + LEAD(DATE_TRUNC('day', p.evt_block_time), 1, NOW()) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change +FROM price_join p +GROUP BY 1, 2, 3, 4, 5, 6, 7 \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml new file mode 100644 index 00000000000..48cf7ead266 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml @@ -0,0 +1,65 @@ +version: 2 + +models: + - name: balancer_v3_gnosis_erc4626_token_mapping + meta: + blockchain: gnosis + project: balancer + contributors: viniabussafi + config: + tags: ['gnosis', 'static', 'token', 'mapping'] + description: > + Mapping of ERC4626 tokens on gnosis, including aTokens and MetaMorpho tokens, and their corresponding underlying tokens. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - erc4626_token + columns: + - name: blockchain + description: "Blockchain." + - name: erc4626_token + description: "Contract address of the erc4626 token." + - name: erc4626_token_name + description: "Name of the erc4626 token." + - name: erc4626_token_symbol + description: "Symbol of the erc4626 token." + - name: underlying_token + description: "Contract address of the underlying token." + - name: underlying_token_symbol + description: "Symbol of the underlying token." + - name: decimals + description: "Decimal precision of the ERC4626 token." + + - name: balancer_v3_gnosis_erc4626_token_prices + meta: + blockchain: gnosis + project: balancer + contributors: viniabussafi + config: + tags: ['gnosis', 'static', 'token', 'prices'] + description: > + Pricing information for ERC4626 tokens on gnosis. Prices are derived from the ratio of underlying tokens deposited or withdrawn to shares issued or burned, multiplied by the underlying token price. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - minute + - wrapped_token + columns: + - name: minute + description: "Timestamp of the event, truncated to the nearest minute." + - name: blockchain + description: "Blockchain." + - name: wrapped_token + description: "Contract address of the wrapped token." + - name: underlying_token + description: "Contract address of the underlying token." + - name: erc4626_token_symbol + description: "Symbol of the erc4626 token." + - name: underlying_token_symbol + description: "Symbol of the underlying token." + - name: decimals + description: "Decimal precision for the token." + - name: median_price + description: "Median price of the static token, computed over the dataset." + - name: next_change + description: "Timestamp of the next expected pricing change or event affecting the price." \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql new file mode 100644 index 00000000000..697c6133ebb --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql @@ -0,0 +1,30 @@ +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'erc4626_tokens_mapping', + materialized = 'table', + file_format = 'delta' + ) +}} + +WITH aave_tokens AS( + +SELECT + erc4626_token, + erc4626_token_name, + erc4626_token_symbol, + underlying_token, + underlying_token_symbol, + decimals +FROM (VALUES + (0x7c16F0185A26Db0AE7a9377f23BC18ea7ce5d644, 'Static Aave Gnosis GNO', 'WaGnoGNO', 0x9C58BAcC331c9aa871AFD802DB6379a98e80CEdb, 'GNO', 18), + (0x773CDA0CADe2A3d86E6D4e30699d40bB95174ff2, 'Static Aave Gnosis wstETH', 'WaGnowstETH', 0x6c76971f98945ae98dd7d4dfca8711ebea946ea6, 'wstETH', 18), + (0x57f664882F762FA37903FC864e2B633D384B411A, 'Static Aave Gnosis WETH', 'WaGnoWETH', 0x6a023ccd1ff6f2045c3309768ead9e68f978f6e1, 'WETH', 18), + (0x51350d88c1bd32Cc6A79368c9Fb70373Fb71F375, 'Static Aave Gnosis USDC', 'waGnoUSDCe', 0x2a22f9c3b484c3629090FeED35F17Ff8F88f76F0, 'USDC', 6) + ) AS temp_table (erc4626_token, erc4626_token_name, erc4626_token_symbol, underlying_token, underlying_token_symbol, decimals) +) + +SELECT + 'gnosis' AS blockchain, + * +FROM aave_tokens \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql new file mode 100644 index 00000000000..9c19ed2da74 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql @@ -0,0 +1,65 @@ +{{ config( + schema = 'balancer_v3_gnosis', + alias = 'erc4626_token_prices', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['minute', 'wrapped_token'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.minute')] + ) +}} + +WITH wrap_unwrap AS( + SELECT + evt_block_time, + wrappedToken, + CAST(depositedUnderlying AS DOUBLE) / CAST(mintedShares AS DOUBLE) AS ratio + FROM {{ source('balancer_v3_gnosis', 'Vault_evt_Wrap') }} + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + + UNION ALL + + SELECT + evt_block_time, + wrappedToken, + CAST(withdrawnUnderlying AS DOUBLE) / CAST(burnedShares AS DOUBLE) AS ratio + FROM {{ source('balancer_v3_gnosis', 'Vault_evt_Unwrap') }} + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + ), + + + price_join AS( + SELECT + w.evt_block_time, + m.underlying_token, + w.wrappedToken, + m.erc4626_token_symbol, + m.underlying_token_symbol, + m.decimals, + ratio * price AS adjusted_price + FROM wrap_unwrap w + JOIN {{ref('balancer_v3_gnosis_erc4626_token_mapping')}} m ON m.erc4626_token = w.wrappedToken + JOIN {{ source('prices', 'usd') }} p ON m.underlying_token = p.contract_address + AND p.blockchain = 'gnosis' + AND DATE_TRUNC('minute', w.evt_block_time) = DATE_TRUNC('minute', p.minute) + {% if is_incremental() %} + AND {{ incremental_predicate('p.minute') }} + {% endif %} + ) + +SELECT + p.evt_block_time AS minute, + 'gnosis' AS blockchain, + wrappedToken AS wrapped_token, + underlying_token, + erc4626_token_symbol, + underlying_token_symbol, + decimals, + APPROX_PERCENTILE(adjusted_price, 0.5) AS median_price, + LEAD(DATE_TRUNC('day', p.evt_block_time), 1, NOW()) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change +FROM price_join p +GROUP BY 1, 2, 3, 4, 5, 6, 7 \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/balancer_liquidity.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/balancer_liquidity.sql index a278ac524f6..f859bcde449 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/balancer_liquidity.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/balancer_liquidity.sql @@ -20,6 +20,8 @@ ref('balancer_v1_ethereum_liquidity') , ref('balancer_v2_base_liquidity') , ref('balancer_v2_zkevm_liquidity') , ref('balancer_cowswap_amm_liquidity') +, ref('balancer_v3_ethereum_liquidity') +, ref('balancer_v3_gnosis_liquidity') ] %} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/_schema.yml index 1bf694eaf01..0279d42c430 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/_schema.yml @@ -93,4 +93,36 @@ models: - *protocol_liquidity_usd - *protocol_liquidity_eth - *pool_liquidity_usd - - *pool_liquidity_eth \ No newline at end of file + - *pool_liquidity_eth + + - name: balancer_v3_ethereum_liquidity + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'balancer', 'pools', 'liquidity'] + description: > + Balancer v3 pools liquidity by token in Ethereum. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - pool_id + - token_address + columns: + - *day + - *pool_id + - *pool_address + - *pool_symbol + - *version + - *blockchain + - *pool_type + - *token_address + - *token_symbol + - *token_balance_raw + - *token_balance + - *protocol_liquidity_usd + - *protocol_liquidity_eth + - *pool_liquidity_usd + - *pool_liquidity_eth \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/balancer_v3_ethereum_liquidity.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/balancer_v3_ethereum_liquidity.sql new file mode 100644 index 00000000000..b47682074b2 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/ethereum/balancer_v3_ethereum_liquidity.sql @@ -0,0 +1,22 @@ + +{% set blockchain = 'ethereum' %} + +{{ + config( + schema = 'balancer_v3_ethereum', + alias = 'liquidity', + materialized = 'table', + file_format = 'delta' + ) +}} + + +{{ + balancer_v3_compatible_liquidity_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/_schema.yml index f03929d48b7..2441ab284f8 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/_schema.yml @@ -9,7 +9,7 @@ models: config: tags: ['gnosis', 'balancer', 'pools', 'liquidity'] description: > - Balancer v2 pools liquidity by token in Arbitrum. + Balancer v2 pools liquidity by token in Gnosis. data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -44,15 +44,53 @@ models: - &token_symbol name: token_symbol description: "Token symbol" - - name: token_balance_raw + - &token_balance_raw + name: token_balance_raw description: 'Raw balance of the token in the pool in the original currency' - - name: token_balance + - &token_balance + name: token_balance description: 'Scaled balance of the token in the pool in the original currency' - - name: protocol_liquidity_usd + - &protocol_liquidity_usd + name: protocol_liquidity_usd description: 'Liquidity of the token in the pool in USD, except BPTs' - - name: protocol_liquidity_eth + - &protocol_liquidity_eth + name: protocol_liquidity_eth description: 'Liquidity of the token in the pool in ETH, except BPTs' - - name: pool_liquidity_usd + - &pool_liquidity_usd + name: pool_liquidity_usd description: 'Liquidity of the token in the pool in USD, including BPTs' - - name: pool_liquidity_eth - description: 'Liquidity of the token in the pool in ETH, including BPTs' \ No newline at end of file + - &pool_liquidity_eth + name: pool_liquidity_eth + description: 'Liquidity of the token in the pool in ETH, including BPTs' + + - name: balancer_v3_gnosis_liquidity + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'balancer', 'pools', 'liquidity'] + description: > + Balancer v3 pools liquidity by token in Gnosis. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - pool_id + - token_address + columns: + - *day + - *pool_id + - *pool_address + - *pool_symbol + - *version + - *blockchain + - *pool_type + - *token_address + - *token_symbol + - *token_balance_raw + - *token_balance + - *protocol_liquidity_usd + - *protocol_liquidity_eth + - *pool_liquidity_usd + - *pool_liquidity_eth \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/balancer_v3_gnosis_liquidity.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/balancer_v3_gnosis_liquidity.sql new file mode 100644 index 00000000000..8ab09e843dd --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/liquidity/gnosis/balancer_v3_gnosis_liquidity.sql @@ -0,0 +1,21 @@ + +{% set blockchain = 'gnosis' %} + +{{ + config( + schema = 'balancer_v3_gnosis', + alias = 'liquidity', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_liquidity_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/balancer_pools_tokens_weights.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/balancer_pools_tokens_weights.sql index d2e50c49ce9..d57a4b0de4a 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/balancer_pools_tokens_weights.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/balancer_pools_tokens_weights.sql @@ -9,14 +9,14 @@ }} {% set balancer_models = [ - ref('balancer_v2_arbitrum_pools_tokens_weights'), - ref('balancer_v2_avalanche_c_pools_tokens_weights'), - ref('balancer_v2_base_pools_tokens_weights'), - ref('balancer_v2_ethereum_pools_tokens_weights'), - ref('balancer_v2_gnosis_pools_tokens_weights'), - ref('balancer_v2_optimism_pools_tokens_weights'), - ref('balancer_v2_polygon_pools_tokens_weights'), - ref('balancer_v2_zkevm_pools_tokens_weights') + ref('balancer_arbitrum_pools_tokens_weights'), + ref('balancer_avalanche_c_pools_tokens_weights'), + ref('balancer_base_pools_tokens_weights'), + ref('balancer_ethereum_pools_tokens_weights'), + ref('balancer_gnosis_pools_tokens_weights'), + ref('balancer_optimism_pools_tokens_weights'), + ref('balancer_polygon_pools_tokens_weights'), + ref('balancer_zkevm_pools_tokens_weights') ] %} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/_schema.yml index 60856c24380..01d6a2e4146 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/_schema.yml @@ -92,4 +92,25 @@ models: - name: start_time description: "LBP start time" - name: end_time - description: "LBP end time" \ No newline at end of file + description: "LBP end time" + + - name: balancer_v3_ethereum_pools_tokens_weights + meta: + blockchain: ethereum + project: balancer + contributors: viniabussafi + config: + tags: ['ethereum', 'balancer', 'pools', 'tokens', 'weights'] + description: > + Token weights in Balancer v3 pools. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - pool_id + - token_address + columns: + - *blockchain + - *version + - *pool_id + - *token_address + - *normalized_weight \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_ethereum_pools_tokens_weights.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_ethereum_pools_tokens_weights.sql index 0396dc86424..7f8285ce1d9 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_ethereum_pools_tokens_weights.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_ethereum_pools_tokens_weights.sql @@ -4,23 +4,26 @@ ) }} - + +{% set balancer_models = [ + ref('balancer_v1_ethereum_pools_tokens_weights'), + ref('balancer_v2_ethereum_pools_tokens_weights'), + ref('balancer_v3_ethereum_pools_tokens_weights') +] %} + SELECT * -FROM -( - SELECT - blockchain, - version, - pool_id, - token_address, - CAST(normalized_weight as double) as normalized_weight - FROM {{ ref('balancer_v1_ethereum_pools_tokens_weights') }} - UNION - SELECT - blockchain, - version, - pool_id, - token_address, - CAST(normalized_weight as double) as normalized_weight - FROM {{ ref('balancer_v2_ethereum_pools_tokens_weights') }} +FROM ( + {% for model in balancer_models %} + SELECT + blockchain, + version, + pool_id, + token_address, + normalized_weight + FROM {{ model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} ) +; diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_v3_ethereum_pools_tokens_weights.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_v3_ethereum_pools_tokens_weights.sql new file mode 100644 index 00000000000..7c442c53d95 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/ethereum/balancer_v3_ethereum_pools_tokens_weights.sql @@ -0,0 +1,61 @@ +{{ + config( + schema='balancer_v3_ethereum', + alias = 'pools_tokens_weights', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['pool_id', 'token_address'] + ) +}} + +-- +-- Balancer v3 Pools Tokens Weights +-- +WITH token_data AS ( + SELECT + pool, + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source('balancer_v3_ethereum', 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + +weighted_pool_factory AS ( + SELECT + call_create.output_pool AS pool_id, + t.pos AS pos, + t.token_address AS token_address, + t2.normalized_weight AS normalized_weight + FROM {{ source('balancer_v3_ethereum', 'WeightedPoolFactory_call_create') }} AS call_create + JOIN token_data td ON td.pool = call_create.output_pool + CROSS JOIN UNNEST(td.tokens) WITH ORDINALITY t(token_address, pos) + CROSS JOIN UNNEST(call_create.normalizedWeights) WITH ORDINALITY t2(normalized_weight, pos) + WHERE t.pos = t2.pos + {% if is_incremental() %} + AND call_create.call_block_time >= date_trunc('day', now() - interval '7' day) + {% endif %} +), + +normalized_weights AS ( + SELECT + pool_id, + token_address, + normalized_weight / POWER(10, 18) AS normalized_weight + FROM weighted_pool_factory +) + +SELECT + 'ethereum' AS blockchain, + '3' AS version, + w.pool_id, + w.token_address, + w.normalized_weight +FROM normalized_weights w +WHERE w.pool_id IS NOT NULL diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/_schema.yml index 053513dd4ea..c0a72b7781c 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/_schema.yml @@ -48,4 +48,25 @@ models: - *version - *pool_id - *token_address - - *normalized_weight \ No newline at end of file + - *normalized_weight + + - name: balancer_v3_gnosis_pools_tokens_weights + meta: + blockchain: gnosis + project: balancer + contributors: viniabussafi + config: + tags: ['gnosis', 'balancer', 'pools', 'tokens', 'weights'] + description: > + Token weights in Balancer v3 pools. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - pool_id + - token_address + columns: + - *blockchain + - *version + - *pool_id + - *token_address + - *normalized_weight \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_gnosis_pools_tokens_weights.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_gnosis_pools_tokens_weights.sql index 4682ebab892..b0773310c4f 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_gnosis_pools_tokens_weights.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_gnosis_pools_tokens_weights.sql @@ -5,14 +5,25 @@ ) }} +{% set balancer_models = [ + ref('balancer_v2_gnosis_pools_tokens_weights'), + ref('balancer_v3_gnosis_pools_tokens_weights') +] %} + SELECT * -FROM -( - SELECT - blockchain, - version, - pool_id, - token_address, - normalized_weight - FROM {{ ref('balancer_v2_gnosis_pools_tokens_weights') }} -) \ No newline at end of file +FROM ( + {% for model in balancer_models %} + SELECT + blockchain, + version, + pool_id, + token_address, + normalized_weight + FROM {{ model }} + WHERE pool_id IS NOT NULL + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) +; diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_v3_gnosis_pools_tokens_weights.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_v3_gnosis_pools_tokens_weights.sql new file mode 100644 index 00000000000..fa93d6184ce --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/pools/gnosis/balancer_v3_gnosis_pools_tokens_weights.sql @@ -0,0 +1,61 @@ +{{ + config( + schema='balancer_v3_gnosis', + alias = 'pools_tokens_weights', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['pool_id', 'token_address'] + ) +}} + +-- +-- Balancer v3 Pools Tokens Weights +-- +WITH token_data AS ( + SELECT + pool, + ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens + FROM ( + SELECT + pool, + tokenConfig, + SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array + FROM {{ source('balancer_v3_gnosis', 'Vault_evt_PoolRegistered') }} + ) AS pool_data + CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index) + GROUP BY 1 + ), + +weighted_pool_factory AS ( + SELECT + call_create.output_pool AS pool_id, + t.pos AS pos, + t.token_address AS token_address, + t2.normalized_weight AS normalized_weight + FROM {{ source('balancer_v3_gnosis', 'WeightedPoolFactory_call_create') }} AS call_create + JOIN token_data td ON td.pool = call_create.output_pool + CROSS JOIN UNNEST(td.tokens) WITH ORDINALITY t(token_address, pos) + CROSS JOIN UNNEST(call_create.normalizedWeights) WITH ORDINALITY t2(normalized_weight, pos) + WHERE t.pos = t2.pos + {% if is_incremental() %} + AND call_create.call_block_time >= date_trunc('day', now() - interval '7' day) + {% endif %} +), + +normalized_weights AS ( + SELECT + pool_id, + token_address, + normalized_weight / POWER(10, 18) AS normalized_weight + FROM weighted_pool_factory +) + +SELECT + 'gnosis' AS blockchain, + '3' AS version, + w.pool_id, + w.token_address, + w.normalized_weight +FROM normalized_weights w +WHERE w.pool_id IS NOT NULL diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/_schema.yml index d4a08fa601a..aab385185a8 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/_schema.yml @@ -37,7 +37,10 @@ models: description: "Blockchain" - &pool_type name: pool_type - description: "Pool attributes, determined by the pool's factory" + description: "Pool attributes, determined by the pool's factory" + - &fee_type + name: fee_type + description: "Labels whether fee comes from swaps or yield for V3. V2 fees are labeled as 'v2'." - &token_address name: token_address description: "Contract address of the token in which protocol fees were collected" diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/balancer_protocol_fee.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/balancer_protocol_fee.sql index 37fef24dc85..d0cf6d8445c 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/balancer_protocol_fee.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/balancer_protocol_fee.sql @@ -8,7 +8,7 @@ ) }} -{% set balancer_v2_models = [ +{% set balancer_models = [ ref('balancer_v2_arbitrum_protocol_fee'), ref('balancer_v2_avalanche_c_protocol_fee'), ref('balancer_v2_base_protocol_fee'), @@ -17,19 +17,22 @@ ref('balancer_v2_optimism_protocol_fee'), ref('balancer_v2_polygon_protocol_fee'), ref('balancer_v2_zkevm_protocol_fee'), + ref('balancer_v3_ethereum_protocol_fee'), + ref('balancer_v3_gnosis_protocol_fee') ] %} SELECT * FROM ( - {% for protocol_fee in balancer_v2_models %} + {% for protocol_fee in balancer_models %} SELECT day, pool_id, pool_address, pool_symbol, - '2' AS version, + version, blockchain, pool_type, + fee_type, token_address, token_symbol, token_amount_raw, diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/_schema.yml index 2de1a5b874b..edc8318cf02 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/_schema.yml @@ -37,7 +37,10 @@ models: description: "Blockchain" - &pool_type name: pool_type - description: "Pool attributes, determined by the pool's factory" + description: "Pool attributes, determined by the pool's factory" + - &fee_type + name: fee_type + description: "Labels whether fee comes from swaps or yield for V3. V2 fees are labeled as 'v2'." - &token_address name: token_address description: "Contract address of the token in which protocol fees were collected" @@ -58,4 +61,37 @@ models: description: "Share of total fee that is directed to Balancer DAO's treasury, as per BIPs 19, 161 and 371" - &treasury_revenue_usd name: treasury_revenue_usd - description: "USD value of fee directed to Balancer DAO's treasury" \ No newline at end of file + description: "USD value of fee directed to Balancer DAO's treasury" + + - name: balancer_v3_ethereum_protocol_fee + meta: + blockchain: ethereum + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['ethereum', 'revenue', 'fees'] + description: > + Daily Protocol Fee collected and Revenue by pool on Balancer, an automated portfolio manager and trading platform. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - pool_id + - fee_type + - token_address + columns: + - *day + - *pool_id + - *pool_address + - *pool_symbol + - *version + - *blockchain + - *pool_type + - *fee_type + - *token_address + - *token_symbol + - *token_amount_raw + - *token_amount + - *protocol_fee_collected_usd + - *treasury_share + - *treasury_revenue_usd \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/balancer_v3_ethereum_protocol_fee.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/balancer_v3_ethereum_protocol_fee.sql new file mode 100644 index 00000000000..dafed52013e --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/ethereum/balancer_v3_ethereum_protocol_fee.sql @@ -0,0 +1,20 @@ +{% set blockchain = 'ethereum' %} + +{{ + config( + schema='balancer_v3_ethereum', + alias = 'protocol_fee', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_protocol_fee_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/_schema.yml index 3b1fa8ffdbe..51e384bdd3a 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/_schema.yml @@ -38,6 +38,9 @@ models: - &pool_type name: pool_type description: "Pool attributes, determined by the pool's factory" + - &fee_type + name: fee_type + description: "Labels whether fee comes from swaps or yield for V3. V2 fees are labeled as 'v2'." - &token_address name: token_address description: "Contract address of the token in which protocol fees were collected" @@ -58,4 +61,37 @@ models: description: "Share of total fee that is directed to Balancer DAO's treasury, as per BIPs 19, 161 and 371" - &treasury_revenue_usd name: treasury_revenue_usd - description: "USD value of fee directed to Balancer DAO's treasury" \ No newline at end of file + description: "USD value of fee directed to Balancer DAO's treasury" + + - name: balancer_v3_gnosis_protocol_fee + meta: + blockchain: gnosis + project: balancer_v3 + contributors: viniabussafi + config: + tags: ['gnosis', 'revenue', 'fees'] + description: > + Daily Protocol Fee collected and Revenue by pool on Balancer, an automated portfolio manager and trading platform. + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - pool_id + - fee_type + - token_address + columns: + - *day + - *pool_id + - *pool_address + - *pool_symbol + - *version + - *blockchain + - *pool_type + - *fee_type + - *token_address + - *token_symbol + - *token_amount_raw + - *token_amount + - *protocol_fee_collected_usd + - *treasury_share + - *treasury_revenue_usd \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/balancer_v3_gnosis_protocol_fee.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/balancer_v3_gnosis_protocol_fee.sql new file mode 100644 index 00000000000..0630e6eec63 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/protocol_fee/gnosis/balancer_v3_gnosis_protocol_fee.sql @@ -0,0 +1,20 @@ +{% set blockchain = 'gnosis' %} + +{{ + config( + schema='balancer_v3_gnosis', + alias = 'protocol_fee', + materialized = 'table', + file_format = 'delta' + ) +}} + +{{ + balancer_v3_compatible_protocol_fee_macro( + blockchain = blockchain, + version = '3', + project_decoded_as = 'balancer_v3', + base_spells_namespace = 'balancer', + pool_labels_spell = source('labels', 'balancer_v3_pools') + ) +}} diff --git a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql index 3cdd312ce0e..141f98c1e86 100644 --- a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql @@ -4725,6 +4725,14 @@ FROM (VALUES , ( 0x9994E35Db50125E0DF82e4c2dde62496CE330999, 'legacyMORPHO', 18) , ( 0xc824a08db624942c5e5f330d56530cd1598859fd, 'hgETH', 18) , ( 0x6e6B7ADfC7Db9fEeB8896418aC3422966f65D0A5, 'NET', 18) + , ( 0xd4fa2d31b7968e448877f69a96de69f5de8cd23e, 'WaEthUSDC', 6) + , ( 0x7bc3485026ac48b6cf9baf0a377477fff5703af8, 'WaEthUSDT', 6) + , ( 0x0bfc9d54fc184518a81162f8fb99c2eaca081202, 'WaEthWETH', 18) + , ( 0x0fe906e030a44ef24ca8c7dc7b7c53a6c4f00ce9, 'waEthLidoWETH', 18) + , ( 0x775f661b0bd1739349b9a2a3ef60be277c5d2d29, 'waEthLidowstETH', 18) + , ( 0xbEEFC01767ed5086f35deCb6C00e6C12bc7476C1, 'csUSDL', 18) + , ( 0xbeef01735c132ada46aa9aa4c54623caa92a64cb, 'steakUSDC', 18) + , ( 0xbdC7c08592Ee4aa51D06C27Ee23D5087D65aDbcD, 'USDL', 18) , ( 0x358d94b5b2F147D741088803d932Acb566acB7B6, 'rSWELL', 18) , ( 0x9Ed15383940CC380fAEF0a75edacE507cC775f22, 'earnETH', 18) , ( 0x66E47E6957B85Cf62564610B76dD206BB04d831a, 'earnBTC', 8) diff --git a/dbt_subprojects/tokens/models/tokens/gnosis/tokens_gnosis_erc20.sql b/dbt_subprojects/tokens/models/tokens/gnosis/tokens_gnosis_erc20.sql index 50bb05cede7..19ca6d3d2d7 100644 --- a/dbt_subprojects/tokens/models/tokens/gnosis/tokens_gnosis_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/gnosis/tokens_gnosis_erc20.sql @@ -7821,4 +7821,8 @@ FROM (VALUES , ( 0xecfd0638175e291ba3f784a58fb9d38a25418904, 'stataGnowstETH', 18) , ( 0xf0e7ec247b918311afa054e0aedb99d74c31b809, 'stataGnoUSDCe', 6) , ( 0xFECB3F7c54E2CAAE9dC6Ac9060A822D47E053760, 'BRLA', 18) + , ( 0x7c16f0185a26db0ae7a9377f23bc18ea7ce5d644, 'waGnoGNO', 18) + , ( 0x773cda0cade2a3d86e6d4e30699d40bb95174ff2, 'waGnowstETH', 18) + , ( 0x57f664882f762fa37903fc864e2b633d384b411a, 'waGnoWETH', 18) + , ( 0x51350d88c1bd32Cc6A79368c9Fb70373Fb71F375, 'waGNOUSDCe', 6) ) AS temp_table (contract_address, symbol, decimals) diff --git a/sources/_subprojects_outputs/dex/_sources.yml b/sources/_subprojects_outputs/dex/_sources.yml index e590eaae00d..82f13efac55 100644 --- a/sources/_subprojects_outputs/dex/_sources.yml +++ b/sources/_subprojects_outputs/dex/_sources.yml @@ -40,6 +40,12 @@ sources: - name: balancer_v2_zkevm tables: - name: trades + - name: balancer_v3_ethereum + tables: + - name: trades + - name: balancer_v3_gnosis + tables: + - name: trades - name: balancer tables: - name: trades diff --git a/sources/_subprojects_outputs/spellbook/_sources.yml b/sources/_subprojects_outputs/spellbook/_sources.yml index 6929b1a6430..65c11b935de 100644 --- a/sources/_subprojects_outputs/spellbook/_sources.yml +++ b/sources/_subprojects_outputs/spellbook/_sources.yml @@ -110,6 +110,12 @@ sources: - name: balancer tables: - name: bpt_prices + - name: balancer_v3_ethereum + tables: + - name: bpt_prices + - name: balancer_v3_gnosis + tables: + - name: bpt_prices - name: addresses_ethereum tables: diff --git a/sources/aave/ethereum/aave_ethereum_sources.yml b/sources/aave/ethereum/aave_ethereum_sources.yml index 28958196631..77bc29c1866 100644 --- a/sources/aave/ethereum/aave_ethereum_sources.yml +++ b/sources/aave/ethereum/aave_ethereum_sources.yml @@ -118,9 +118,9 @@ sources: - *evt_tx_hash - *id - - name: erc4626TokenLM_evt_Initialized + - name: StaticATokenLM_evt_Initialized - - name: erc4626TokenFactory_evt_StaticTokenCreated + - name: StaticATokenFactory_evt_StaticTokenCreated - name: aave_v3_ethereum description: "Decoded contracts for Aave v3 on Ethereum" diff --git a/sources/aave/gnosis/aave_gnosis_sources.yml b/sources/aave/gnosis/aave_gnosis_sources.yml index 9d44db97d25..f5e9101db8c 100644 --- a/sources/aave/gnosis/aave_gnosis_sources.yml +++ b/sources/aave/gnosis/aave_gnosis_sources.yml @@ -4,4 +4,4 @@ sources: - name: aave_v3_gnosis description: "Decoded contracts for Aave v3 on Gnosis" tables: - - name: AToken_evt_Initialized \ No newline at end of file + - name: AToken_evt_Initialized diff --git a/sources/balancer/ethereum/balancer_ethereum_sources.yml b/sources/balancer/ethereum/balancer_ethereum_sources.yml index 256f89463c4..6cf62852fc6 100644 --- a/sources/balancer/ethereum/balancer_ethereum_sources.yml +++ b/sources/balancer/ethereum/balancer_ethereum_sources.yml @@ -352,7 +352,7 @@ sources: description: "veBAL holder that voted for the gauge" - name: weight description: "Weight of the user for the gauge" - + - name: balancer_v3_ethereum description: > Decoded tables related to Balancer V3, an automated portfolio manager and trading platform, on Ethereum. @@ -385,4 +385,4 @@ sources: - name: Vault_evt_LiquidityRemovedFromBuffer - - name: Vault_evt_SwapFeePercentageChanged \ No newline at end of file + - name: Vault_evt_SwapFeePercentageChanged diff --git a/sources/balancer/gnosis/balancer_gnosis_sources.yml b/sources/balancer/gnosis/balancer_gnosis_sources.yml index af033124ccb..b4afef0be0d 100644 --- a/sources/balancer/gnosis/balancer_gnosis_sources.yml +++ b/sources/balancer/gnosis/balancer_gnosis_sources.yml @@ -277,4 +277,4 @@ sources: - name: Vault_evt_LiquidityRemovedFromBuffer - - name: Vault_evt_SwapFeePercentageChanged \ No newline at end of file + - name: Vault_evt_SwapFeePercentageChanged From 52fe09792041e5e9102841ac04a887b0a2390fba Mon Sep 17 00:00:00 2001 From: hildobby Date: Thu, 12 Dec 2024 17:27:48 +0100 Subject: [PATCH 28/59] add invalid pubkeys (#7299) --- .../staking/ethereum/staking_ethereum_flows.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dbt_subprojects/hourly_spellbook/models/_sector/staking/ethereum/staking_ethereum_flows.sql b/dbt_subprojects/hourly_spellbook/models/_sector/staking/ethereum/staking_ethereum_flows.sql index db36a29b6a5..dd6b3a7f087 100644 --- a/dbt_subprojects/hourly_spellbook/models/_sector/staking/ethereum/staking_ethereum_flows.sql +++ b/dbt_subprojects/hourly_spellbook/models/_sector/staking/ethereum/staking_ethereum_flows.sql @@ -18,6 +18,19 @@ WITH invalid_pubkeys AS ( , (0x86f473a006c566f1648a82c74cdfbd4a3cb2ea04eb2e0d49ef381ab2562576888554ef3d39e56996f24c804abb489600) , (0x8c69edd7a8e8da5330787952a1ad5075516e6fd4bda1586d62dd64701f7628d5229eb7f929017dea9ae6995f9c69ef5e) , (0x80a29e569e8ced0be1fff42c845a59449aecf8a2503542e4e76763ccc0265e683e2d5d46618cc829349293ed08ff49ff) + , (0xb913a27913c8a74c05cc31c1a690ecad05a59e405bd7c5e8f6eab8e426e041a98b26cf40b04356b4d92ac20a56b7dcae) + , (0x86af099d9134b2994b835cced1fadfb4587dddfc4010470db9d8875cffd9e5a62a197db7d9c0266fb67c5175feb7ef51) + , (0x8107543db1d5c69be127b3eb84c0f7b8157b892482f7e98b85b83d9a6be75e7c24a645df6283d46b16285862530cad77) + , (0x8e05d99c557001d06f8240d46899b829a8cc77ac57b3d16359279cad707cfe5f223a3374987ab73a379ee358dd05d524) + , (0xaebf3fbab24f55df829e2bc939bb987cbdb3edea7d4cb8877e422f1185d03a22f3a0f6449a1d6d1b912ec09ed11f2bb3) + , (0xac3a0887866d5d45555904e2cb35e1b89b4c338c19001b0cc1184c9f95c5a731ccef70dc4c4fed7709c2106042a119c9) + , (0xa03840dd6af6555442e3fc0d62de8dde77970f45175ea9926327372b5c83542f67cdd06e14e1daa44a3ae23e4d8eef52) + , (0x828116d0d2e945f1483ec7c6c135a8e00814588879e9a12a67c42268c339388b6f796f6c858e673f6000f5d028b913da) + , (0x816827749a5194cf8389419e88d87f6786436daaa5546b92c68af015f0b7e17c66f4bb30b18872f3f051c2bc213ecaab) + , (0xa0ab932b24d80a7a96f0fb32ce2aa724625eb090c70cb9c977f0bc5909503629155335be021e58b245e11a77c847a11c) + , (0x8d135f9185f635be5e3d738c835a02d5efab05bc5fa38ce4cb6d02156446aa3cc1ce9cc38576ec519af815dc39d52c81) + , (0x8151e62f956cf1562007d9620fd4e91c029fb43959d1a7d1d2168c2943d65a3ef31764d1cc2d2540fb26ce86efad2ffd) + , (0x9714e943c81d802f3c858f284fff25779818a903c034a3de42da7a2b63ae6632c52b2be0982007e8090d0d334f8cf656) --, (0x00) -- This is a dummy pubkey used to refresh the table ) AS temp_table (pubkey) ) From e093dd3b9ed65c2f330683bf9dcc4f51fe9cce81 Mon Sep 17 00:00:00 2001 From: whale_hunter <143016036+whalehunting@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:29:55 +0100 Subject: [PATCH 29/59] feat: add prophetbots solana bot trades spellbook (#7285) * feat: add prophetbots solana bot trades spellbook * fix: sum balance change * chore: fix seed --- .../_sector/dex/bot_trades/solana/_schema.yml | 22 +++ .../solana/dex_solana_bot_trades.sql | 2 + .../prophetbots_solana_bot_trades.sql | 154 ++++++++++++++++++ .../prophetbots_solana_trades_seed.csv | 21 +++ 4 files changed, 199 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/prophetbots_solana_bot_trades.sql create mode 100644 dbt_subprojects/solana/seeds/prophetbots/prophetbots_solana_trades_seed.csv diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml index a48042380b2..34b9e61badd 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml @@ -688,3 +688,25 @@ models: - check_bot_trades_seed: seed_file: ref('tradewiz_solana_trades_seed') blockchain: solana + + - name: prophetbots_solana_bot_trades + meta: + blockchain: solana + sector: dex + project: prophetbots + contributors: whale_hunter + config: + tags: ["solana", "dex", "prophetbots", "trades"] + description: > + ProphetBots trades on Solana + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_id + - tx_index + - outer_instruction_index + - inner_instruction_index + - check_bot_trades_seed: + seed_file: ref('prophetbots_solana_trades_seed') + blockchain: solana diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql index 3a46f240904..1432bf2fbb6 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql @@ -36,6 +36,8 @@ , ref('wifbot_solana_bot_trades') , ref('autosnipe_solana_bot_trades') , ref('bitfoot_solana_bot_trades') + , ref('tradewiz_solana_bot_trades') + , ref('prophetbots_solana_bot_trades') ] %} {% for bot in solana_trading_bot %} diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/prophetbots_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/prophetbots_solana_bot_trades.sql new file mode 100644 index 00000000000..c52feb51754 --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/prophetbots_solana_bot_trades.sql @@ -0,0 +1,154 @@ +{{ config( + alias = 'bot_trades', + schema = 'prophetbots_solana', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] + ) +}} + +{% set project_start_date = '2024-12-02' %} +{% set fee_receiver_1 = '55vkTc7nZoUQM92AfQG7T8bkNKD4TbWeBPRg8KjyUZre' %} -- TeamFeeWallet +{% set fee_receiver_2 = 'Hgckz7Sv8Q5grhLXxDFXGaJD6StPE7Yu8gz611nn1wKS' %} -- RevshareFeeWallet +{% set wsol_token = 'So11111111111111111111111111111111111111112' %} + +WITH + allFeePayments AS ( + SELECT + tx_id, + 'SOL' AS feeTokenType, + sum(balance_change) / 1e9 AS fee_token_amount, + '{{wsol_token}}' AS fee_token_mint_address + FROM + {{ source('solana','account_activity') }} + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND tx_success + AND balance_change > 0 + AND (address = '{{fee_receiver_1}}' OR address = '{{fee_receiver_2}}') + GROUP BY tx_id + ), + botTrades AS ( + SELECT + trades.block_time, + CAST(date_trunc('day', trades.block_time) AS date) AS block_date, + CAST(date_trunc('month', trades.block_time) AS date) AS block_month, + 'solana' AS blockchain, + amount_usd, + IF( + token_sold_mint_address = '{{wsol_token}}', + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_mint_address AS token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_mint_address AS token_sold_address, + fee_token_amount * price AS fee_usd, + fee_token_amount, + IF(feeTokenType = 'SOL', 'SOL', symbol) AS fee_token_symbol, + fee_token_mint_address AS fee_token_address, + project, + version, + token_pair, + project_program_id AS project_contract_address, + trader_id AS user, + trades.tx_id, + tx_index, + outer_instruction_index, + inner_instruction_index + FROM + {{ ref('dex_solana_trades') }} AS trades + JOIN allFeePayments AS feePayments ON trades.tx_id = feePayments.tx_id + LEFT JOIN {{ source('prices', 'usd') }} AS feeTokenPrices ON ( + feeTokenPrices.blockchain = 'solana' + AND fee_token_mint_address = toBase58 (feeTokenPrices.contract_address) + AND date_trunc('minute', block_time) = minute + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + JOIN {{ source('solana','transactions') }} AS transactions ON ( + trades.tx_id = id + {% if is_incremental() %} + AND {{ incremental_predicate('transactions.block_time') }} + {% else %} + AND transactions.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + WHERE + trades.trader_id != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND trades.trader_id != '{{fee_receiver_2}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_2}}' -- Exclude trades signed by FeeWallet + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestInnerInstructionIndexForEachTrade AS ( + SELECT + tx_id, + outer_instruction_index, + MAX(inner_instruction_index) AS highestInnerInstructionIndex + FROM + botTrades + GROUP BY + tx_id, + outer_instruction_index + ) +SELECT + block_time, + block_date, + block_month, + 'ProphetBots' as bot, + blockchain, + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + project, + version, + token_pair, + project_contract_address, + user, + botTrades.tx_id, + tx_index, + botTrades.outer_instruction_index, + COALESCE(inner_instruction_index, 0) AS inner_instruction_index, + IF( + inner_instruction_index = highestInnerInstructionIndex, + true, + false + ) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestInnerInstructionIndexForEachTrade ON ( + botTrades.tx_id = highestInnerInstructionIndexForEachTrade.tx_id + AND botTrades.outer_instruction_index = highestInnerInstructionIndexForEachTrade.outer_instruction_index + ) +ORDER BY + block_time DESC, + tx_index DESC, + outer_instruction_index DESC, + inner_instruction_index DESC diff --git a/dbt_subprojects/solana/seeds/prophetbots/prophetbots_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/prophetbots/prophetbots_solana_trades_seed.csv new file mode 100644 index 00000000000..a197c0d8b1c --- /dev/null +++ b/dbt_subprojects/solana/seeds/prophetbots/prophetbots_solana_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction +2024-12-05 16:53:54.000 UTC,2024-12-05,2024-12-01,ProphetBots,solana,11.381475,Buy,208793.948484,Luma,ATnB3fxu6HGjieMtQYJgK4kVVyyZVdVXFigF67pLpump,0.0475,SOL,So11111111111111111111111111111111111111112,0.599025,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Luma,4V1fmQBuj5Zy9qkr5sBqip4UXyirfzVybWgqMGf4hYgB,BxqQCDXfuUJaCw51cQmzXpfYSAt7vKSyHR1WeGtxMfBD,5KYz3YEmBxtqqHrqLvQcrSERc6XDzVntNomZSkmFGwmQ419Jo9L8DHKwxgmWdmpSMLBMhu9XC3CH4z3Bxyvdx1aJ,669,8,0,true +2024-12-05 17:07:09.000 UTC,2024-12-05,2024-12-01,ProphetBots,solana,10.77747650966,Sell,0.045045041,SOL,So11111111111111111111111111111111111111112,533.792777,,ZK6mVzkrj9uqEfyFnCwYCLaW1SJkpdevAujqrpkpump,0.5388735742599999,0.002252251,SOL,So11111111111111111111111111111111111111112,raydium,4,,26e6nfrCFsG2z2i2iePrF2bd2M9SiTKUfPhc5PjRfo85,7dCS2PCbi6xxEcd2UP4WX1vJHtkXuBCRT1PRLVkEJGqR,2VnjsoeVPSVAXpp9KatV7SxEx8apm156x7jupbDH6uMjkRU3jZqFVFsXFekXdXUCFxW1j4ABghWGVAWyz9pUWmD8,127,3,0,true +2024-12-03 16:28:35.000 UTC,2024-12-03,2024-12-01,ProphetBots,solana,10.603900000000001,Buy,134166.826871,빌리,D2HR3Hs8eyD1uj8WKpxw1qwpBchPLMUxwC6PVv3Rpump,0.0475,SOL,So11111111111111111111111111111111111111112,0.5581,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,빌리-SOL,DGY9fRjKYnbRqjyRXcHugjqhv7wScdtt2umsrbU2H3AH,FpzkRxf5DZroddaQUQDDhHuEaksAnyQ4foWEYgLorGUL,3WYyKHY9HEqZxeVDmKC3tw6UiaDNmQZmuE7riX3F1ndzBNn1HfusUBQkTZDhR1zRYYMHD1nSfBSTQop67uofZd9C,415,8,0,true +2024-12-06 15:10:45.000 UTC,2024-12-06,2024-12-01,ProphetBots,solana,11.268424999999999,Buy,129045.650649,KENTA,GupH283BAzrWbYH5WfXpm1DxrDuacDaANWjWzeKjpump,0.0475,SOL,So11111111111111111111111111111111111111112,0.593075,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-KENTA,FG7UJVJyuybqGbss85BvjVnsN8iUjyuWdz9Jn1a6ptbz,7dCS2PCbi6xxEcd2UP4WX1vJHtkXuBCRT1PRLVkEJGqR,5Nqmh81Mf55wqFt8WeiMPmLotbyvjNWCzjuxCU5gdpfuHtVetUjE3rzSi96jbeVHozRvTjtbL8k9aehyATXeH78E,1422,8,0,true +2024-12-09 17:19:41.000 UTC,2024-12-09,2024-12-01,ProphetBots,solana,2.136835,Buy,17473.63405,WOLF,EpPk97NPq4FnEAXnVgmXtfzQMkvsctc67mkhfyN8pump,0.0095,SOL,So11111111111111111111111111111111111111112,0.11246500000000001,0.0005,SOL,So11111111111111111111111111111111111111112,raydium,4,WOLF-SOL,6zWWGQWWvzTRcZTQrMf4NQJsJssVMBvQm3RvLNENdKD9,7dCS2PCbi6xxEcd2UP4WX1vJHtkXuBCRT1PRLVkEJGqR,2u12GokXffjurqYJYX4WQdnKWppBnS37LcvqrKq47Spg4TYh2ZWuso5WLKajpgcqpiEDaN2W6nRimrHZGseXhwmz,1951,9,0,true +2024-12-09 23:26:11.000 UTC,2024-12-09,2024-12-01,ProphetBots,solana,101.63727,Buy,110764.467679,LANA,EsciG7G8g8xVeg9dpy8yJCAavrWNDaykmoMERU8f4Lhv,0.4655,SOL,So11111111111111111111111111111111111111112,5.34933,0.0245,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LANA,8wont5zeLweXUNPU9ntUDb1nfA87t8THvQ9o6SSPZ5xw,BnLwsxtMBqeVSnvub5WMeys8HH51zjLAgLp6zwZsytE9,4jwVMkhhhsTo8igckryRqhF2LpQGkhzrbZyKzT2F6fvjdyHAL6BNkGNCtwhBFhNVGWFT6YZVzxhq1LGeU7CtnMct,165,9,0,true +2024-12-09 23:26:16.000 UTC,2024-12-09,2024-12-01,ProphetBots,solana,31.113449999999997,Buy,33360.555797,LANA,EsciG7G8g8xVeg9dpy8yJCAavrWNDaykmoMERU8f4Lhv,0.1425,SOL,So11111111111111111111111111111111111111112,1.63755,0.0075,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LANA,8wont5zeLweXUNPU9ntUDb1nfA87t8THvQ9o6SSPZ5xw,Dn6kWtwBVRWd7rXtCo2718rUfXUTLgoL452BvGSy6a2U,xopZtoxfufcNxRqcToE14p1M6V5C8EPAxTHW1WCCFTNeBXPPmEdPJpEgX84buqiNFxcAoXvbtFMTw6U9yLgqYpv,1980,9,0,true +2024-12-04 03:39:56.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,21.546407681999998,Sell,0.090341332,SOL,So11111111111111111111111111111111111111112,10151.01789,CHAINSAW,7nFvyQr2mwHLjBBECj3MsUxeAbP2D89p7Tx6bQtBpump,1.104251661,0.004629986,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-CHAINSAW,Eg4hFsUc2MwiBumykUe5dnQws37M3P3WzdGFKpQuqp5t,4Q2nw5FenKdW21hbK3n3VYHmwH7fZqnsC4wMQUyRoc35,3JAxGbnUjSRUe2T4jiDfMGmNvaZ9hFojzSexQw1Ki8NKEuPWJRJMCpMcgDfq9R3CyMwywH9JE7RcTfJDHZdA66Bt,1410,3,0,true +2024-12-04 00:56:14.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,169.404,Buy,359939.716201,ASHLEY,B2GoHwUP5zdPb7NhmvS8fVWcW7AkFE3KD3Rns12Mcpe4,0.7125,SOL,So11111111111111111111111111111111111111112,8.915999999999999,0.0375,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-ASHLEY,AFWtVR884tYLm5XXybRnNP8KZz6pavM7dcWmQUZEpwcw,DBRKwVjWGvZNKFnhuZCv6AHZohbDqBd9Am6khga6UcJA,L3N81RscW5kZqJ4HbD9YpB3tZtd3zohhgrtgykjnjv242R8EUMJTYYShe4J7uVsJ6KJBj8u5kDNeT1kzfLmAAbG,1994,9,0,true +2024-12-09 23:26:06.000 UTC,2024-12-09,2024-12-01,ProphetBots,solana,103.7115,Buy,128256.133394,LANA,EsciG7G8g8xVeg9dpy8yJCAavrWNDaykmoMERU8f4Lhv,0.475,SOL,So11111111111111111111111111111111111111112,5.458500000000001,0.025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LANA,8wont5zeLweXUNPU9ntUDb1nfA87t8THvQ9o6SSPZ5xw,Hc7ntYHQ22bxMdjhcWrBhD539u1S2CWjiQe8vvEqFbdS,5VdcNFNTEG4sf5U7hZywQGjixquR1yGaCfmuhRSUvEr3WR9fw6zA5f3V4qpTPAaK4AkBKmJtqWtj4pfsFNLwG3bs,1480,9,0,true +2024-12-04 17:41:44.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,10.831425,Buy,61992.280328,PABLO,s48HTTPaaPga5AUXaWHXUvQbo1ASmrt3ypparC1pump,0.0475,SOL,So11111111111111111111111111111111111111112,0.570075,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-PABLO,EzjDsgyNcg5QGHVJABaensJ5aeRKpcqa8x9E6CAUdQzA,FZdJmNuuGBDPkd3uevTiPW5tHcMPcCGdVBuu78yCqPCy,Qz4BFs896AYCUCp9SoAebd55yaZTCah4FmP89CWpBZkT1TjrFgYj2uFzQzKxfidAhg5tEBBPiSFwn2brGp6Z2zA,1240,8,0,true +2024-12-04 17:41:35.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,10.831425,Buy,85020.722187,PABLO,s48HTTPaaPga5AUXaWHXUvQbo1ASmrt3ypparC1pump,0.0475,SOL,So11111111111111111111111111111111111111112,0.570075,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-PABLO,EzjDsgyNcg5QGHVJABaensJ5aeRKpcqa8x9E6CAUdQzA,GK1dN6LKFGmS1dhYcjBnLi5zDBThdasCsQ9TwsndNwAX,593qPy9oKsywzormU8Airf7zHmPL3ZvzFMgJTPQjbTEdktatLMKU1rMft9beRQRoGKqegv7d15ccKnqhsje3LxxY,1464,8,0,true +2024-12-05 20:23:43.000 UTC,2024-12-05,2024-12-01,ProphetBots,solana,10.9611,Buy,152919.975464,CHILLKITTY,8sDdRLs9yLcjrKijAQTZTAQAyAHqw7jXsYBF9Ugipump,0.0475,SOL,So11111111111111111111111111111111111111112,0.5769,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-CHILLKITTY,ASR4CbtEQxegoavEVK9qb9eAbrWi5JBfffHRTQrXLrh2,7dCS2PCbi6xxEcd2UP4WX1vJHtkXuBCRT1PRLVkEJGqR,ea3qhrUjuaM4vBLxfWYjgKYbdTKNoAAHegPNNfAz7oe3fLYcwufh1vAcKmnwCyPMLT45qGyZFRur3ucnwVdiSdn,362,9,0,true +2024-12-09 16:21:24.000 UTC,2024-12-09,2024-12-01,ProphetBots,solana,53.430375,Buy,101861.130337,OCUS,F1ivEAt4ZmFhLeRs2LiLG3956TkzqF9YR23rKPMPpump,0.2375,SOL,So11111111111111111111111111111111111111112,2.812125,0.0125,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-OCUS,5FsRuKzNb5xPtUGRC7XSpS4YxeVREwXXgX8QojB2JueL,AVyFf2NNrCjf3b8comQvZ4bpPWZcgJ29pGS8UV5i6QJd,2UCrtGwkpwNQLVAtBHuyqTycki9cWYyRWqBxykUNTFmxi1FxbciNiCtkcjGJ13mEUePxGTDgSYvKA5mFq3wACWLV,369,8,0,true +2024-12-07 17:24:14.000 UTC,2024-12-07,2024-12-01,ProphetBots,solana,114.4085,Buy,180095.388017,$TEREZA,9sqa899GR5UqAusC4nikGKRiVxi2d9MsNJcKLfAKpump,0.475,SOL,So11111111111111111111111111111111111111112,6.0215000000000005,0.025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-$TEREZA,HJNjqkhhsYn27kkT1ULy6rSoVCNG4RaQ5r8VS7xfvWZE,DBRKwVjWGvZNKFnhuZCv6AHZohbDqBd9Am6khga6UcJA,5HADhaBjPjCagLjbHRnoKrGAt1d1k87zedme4idYBLJBV7wYQ491MYyVSYnUjvdFbK6kZLaZVN1EPf5soxupiMH3,1928,8,0,true +2024-12-08 13:05:41.000 UTC,2024-12-08,2024-12-01,ProphetBots,solana,2.23763,Buy,13629.95139,SKICHILL,FTE6GymYs7Q38Sjj3wu6RpAKvGyssMzM3j1wkhjUpump,0.0095,SOL,So11111111111111111111111111111111111111112,0.11777,0.0005,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-SKICHILL,Hsqh3aSb9e7azfCVENW4JFPSiQ2668uY2eEYV4SyokTN,6JxrWa9ZDbrB756kTxk3iHQLY3B2p6Jm3AAcEsZUMh2w,41fxVaks6CjNgt21i9UWo9w1xT4rxD5W8Def5pjnnA3f5PDZ9YAcbnGTDNwYYe1pvYqncLAGsUWqSxTAFQsyztVR,932,8,0,true +2024-12-07 17:43:47.000 UTC,2024-12-07,2024-12-01,ProphetBots,solana,12.71012795631,Sell,0.052710687,SOL,So11111111111111111111111111111111111111112,17489.675256,$TEREZA,9sqa899GR5UqAusC4nikGKRiVxi2d9MsNJcKLfAKpump,0.66360471006,0.002752062,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-$TEREZA,HJNjqkhhsYn27kkT1ULy6rSoVCNG4RaQ5r8VS7xfvWZE,BeWmKAhLsMBzqJto1gwFza6UZbDcxFx24Drr9Lzsqiy5,3XXKYQTWoRkqHx2hGBXJpiBCUmvCzmhyXTrzZqo9LdwFVLmUomrudfo9vTKdUXfAz6tuCZoYSj5vwCxFXYC3My6i,186,3,0,true +2024-12-04 19:34:09.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,110.29499999999999,Buy,384509.740192,FranceCTO,5rLBiuBHD3hPY3Lk5NinJWgW8nXJaGwwMBtdoNvxpump,0.475,SOL,So11111111111111111111111111111111111111112,5.805,0.025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FranceCTO,AD3NnBV7YqdWfuDtHatoGj1GcVgivbzv8jHDmiNZdNWS,5FZJAxJPRLFXwT81NV7c4DkpL83dd1F3MscoAzzekJet,4efvwKH5brMVbame7Uvh35rQEBRV6W3SkyjVk9YWhSBV14NCvU6S6qXsWQnyxk2KY5KRVX6Dja13BLLW1AjwrTbY,219,9,0,true +2024-12-04 03:32:27.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,49.769169999999995,Buy,48037.908896,CHAINSAW,7nFvyQr2mwHLjBBECj3MsUxeAbP2D89p7Tx6bQtBpump,0.209,SOL,So11111111111111111111111111111111111111112,2.61943,0.011,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-CHAINSAW,Eg4hFsUc2MwiBumykUe5dnQws37M3P3WzdGFKpQuqp5t,wv1cUoRGjXqmnH4mMyAkLPMuUpZHUygCENG6f2KBdTx,3nwNf8SKMc7CFgEFpd3mPRvYgRxdvoQYCuyc8tHbi4HXR8XPeG1BJb8QQuhr9HYjrSmJjVsKfHpkVtZZwk54iGYM,1714,9,0,true +2024-12-04 03:32:31.000 UTC,2024-12-04,2024-12-01,ProphetBots,solana,11.311175,Buy,10419.082351,CHAINSAW,7nFvyQr2mwHLjBBECj3MsUxeAbP2D89p7Tx6bQtBpump,0.0475,SOL,So11111111111111111111111111111111111111112,0.595325,0.0025,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-CHAINSAW,Eg4hFsUc2MwiBumykUe5dnQws37M3P3WzdGFKpQuqp5t,BV3wm6iLVJtDTxC57pWLQYgCcrxnHhsaeNLWQjWVUhws,2D64xoBUAktDMyiWzJRpvW21jfhPn9pJGofAqMEPkM55Ks7XqwSH3d98yQ8N8GbPqxQPvtmcgdu19FsWmz9Rfobd,124,9,0,true From d47ed34f544f3aeff6b481e7aeb2462ac10685ba Mon Sep 17 00:00:00 2001 From: clizzard <145153355+clizzard7@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:31:28 +0100 Subject: [PATCH 30/59] add cswap solana (#7297) * add cswap solana * add correct seed * fix seeds --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../_sector/dex/bot_trades/solana/_schema.yml | 23 +++ .../platforms/cswap_solana_bot_trades.sql | 152 ++++++++++++++++++ .../platforms/tradewiz_solana_bot_trades.sql | 2 +- .../seeds/cswap/cswap_solana_trades_seed.csv | 21 +++ .../tradewiz/tradewiz_solana_trades_seed.csv | 40 ++--- 5 files changed, 217 insertions(+), 21 deletions(-) create mode 100644 dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/cswap_solana_bot_trades.sql create mode 100644 dbt_subprojects/solana/seeds/cswap/cswap_solana_trades_seed.csv diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml index 34b9e61badd..bdf23c8aba1 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml @@ -689,6 +689,29 @@ models: seed_file: ref('tradewiz_solana_trades_seed') blockchain: solana + + - name: cswap_solana_bot_trades + meta: + blockchain: solana + sector: dex + project: ChainSwap + contributors: whale_hunter + config: + tags: ["solana", "dex", "cswap", "trades"] + description: > + ChainSwap trades on Solana + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_id + - tx_index + - outer_instruction_index + - inner_instruction_index + - check_bot_trades_seed: + seed_file: ref('cswap_solana_trades_seed') + blockchain: solana + - name: prophetbots_solana_bot_trades meta: blockchain: solana diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/cswap_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/cswap_solana_bot_trades.sql new file mode 100644 index 00000000000..22c84a1da47 --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/cswap_solana_bot_trades.sql @@ -0,0 +1,152 @@ +{{ config( + alias = 'bot_trades', + schema = 'cswap', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] + ) +}} + +{% set project_start_date = '2024-12-06' %} +{% set fee_receiver_1 = 'CSWAP5SpPcVjvpsA1H2n2HjNjMsRaPnZuX8H8bVJN5wy' %} +{% set wsol_token = 'So11111111111111111111111111111111111111112' %} + +WITH + allFeePayments AS ( + SELECT + tx_id, + 'SOL' AS feeTokenType, + balance_change / 1e9 AS fee_token_amount, + '{{wsol_token}}' AS fee_token_mint_address + FROM + {{ source('solana','account_activity') }} + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND tx_success + AND balance_change > 0 + AND ( + address = '{{fee_receiver_1}}' + ) + ), + botTrades AS ( + SELECT + trades.block_time, + CAST(date_trunc('day', trades.block_time) AS date) AS block_date, + CAST(date_trunc('month', trades.block_time) AS date) AS block_month, + 'solana' AS blockchain, + amount_usd, + IF( + token_sold_mint_address = '{{wsol_token}}', + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_mint_address AS token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_mint_address AS token_sold_address, + fee_token_amount * price AS fee_usd, + fee_token_amount, + IF(feeTokenType = 'SOL', 'SOL', symbol) AS fee_token_symbol, + fee_token_mint_address AS fee_token_address, + project, + version, + token_pair, + project_program_id AS project_contract_address, + trader_id AS user, + trades.tx_id, + tx_index, + outer_instruction_index, + inner_instruction_index + FROM + {{ ref('dex_solana_trades') }} AS trades + JOIN allFeePayments AS feePayments ON trades.tx_id = feePayments.tx_id + LEFT JOIN {{ source('prices', 'usd') }} AS feeTokenPrices ON ( + feeTokenPrices.blockchain = 'solana' + AND fee_token_mint_address = toBase58 (feeTokenPrices.contract_address) + AND date_trunc('minute', block_time) = minute + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + JOIN {{ source('solana','transactions') }} AS transactions ON ( + trades.tx_id = id + {% if is_incremental() %} + AND {{ incremental_predicate('transactions.block_time') }} + {% else %} + AND transactions.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + WHERE + trades.trader_id != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestInnerInstructionIndexForEachTrade AS ( + SELECT + tx_id, + outer_instruction_index, + MAX(inner_instruction_index) AS highestInnerInstructionIndex + FROM + botTrades + GROUP BY + tx_id, + outer_instruction_index + ) +SELECT + block_time, + block_date, + block_month, + 'ChainSwap' as bot, + blockchain, + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + project, + version, + token_pair, + project_contract_address, + user, + botTrades.tx_id, + tx_index, + botTrades.outer_instruction_index, + COALESCE(inner_instruction_index, 0) AS inner_instruction_index, + IF( + inner_instruction_index = highestInnerInstructionIndex, + true, + false + ) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestInnerInstructionIndexForEachTrade ON ( + botTrades.tx_id = highestInnerInstructionIndexForEachTrade.tx_id + AND botTrades.outer_instruction_index = highestInnerInstructionIndexForEachTrade.outer_instruction_index + ) +ORDER BY + block_time DESC, + tx_index DESC, + outer_instruction_index DESC, + inner_instruction_index DESC diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql index 526385d028c..5227bed50b8 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/tradewiz_solana_bot_trades.sql @@ -111,7 +111,7 @@ SELECT block_time, block_date, block_month, - 'Solgun' as bot, + 'Tradewiz' as bot, blockchain, amount_usd, type, diff --git a/dbt_subprojects/solana/seeds/cswap/cswap_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/cswap/cswap_solana_trades_seed.csv new file mode 100644 index 00000000000..012bac505d7 --- /dev/null +++ b/dbt_subprojects/solana/seeds/cswap/cswap_solana_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction +2024-12-08 17:53:40.000 UTC,2024-12-08,2024-12-01,ChainSwap,solana,236.41,Buy,5166634.046655,RON,ArVrcEbaLhZPdvfxa4PdXBykhSUTDbKYqKfd6m5DPVvd,1,SOL,So11111111111111111111111111111111111111112,0.70923,0.003,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-RON,6w6ogSTWjpxwNvtBFgNGMyYguxdbxp13yEX2oLtozS9T,FvQ614xsLAfD8TxSszCydwhRJzGECDQBgY1oXaeWPk9P,2vmFMnKPV7SQMAPMuTX6cRsmwiRRYrk3vZAaJ4uM1A6K2T3sMsoLakb6zF6sXQW5TnMF1TYQxvesyb8rPJzSSbT3,2693,4,1,true +2024-12-08 00:56:13.000 UTC,2024-12-08,2024-12-01,ChainSwap,solana,631.0501005872401,Sell,2.638389918,SOL,So11111111111111111111111111111111111111112,101040.940582,LIMITLESS,44o1iDPGuzHxQgDrPE3HRb2WXwGLsBsY7yMttX47pump,2.04393445538,0.008545591,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LIMITLESS,14YuCFitrqHrPTzP1hP5fF1pQpnSoQNVucRnzJbYyMB7,8uciVcc4zoGmEQf1ys1ztrLMwPZiBNfGshG2BNrBjQGk,2QZ9boZwkssx1KPm1MDnNaL1pSZk8bawqShH2ACe6YmzJboaAjGFzk6ad8SSWFdb57pSqEhzQzgeEirWQDsimW8k,938,2,1,true +2024-12-09 22:47:21.000 UTC,2024-12-09,2024-12-01,ChainSwap,solana,3.4000866542,Sell,0.01588306,SOL,So11111111111111111111111111111111111111112,32.65,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,0.01081438826,0.000050518,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRIFFAIN,CpsMssqi3P9VMvNqxrdWVbSBCwyUHbGgNcrw7MorBq3g,DoKCDiii9JAevz8fW9PM8gagkuuHGSaAzMBsfZarNhUt,mTcDV84pRnFFu2JZJpFDjWRdDdotiEoKTu82yu6HeGZbouVy6UDkDtJetkn5JauBmmb5pi7DaMo8Bk7e888w5Nu,1771,2,1,true +2024-12-07 10:11:03.000 UTC,2024-12-07,2024-12-01,ChainSwap,solana,2.39099107707,Sell,0.010026381,SOL,So11111111111111111111111111111111111111112,827830824.8232,SHIKOKU,5Jng6jkLKU1o8BNrCzTEMXMFvPjNJZTpdWR3Hq4RHJb6,0.02362689219,0.000099077,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-SHIKOKU,FyQuZLz6LLJxKhSwXkMpZ5Po6G8bcRv8Ub2s3paU74pC,oN3T43asj7TyEViAgQaFdMmmsScScDNZ6C6V4UeuhXA,LFWJiVrsVbktPzNg2soY96n8NShX6eAae2UP8oK8Fo7BuqbCUX4hwNVfkPH9VYbrzAmBnjAQbGknCz6UvDANe8a,1826,2,1,true +2024-12-08 01:44:03.000 UTC,2024-12-08,2024-12-01,ChainSwap,solana,237.46446197876,Sell,0.994324018,SOL,So11111111111111111111111111111111111111112,2665.625316,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,0.71856949706,0.003008833,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRIFFAIN,CpsMssqi3P9VMvNqxrdWVbSBCwyUHbGgNcrw7MorBq3g,DkGiBk9TAUUm42rDcPWQqRstMWB4pRvN9WxuBm91WX98,5ZNNzu8kj4QFjSYXVKpTdrztYYd2UHW1dAsDtVc8g3P9Ra2fpS9nzTVthvhXcYw3NVttQ4enHcMCGJpcQd6HFYJ2,49,2,1,true +2024-12-08 13:54:49.000 UTC,2024-12-08,2024-12-01,ChainSwap,solana,134.775385344,Buy,10618804.080094,KITTY,7TgVhQwYofgkUHrKqyX6NWc4iVb2nnG87xxU3M3pump,0.568672512,SOL,So11111111111111111111111111111111111111112,0.3555,0.0015,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-KITTY,89WiiZ66AfCsEckK4YhLwxVYZhAPmDcaePkzdnXeHbBP,Ex9E6pWf8CXyUby5TubhGkxuCk7NZnLJhyD65D5qidvq,2XkLYLPszv9nWBzYbY2jepTQ4JT2bkcvpkNPSKhyAj7FTfbAoj8tLABwmRwU7ZM2PHW2vyM3o8v6hr6BboYxcSpF,1369,2,4,true +2024-12-10 23:42:51.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,642.24,Buy,2557653.017448,NORM,H4WLmYEc1Q9Fk8cbvieqiqc8ALbYA5W2bhbKdkJE2bth,3,SOL,So11111111111111111111111111111111111111112,1.92672,0.009,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-NORM,CYSLB654GfgYE1gQBe9TNbnyZB2kKFTPd5bweM5vLBGw,A9wB4BddvvsKnGM3BTJSWjhQ4ibs7piBtCXd5bdhJZVY,37Jpm6RfG9joRi9KtGKNF91rnEMHrocD128SRGaNmPkBCpVT6cwRkpDE9Yv3vhq1DbPH6sVbabn4mrr9xE8gxxEq,1747,5,1,true +2024-12-10 01:33:26.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,10770.450408761199,Sell,49.14871958,SOL,So11111111111111111111111111111111111111112,81469.171522,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,32.43617846748,0.148015782,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRIFFAIN,CpsMssqi3P9VMvNqxrdWVbSBCwyUHbGgNcrw7MorBq3g,88XssAnXed7S4JGS4CFbR7isBtTUbk7CWRtWEbiwCWYX,4FtUdRsx4FKj5kWJDWxsjL3vakYEcWrUUdC2UwS9rcGCyv452zqhJTPF3wBURsYeq6FiHVuBzSXsFfNZqwyoRUEk,98,2,1,true +2024-12-10 22:21:04.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,105.91416678052,Sell,0.497062919,SOL,So11111111111111111111111111111111111111112,546.471895,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,0.32020319996,0.001502737,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRIFFAIN,CpsMssqi3P9VMvNqxrdWVbSBCwyUHbGgNcrw7MorBq3g,BYCcoE7DoBN8e3SijRC7rhTnjeGsWPZxNJYb3pYY6EiG,3v7TwmmYUrkuETW5f7MciiP5rdQSXVnpigson5hZEgwi2ELRasKMkwowr7JnrJ1waC6hv3grfLDjgkAfJtzBEEaj,2187,2,1,true +2024-12-11 04:55:07.000 UTC,2024-12-11,2024-12-01,ChainSwap,solana,24.38229109238,Sell,0.111344831,SOL,So11111111111111111111111111111111111111112,1142698.17840853,G.O.A.T,9ko9A8Fc7yRTrYxmtQDAk36WAeLtczrsPmaf5yH1RPTS,0.07536503271999999,0.000344164,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-G.O.A.T,3dzEMpvw7MZm8vq9nKKqxnc7fCUpSqUQEAbRweZQArmR,8igyY1CBBC8VnkcXjPtdMQmfdtc1uTQ7Xsd6tiuj7bgv,ySQVVBRdFDEUYJfT6Ed2kQKNTXHy4yWrYzcPmgm5h4F3yuSHbiSyLE2BownRAe6S8NeZsDJ1AGG3Q5by2EoCpZq,1556,4,1,true +2024-12-09 22:13:27.000 UTC,2024-12-09,2024-12-01,ChainSwap,solana,542.30566942638,Buy,7730247.791584,Mario Bros,Hbdb8rSS7NdpDeuVx5NYUyR9VMKPAmo2nVim2pyDpump,2.508119829,SOL,So11111111111111111111111111111111111111112,1.29732,0.006,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Mario Bros,qc2ZDKTXxJ26EWEQKGWPJdfUP5UotNjzt7gNyQvogu5,2v7nLcvVde3fXASST46xX8u1qdcffAXXGtJLZfKD29sH,3uWWHnQSKPwtbPLBEDjdHCcUkvQa6gFvC6RQHcyEC91cMJgB9Bncr4GduNPvmJV3PW3MAA31AKJ7V94iYa359Sqf,2225,1,4,true +2024-12-08 17:03:15.000 UTC,2024-12-08,2024-12-01,ChainSwap,solana,117.94,Buy,890505.749209,Luffy,9yP5HSq8HK3aN49D8GwJoYbuFGcKnRuvQNmYxrRJpump,0.5,SOL,So11111111111111111111111111111111111111112,0.35382,0.0015,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Luffy,tz5JL88F5tvAUPJE9RbShYB8z16LcpqHL4emmJw6ten,FV3Tt7JtCQ7w6WKsbZxP5r2AN9sX7LCjZroWMJpoKjp3,4g2bQ2m8L8HpJDts6MiSs8sQdH6okXXJPCaEmNhzTj9P3UnmL2RJpx9C9v1txXfaEZGMu7RKY8zyax2VbFz4NmnH,508,5,1,true +2024-12-10 16:03:14.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,403.94903408934,Sell,1.953426346,SOL,So11111111111111111111111111111111111111112,4982686.673278,Kawaī,csdwe8ZRjbchjCWdeSZ8GSNw4XWbRzmCewzD9dUpump,1.3224425222099998,0.006395099,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Kawaī,4VZSRvRECysU1R75kkHqeqnGzgqcBKsL9MaQ5fbgR56E,8BXTUKqo2nws3HJ5SwQHzSH6bECyP5mMyPvj2MuHJdef,8Sd4aEp14mkpVvUW5bdySG2Ee2a9K86pWjyJgw3hKTchE4H1WvjcCjVuhTihHDSGxbHREgpRTFXkmtZuK2menvC,2154,2,1,true +2024-12-07 15:50:02.000 UTC,2024-12-07,2024-12-01,ChainSwap,solana,234.91497195601,Sell,0.978771601,SOL,So11111111111111111111111111111111111111112,115245.068956,VIRGIN,2MtX583jbXXKnE4BvghsEi4ud3uQJEkJVnt7oWc3pump,0.7168522676,0.00298676,SOL,So11111111111111111111111111111111111111112,raydium,4,VIRGIN-SOL,3CLYXZAcAFBjqAfAwxeSUrpbx2ACZyfnGjZ1WvZ1utYj,88XssAnXed7S4JGS4CFbR7isBtTUbk7CWRtWEbiwCWYX,2v49qpLkXkUySc4uyGLQWWQ3TqPsTDTT7zCQLxNQdRWbWWQJEdCJoUaj6jVK4gUAtu9JwWzLv5qrjkw5yT2xx8XU,265,2,1,true +2024-12-10 00:31:19.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,213.85821178848,Sell,0.977771634,SOL,So11111111111111111111111111111111111111112,1731.008892,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,0.65331532768,0.002986994,SOL,So11111111111111111111111111111111111111112,meteora,2,SOL-GRIFFAIN,EN5ji4DsJCaLNXzkJVEX9sgsN6XMxq5ssLkCxgiDJiM9,GPqcYH2yP1w4LPpQN65mDcU5CdBbRmS4QvVAJP8Lo4x7,4DNXtgt1kp3UheG49gSQcfwSNnXuRo4jV1qXZ2azpLznwYUkwos5AH1x3EfPiMcCfguh9CnTkUL1qUzihcnMiuF9,535,2,1,true +2024-12-10 11:43:17.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,3143.608268642,Sell,14.5544158,SOL,So11111111111111111111111111111111111111112,1212813.794696,yumi,FhbqJtDE5XXk9Netjvrx5nhmnXxFRAS9eoGLwMi2pump,16.44982237489,0.076160111,SOL,So11111111111111111111111111111111111111112,meteora,2,yumi-SOL,5SeoX3xAe7F9xxDxXVLvRyRtYfSi8yaB5NakpJt7DHEL,DkGiBk9TAUUm42rDcPWQqRstMWB4pRvN9WxuBm91WX98,1NfcHQvucLuAGsfTpz1cM3UG8WPL4RCe3BCqk2AzXf6KbFpqHC6q1Ez1UQbD3ksvAQL3dKQ8MsFJyWp1KApqE3i,642,2,2,false +2024-12-10 03:05:28.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,851.474647641,Sell,3.997533557,SOL,So11111111111111111111111111111111111111112,5151.222836,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,2.585950995,0.012140615,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRIFFAIN,CpsMssqi3P9VMvNqxrdWVbSBCwyUHbGgNcrw7MorBq3g,BYCcoE7DoBN8e3SijRC7rhTnjeGsWPZxNJYb3pYY6EiG,3YpZyrUnMJYW28pVc6nfwVbMHXfUeD7iAgt2nJ5JmUxURT4urXGrxtsJyamkfU4cgeAgv18QPCteguiEYAxixN4d,1867,2,1,true +2024-12-10 01:05:46.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,6466.025441846759,Sell,29.630764558,SOL,So11111111111111111111111111111111111111112,49584.452082,GRIFFAIN,KENJSUYLASHUMfHyy5o4Hp2FdNqZg1AsUPhfH2kYvEP,19.09584824328,0.087507324,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRIFFAIN,CpsMssqi3P9VMvNqxrdWVbSBCwyUHbGgNcrw7MorBq3g,DkGiBk9TAUUm42rDcPWQqRstMWB4pRvN9WxuBm91WX98,vsBxekU3cg4LGUYs4vRCSsdVzdKZUWQZEMZzr6ooXcwTVuCnoK66xjUnyb7m4zZm5DfMpPSrgEgdVAcwZ46Tq5Z,990,2,1,true +2024-12-10 18:46:45.000 UTC,2024-12-10,2024-12-01,ChainSwap,solana,220.71725077265,Sell,1.047990365,SOL,So11111111111111111111111111111111111111112,60094.458308,yumi,FhbqJtDE5XXk9Netjvrx5nhmnXxFRAS9eoGLwMi2pump,0.63149955291,0.002998431,SOL,So11111111111111111111111111111111111111112,raydium,4,yumi-SOL,EvWxeXEmTgMVXLfqcugKvCDQ5yZPGHt8iq9N6ELWKf6,Cq6FCvEqBmfsSa89377vNH1H1b7NDJmqNZMvAw74HRYS,2w59JeJdbzDfABXkmC3ZSohbEgnssKGGhXqVtpWC3FWoVGiqv752SpSeatUZTVvSSNhr8zVbyVRHYEaGpsBvD8Gv,1074,2,1,true +2024-12-09 17:28:52.000 UTC,2024-12-09,2024-12-01,ChainSwap,solana,673.83,Buy,541278.944723,POLYBOT,CXLsXyGNRznVaJRKCcv7Cne3sbxGRBy4BdQMo3Tjpump,3,SOL,So11111111111111111111111111111111111111112,2.02149,0.009,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-POLYBOT,69hWrBg3g86BmRhJ8zAZGz2zP53CyKHCaBoFvqkarqm7,DkGiBk9TAUUm42rDcPWQqRstMWB4pRvN9WxuBm91WX98,4xzRFQPbqB3wJZPNgJLpqT91gJ5a2y1Yh4HUzwzhUr2NzNM7jhZhSWWSCyWYMBoFPBzy1cRFUuPC5TX1CHZ3Rs8D,2003,4,1,true diff --git a/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv index 2525f27cce6..b3d981bede7 100644 --- a/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv +++ b/dbt_subprojects/solana/seeds/tradewiz/tradewiz_solana_trades_seed.csv @@ -1,21 +1,21 @@ block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction -2024-12-03 03:41:57.000 UTC,2024-12-03,2024-12-01,Solgun,solana,5.10029703516,Buy,130284.164414,VIBE,AB4oELjeXLyeV3t3evMgq3zXrxXAJ1WdRy9qgfinH1oq,0.022402148,SOL,So11111111111111111111111111111111111111112,0.04940438999999999,0.000217,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,VIBE-SOL,AKtxTJHHtyDTBUerXQhWQSY4L8oBreBqPHkiPbkyoL3s,8dRLn2NMHsN7Fu1nxCbfQDujQ4wyPaqmnfqHyfVvBnmj,5xYqUmnvW78WSa1VL9G3H2fkRcBbaGRfr95iDTH9JWNTAheT7pnNguneMDzkge9Bqs5KTK8boRHKHPupBtW4hBL1,1139,7,4,true -2024-12-07 15:17:10.000 UTC,2024-12-07,2024-12-01,Solgun,solana,2.1149015703,Sell,0.008739985,SOL,So11111111111111111111111111111111111111112,312509.10149,CELLAI3,5q4maTRYm1ZPuFZqFzwYRJ7jAA5xkprmndJdGHUnpjwT,0.0209373195,0.000086525,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-CELLAI3,ARUgR3UzG2KCXWAXQNSHdPuuZ5Spmmm8vTENUn3UK4HU,F6coj9TDxLDw1wSFYsQMPmA2iPYhNxu17qBSEZ8vZuDm,5769zBmZrWqPGqWU9QA2LH5CrjPnJTWePmtC8bgTtciupjiLNuNaYuLS3FEkN7eThaQj57NcTc1Hz39VnkdtDgbB,2030,3,2,true -2024-12-06 19:47:39.000 UTC,2024-12-06,2024-12-01,Solgun,solana,5.72664660432,Buy,103332.858778,PAWBLEMS,4fLAZZCA3Mrbj6Wa9PkTUHfUWu3JE2RqGHvDrXiMpump,0.023872964,SOL,So11111111111111111111111111111111111111112,0.047976000000000005,0.0002,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-PAWBLEMS,kLCF7gruvuMfBLvwSL4XZQK2ENKYKm97LFuLAKkxC5b,DksfCf1U4arH6eq9KJJJtHDiYypi3gzwnxbup2266uae,szuK2Sdz9ps9Go88t1FVjFRezggBdh1RqWETVD8voF78mJm97H3pnt6EYrN9aUmYHUmGAvKfrTs46tJfC2F1GCt,1583,5,4,true -2024-12-06 11:48:01.000 UTC,2024-12-06,2024-12-01,Solgun,solana,65.36809866600001,Sell,0.279350849,SOL,So11111111111111111111111111111111111111112,371324.880761,HYPHAL,2gsxJ1fyLi3kzvjQx8Wd8y323Ptigzj9ti54grHkpump,0.658593702,0.002814503,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-HYPHAL,2YeDSYdr9mMvEtiWskGvHbt6WSZ6wHW9xGi8DuB9GjdJ,Ed1rA4YjgtJMExjQ6sjWrEEj7xaeL6f4yvT39sm8fZo8,3VN42UsvXsxqnUDiTDQaHhrWmGCwoWnfJJWyPbz7nvja5n9EZohisb7ZR4ojD3LzDG8rquC74kJY3Z3bHac6FYqh,64,4,0,true -2024-12-03 06:01:15.000 UTC,2024-12-03,2024-12-01,Solgun,solana,12.54215028618,Sell,0.055040814,SOL,So11111111111111111111111111111111111111112,434296.831722,Pooch,csrNabqJ6eKoueCiVxGWTTbN3Bri1MhwbwoCN2Wpump,0.12432632773999999,0.000545602,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Pooch,H82eA83r3Djyt1s4T6213puNzM1As8w2tCSYjgv6kjFz,66gfZJ3yVCxhRY3VvSKxTYhrW4HAUvhbVFRCRrgoATkE,3hhXSyxnoA3U1eYsY2P8EjAqen3mWBscPoPK4pudmobfENN1VKCzAHKPaBQRk4S98W9fB41tm4MgUJ1NE4F4ZjNt,413,3,2,true -2024-12-02 18:50:09.000 UTC,2024-12-02,2024-12-01,Solgun,solana,33.5145,Buy,1090000.477547,FWILL,8aMmdyEH97qmtxge5odgJE45KwwXQkaSZEjM13EDpump,0.15,SOL,So11111111111111111111111111111111111111112,0.335145,0.0015,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FWILL,5QiczEwrbzHNooa54TVkt4oe2HFHLd9u2TCJNX46TJjj,A3kLfkrESnvmZUs54BKYS48bPq4DJT11psfFC5ADMSwS,op8TBkDy4D5gr7xX4VCiC6VvEXCuYV4zo3fhHCEJMFu4Athih23wEzHw24tA1WmjQd4eZ4swA8o7Dii6fk1vC6j,181,6,0,true -2024-12-02 05:59:04.000 UTC,2024-12-02,2024-12-01,Solgun,solana,0.012848601869999999,Sell,0.000056277,SOL,So11111111111111111111111111111111111111112,58135.635755705,LIR,Agdtmev8iBbBJZjXDf6u18M51Bu173gsJTzcWB1patsJ,0.00012876684,5.64e-7,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LIR,84PapxynnSyNL7LRg4vKg41SVrmU2NqRxaudeQe1NYUb,9gxBDddAvsVUmcLAwJWhnKr5dMN8rWkdqn8ZZ4V272bo,3UBAiPDUWx2jpU4H7u3VV4LkkYXMjxUtbnK2uj71mkHb2T22iWFxvtAnjac4rwQA5itbZb9VahYM4XRbvpEEyxdr,552,4,0,true -2024-12-02 20:48:46.000 UTC,2024-12-02,2024-12-01,Solgun,solana,174.81549384187,Buy,14986480.13779,FAITH,CK3TMoqHy5ApzRwooxbxMAbpGDK2w6KpQMdv2rDhpump,0.782137237,SOL,So11111111111111111111111111111111111111112,1.6763249999999998,0.0075,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-FAITH,5PBojHVrS9C6F6kjTosfVEUoBzED8PCxpFdeouWzDQbd,2kNoYHNtpaefWZ3JptWug7H7pyzXZXYYTQufNbnt3Zzy,3gPFS1BtQArtTN2hZwbrUxKETHiHRts4YCcCEZqFxoGo9E8zpYztsjdtNsUmbK14qNT7VdieNJXbY9tBTembavCB,413,7,4,true -2024-12-02 18:06:48.000 UTC,2024-12-02,2024-12-01,Solgun,solana,30.016174407599998,Buy,1809787.710499,ART,CuSXYspmP7khJiov8Kr3XaExcuhPadVDHGm3K3rvpump,0.13517146,SOL,So11111111111111111111111111111111111111112,0.29361128496,0.001322216,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-ART,GD5CvxQZwKLfqMpvMTeLkTecnMVZGNPXnUk597W1j8hb,H9kHjTaJqVrrSuZGwJWgQnjyhVmGANpqsA6eoqbbZrTk,2SZ1av6twfq14Um3gM8KuRopdqdJpDtESqidS725tKMiDMa1zQu2wnr76JMsbXZMrH5LiUyGv9XPUqMkmwbjW4S9,2628,5,4,true -2024-12-01 23:25:30.000 UTC,2024-12-01,2024-12-01,Solgun,solana,12.139543894200001,Buy,1715893.906365,Bio AI,7LL7cbEfKLMiH4oUmfnfYgAhUnnVRWGeksE6iXjDpump,0.050942274,SOL,So11111111111111111111111111111111111111112,0.2383,0.001,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Bio AI,76BfryV9K9e9o8zF69csrqFLaar2PpdYZ8hN7ZRE4Y3e,H5PQNrAj7uKYK44ksRqym8sAfuasNC4xHJZW88WbzDyT,3YXaTrff9qmYHko2ZD5veVtAcF7XfJa8eCmFBmMtV5dLdowbCn2D2ArfkymQ2ZLon5ExhEYr5qbM54JJbE4NiZtU,475,7,4,true -2024-12-03 20:40:46.000 UTC,2024-12-03,2024-12-01,Solgun,solana,148.38664547177999,Buy,17917109.708311,WILDMNKY,75fv3J8gw9mp64ZVcAXjmr1qT8PqJjE1Cc4ocjmCpump,0.645552273,SOL,So11111111111111111111111111111111111111112,1.3791600000000002,0.006,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,WILDMNKY-SOL,7SV8yk5hXtcXNqgohc9tJMdmAbBnLpbKr8oURHdh8s8g,4Nw2v6XZJgaWHmtETqMWtQyTHaGDtQZ27x3otcvjQAHT,ep8K6Nvu25kw1yVWqENPHv2Y4mGRW5T7kMBxeXnmujd6WoDxzSfVtqASa2fz5toNTyX1JDWecAoMHxTdEsXnkxU,601,7,4,true -2024-12-03 00:15:12.000 UTC,2024-12-03,2024-12-01,Solgun,solana,5.12252529562,Sell,0.022631993,SOL,So11111111111111111111111111111111111111112,27129.461569,anthrupad,2okJ4wqqDFFD2Tce3QaU8L866KxBJzJXVoMtLR1ppump,0.052636498700000006,0.000232555,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-anthrupad,FAfDgkWFexH1BA2D1i5pc2uRR8ZYcvt8jY84CK7e9My8,31M2cNExVg8bwB2HwzAzA1w9pHwLeP67ejygZ44UB1UR,2HryhZK2dRLqZeTowRQ4Rd7BGwL9ekAgyjDuaFvYAJyyVZHn5C81cTGGc66eMRg9MhiiWEdU7j1yWKfJDjdfbMRU,499,4,0,true -2024-12-07 18:47:05.000 UTC,2024-12-07,2024-12-01,Solgun,solana,47.17441114759,Sell,0.195040357,SOL,So11111111111111111111111111111111111111112,12392.36946,AFITR,83Ur4XYc5QyRNBshoW2Yu9cCLyNDV9s5krBu11JWDxxq,0.47612907671000004,0.001968533,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-AFITR,5JtbDjPqfN6K4mDF2c5rvJZXXryQbihjViSP2XoMohSV,8uAied3jRqG1mPRzqsmLaeRDz3F22BkarmfgAvoqNWN,3Lr37rTZUVk9m1iYz3h8sg6Dt4dMHm4PhLmuPUe7G1Z3tMxraN54M8boZMP5wdXGqs8NGQPHMDaXLZoKAq1zMZLE,238,4,0,true -2024-12-01 18:04:29.000 UTC,2024-12-01,2024-12-01,Solgun,solana,32.99628642063,Buy,438388.5076,Metanoia,2dPFQnktvPKZe3hjkstWfJx8tDrH8FW9xoiBjZMEpump,0.139218963,SOL,So11111111111111111111111111111111111111112,0.2986326,0.00126,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Metanoia,FpTruBXx6UcyBiVr2Rj7UMf9tNTpJgTyGpXynpuo9HzA,GhqrMrC4tY8zTDvWHiHG7ag4MWPqQ7qLZmeochY6aosY,29TPJAT4vP7rUmMETneBknCmH8UpzdeFaB6ZpJTmzVobE6kTou8KBgqvPPLb588ettce6HRVFUPcSbuSrRJpJnzG,620,5,4,true -2024-12-01 02:27:09.000 UTC,2024-12-01,2024-12-01,Solgun,solana,19.69581542709,Sell,0.083524089,SOL,So11111111111111111111111111111111111111112,313146.63071,GSC,F86FtKmg8vnaCfDoJF7U21NbmLprVazE3L2PshmCpump,0.26366694273,0.001118133,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GSC,7H6MS1nqg9Y6NN8srD9UCovDQwVxuZaozgXrMT7kcH61,35cx6mhp2Nsi33d7QfwnqVrYoEcyWpc17PtgMrCnwfxt,5K5isZKhK8mwM4rZ7akSejvwByViNxZ5GcCMcsqWENmT42WkJ4kUoYsAJQkeq73pwiEBgjwEoUEgRNB8jCt9Bugp,541,4,0,true -2024-12-04 17:48:52.000 UTC,2024-12-04,2024-12-01,Solgun,solana,29.19816884426,Sell,0.128270302,SOL,So11111111111111111111111111111111111111112,1239313.36229,Pretty,8pTpjjcq5pG7NXDm99NinoT1vRoH99X7s3C1Go3upump,0.30598138415,0.001344205,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Pretty,HBiCDY9CVNWENM8vTkHekJoNAeaLZAJoHr7EwvmAf5WY,GhqrMrC4tY8zTDvWHiHG7ag4MWPqQ7qLZmeochY6aosY,53KrU2unKY95pnSKSq3empC6woBXPxQShDv3EmpDNnn1knesLfxHtCetCkbuNPRWPXr3tj3WKuu8B63Ypawx8yvb,977,3,2,true -2024-12-04 04:50:48.000 UTC,2024-12-04,2024-12-01,Solgun,solana,77.97821513972,Buy,1428724.561159,eigenrobot,9qpaeuNMNWwHfa7keQ51D5VJNt2jessw8khxWis5pump,0.327832402,SOL,So11111111111111111111111111111111111111112,0.7135800000000001,0.003,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-eigenrobot,7Ys5V2wmjz3LihubgfnCdSZnwEUvBNBXXLvpAWZLRiS5,5q853P8PLeYiE5PjXg24kQ9nXrf5Qn6r55A9urS4hYF6,2XPV2bKhVjYGjuLGWhxjUK69mPFqyfFYcBM6HTSnErYLxHP1cNSksWp4i3VbF7QesYJtQnPvtt5aixEbkr137nz1,959,7,4,true -2024-12-04 13:04:31.000 UTC,2024-12-04,2024-12-01,Solgun,solana,0.00010256337,Sell,4.41e-7,SOL,So11111111111111111111111111111111111111112,117382.719111186,FCI,7s9q7HcfMhhf4XoFtHUzJWyAPAN3mSEmBDfB17iM67QJ,9.3028e-7,4e-9,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FCI,8AfbS9Ji49Jj4UkZmm4whYACyUN8QQQa1dsj9JuYQjFs,EcpfuULFUmouh7XU4uESrksywAwoND2PBu7T3LJwXJmM,4N2Fw8hjiaKSBHnkc7nbH4C6TbjFtpE5RHas2uQKkJvNVUSbwUuTST6GhTueLsrMJe4ho9Sr13xm3V1QQtNzGcSf,460,4,0,true -2024-12-04 20:35:20.000 UTC,2024-12-04,2024-12-01,Solgun,solana,46.792,Buy,66517.719092,convo,KnrmAHo1wW2fGqVXcSZSpo6kmiKam5vNbDByrURpump,0.2,SOL,So11111111111111111111111111111111111111112,0.46792,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-convo,Exy58ud7t8LHN1EBnrcRRwRQmz1uSUC3hQF9oHpM1Box,5CEbKj8WbqKkah7nhYxkXGvN97xvz5hxd8fQjYnW5NTV,rNFm4bvTJvPVtqQGoBhyFBPhFbJftPVxoasZccwEhaaJqBA4GMVurr3aJTexbDm4p673RSnpTHpnmxLshHmGEK6,38,5,0,true -2024-12-06 07:59:32.000 UTC,2024-12-06,2024-12-01,Solgun,solana,47.796,Buy,8548.498569,PIZZAGUY,5cN4dWZKqxrpwCCBcwHpsAgus1s7WWyRaguR8YbRiECY,0.2,SOL,So11111111111111111111111111111111111111112,0.47796,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-PIZZAGUY,FgYufZ6c7NpcMvRnbB6tYupsEoyiPCtBw9dZ3bVo8kHE,1EjTzPniAGBxzvi4TBfocfq4Zxuj2GCK42aWnNJEoSS,619fmC5opZwwwAnnci5snSqtVcp9cqPYW2SAiwq8DZQrPVS8F1cLDZW9fzwxdjBYbBKZtBTeMyA9rjeCkLU1U7uE,1700,6,0,true +2024-12-03 03:41:57.000 UTC,2024-12-03,2024-12-01,Tradewiz,solana,5.10029703516,Buy,130284.164414,VIBE,AB4oELjeXLyeV3t3evMgq3zXrxXAJ1WdRy9qgfinH1oq,0.022402148,SOL,So11111111111111111111111111111111111111112,0.04940438999999999,0.000217,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,VIBE-SOL,AKtxTJHHtyDTBUerXQhWQSY4L8oBreBqPHkiPbkyoL3s,8dRLn2NMHsN7Fu1nxCbfQDujQ4wyPaqmnfqHyfVvBnmj,5xYqUmnvW78WSa1VL9G3H2fkRcBbaGRfr95iDTH9JWNTAheT7pnNguneMDzkge9Bqs5KTK8boRHKHPupBtW4hBL1,1139,7,4,true +2024-12-07 15:17:10.000 UTC,2024-12-07,2024-12-01,Tradewiz,solana,2.1149015703,Sell,0.008739985,SOL,So11111111111111111111111111111111111111112,312509.10149,CELLAI3,5q4maTRYm1ZPuFZqFzwYRJ7jAA5xkprmndJdGHUnpjwT,0.0209373195,0.000086525,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-CELLAI3,ARUgR3UzG2KCXWAXQNSHdPuuZ5Spmmm8vTENUn3UK4HU,F6coj9TDxLDw1wSFYsQMPmA2iPYhNxu17qBSEZ8vZuDm,5769zBmZrWqPGqWU9QA2LH5CrjPnJTWePmtC8bgTtciupjiLNuNaYuLS3FEkN7eThaQj57NcTc1Hz39VnkdtDgbB,2030,3,2,true +2024-12-06 19:47:39.000 UTC,2024-12-06,2024-12-01,Tradewiz,solana,5.72664660432,Buy,103332.858778,PAWBLEMS,4fLAZZCA3Mrbj6Wa9PkTUHfUWu3JE2RqGHvDrXiMpump,0.023872964,SOL,So11111111111111111111111111111111111111112,0.047976000000000005,0.0002,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-PAWBLEMS,kLCF7gruvuMfBLvwSL4XZQK2ENKYKm97LFuLAKkxC5b,DksfCf1U4arH6eq9KJJJtHDiYypi3gzwnxbup2266uae,szuK2Sdz9ps9Go88t1FVjFRezggBdh1RqWETVD8voF78mJm97H3pnt6EYrN9aUmYHUmGAvKfrTs46tJfC2F1GCt,1583,5,4,true +2024-12-06 11:48:01.000 UTC,2024-12-06,2024-12-01,Tradewiz,solana,65.36809866600001,Sell,0.279350849,SOL,So11111111111111111111111111111111111111112,371324.880761,HYPHAL,2gsxJ1fyLi3kzvjQx8Wd8y323Ptigzj9ti54grHkpump,0.658593702,0.002814503,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-HYPHAL,2YeDSYdr9mMvEtiWskGvHbt6WSZ6wHW9xGi8DuB9GjdJ,Ed1rA4YjgtJMExjQ6sjWrEEj7xaeL6f4yvT39sm8fZo8,3VN42UsvXsxqnUDiTDQaHhrWmGCwoWnfJJWyPbz7nvja5n9EZohisb7ZR4ojD3LzDG8rquC74kJY3Z3bHac6FYqh,64,4,0,true +2024-12-03 06:01:15.000 UTC,2024-12-03,2024-12-01,Tradewiz,solana,12.54215028618,Sell,0.055040814,SOL,So11111111111111111111111111111111111111112,434296.831722,Pooch,csrNabqJ6eKoueCiVxGWTTbN3Bri1MhwbwoCN2Wpump,0.12432632773999999,0.000545602,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Pooch,H82eA83r3Djyt1s4T6213puNzM1As8w2tCSYjgv6kjFz,66gfZJ3yVCxhRY3VvSKxTYhrW4HAUvhbVFRCRrgoATkE,3hhXSyxnoA3U1eYsY2P8EjAqen3mWBscPoPK4pudmobfENN1VKCzAHKPaBQRk4S98W9fB41tm4MgUJ1NE4F4ZjNt,413,3,2,true +2024-12-02 18:50:09.000 UTC,2024-12-02,2024-12-01,Tradewiz,solana,33.5145,Buy,1090000.477547,FWILL,8aMmdyEH97qmtxge5odgJE45KwwXQkaSZEjM13EDpump,0.15,SOL,So11111111111111111111111111111111111111112,0.335145,0.0015,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FWILL,5QiczEwrbzHNooa54TVkt4oe2HFHLd9u2TCJNX46TJjj,A3kLfkrESnvmZUs54BKYS48bPq4DJT11psfFC5ADMSwS,op8TBkDy4D5gr7xX4VCiC6VvEXCuYV4zo3fhHCEJMFu4Athih23wEzHw24tA1WmjQd4eZ4swA8o7Dii6fk1vC6j,181,6,0,true +2024-12-02 05:59:04.000 UTC,2024-12-02,2024-12-01,Tradewiz,solana,0.012848601869999999,Sell,0.000056277,SOL,So11111111111111111111111111111111111111112,58135.635755705,LIR,Agdtmev8iBbBJZjXDf6u18M51Bu173gsJTzcWB1patsJ,0.00012876684,5.64e-7,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LIR,84PapxynnSyNL7LRg4vKg41SVrmU2NqRxaudeQe1NYUb,9gxBDddAvsVUmcLAwJWhnKr5dMN8rWkdqn8ZZ4V272bo,3UBAiPDUWx2jpU4H7u3VV4LkkYXMjxUtbnK2uj71mkHb2T22iWFxvtAnjac4rwQA5itbZb9VahYM4XRbvpEEyxdr,552,4,0,true +2024-12-02 20:48:46.000 UTC,2024-12-02,2024-12-01,Tradewiz,solana,174.81549384187,Buy,14986480.13779,FAITH,CK3TMoqHy5ApzRwooxbxMAbpGDK2w6KpQMdv2rDhpump,0.782137237,SOL,So11111111111111111111111111111111111111112,1.6763249999999998,0.0075,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-FAITH,5PBojHVrS9C6F6kjTosfVEUoBzED8PCxpFdeouWzDQbd,2kNoYHNtpaefWZ3JptWug7H7pyzXZXYYTQufNbnt3Zzy,3gPFS1BtQArtTN2hZwbrUxKETHiHRts4YCcCEZqFxoGo9E8zpYztsjdtNsUmbK14qNT7VdieNJXbY9tBTembavCB,413,7,4,true +2024-12-02 18:06:48.000 UTC,2024-12-02,2024-12-01,Tradewiz,solana,30.016174407599998,Buy,1809787.710499,ART,CuSXYspmP7khJiov8Kr3XaExcuhPadVDHGm3K3rvpump,0.13517146,SOL,So11111111111111111111111111111111111111112,0.29361128496,0.001322216,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-ART,GD5CvxQZwKLfqMpvMTeLkTecnMVZGNPXnUk597W1j8hb,H9kHjTaJqVrrSuZGwJWgQnjyhVmGANpqsA6eoqbbZrTk,2SZ1av6twfq14Um3gM8KuRopdqdJpDtESqidS725tKMiDMa1zQu2wnr76JMsbXZMrH5LiUyGv9XPUqMkmwbjW4S9,2628,5,4,true +2024-12-01 23:25:30.000 UTC,2024-12-01,2024-12-01,Tradewiz,solana,12.139543894200001,Buy,1715893.906365,Bio AI,7LL7cbEfKLMiH4oUmfnfYgAhUnnVRWGeksE6iXjDpump,0.050942274,SOL,So11111111111111111111111111111111111111112,0.2383,0.001,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Bio AI,76BfryV9K9e9o8zF69csrqFLaar2PpdYZ8hN7ZRE4Y3e,H5PQNrAj7uKYK44ksRqym8sAfuasNC4xHJZW88WbzDyT,3YXaTrff9qmYHko2ZD5veVtAcF7XfJa8eCmFBmMtV5dLdowbCn2D2ArfkymQ2ZLon5ExhEYr5qbM54JJbE4NiZtU,475,7,4,true +2024-12-03 20:40:46.000 UTC,2024-12-03,2024-12-01,Tradewiz,solana,148.38664547177999,Buy,17917109.708311,WILDMNKY,75fv3J8gw9mp64ZVcAXjmr1qT8PqJjE1Cc4ocjmCpump,0.645552273,SOL,So11111111111111111111111111111111111111112,1.3791600000000002,0.006,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,WILDMNKY-SOL,7SV8yk5hXtcXNqgohc9tJMdmAbBnLpbKr8oURHdh8s8g,4Nw2v6XZJgaWHmtETqMWtQyTHaGDtQZ27x3otcvjQAHT,ep8K6Nvu25kw1yVWqENPHv2Y4mGRW5T7kMBxeXnmujd6WoDxzSfVtqASa2fz5toNTyX1JDWecAoMHxTdEsXnkxU,601,7,4,true +2024-12-03 00:15:12.000 UTC,2024-12-03,2024-12-01,Tradewiz,solana,5.12252529562,Sell,0.022631993,SOL,So11111111111111111111111111111111111111112,27129.461569,anthrupad,2okJ4wqqDFFD2Tce3QaU8L866KxBJzJXVoMtLR1ppump,0.052636498700000006,0.000232555,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-anthrupad,FAfDgkWFexH1BA2D1i5pc2uRR8ZYcvt8jY84CK7e9My8,31M2cNExVg8bwB2HwzAzA1w9pHwLeP67ejygZ44UB1UR,2HryhZK2dRLqZeTowRQ4Rd7BGwL9ekAgyjDuaFvYAJyyVZHn5C81cTGGc66eMRg9MhiiWEdU7j1yWKfJDjdfbMRU,499,4,0,true +2024-12-07 18:47:05.000 UTC,2024-12-07,2024-12-01,Tradewiz,solana,47.17441114759,Sell,0.195040357,SOL,So11111111111111111111111111111111111111112,12392.36946,AFITR,83Ur4XYc5QyRNBshoW2Yu9cCLyNDV9s5krBu11JWDxxq,0.47612907671000004,0.001968533,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-AFITR,5JtbDjPqfN6K4mDF2c5rvJZXXryQbihjViSP2XoMohSV,8uAied3jRqG1mPRzqsmLaeRDz3F22BkarmfgAvoqNWN,3Lr37rTZUVk9m1iYz3h8sg6Dt4dMHm4PhLmuPUe7G1Z3tMxraN54M8boZMP5wdXGqs8NGQPHMDaXLZoKAq1zMZLE,238,4,0,true +2024-12-01 18:04:29.000 UTC,2024-12-01,2024-12-01,Tradewiz,solana,32.99628642063,Buy,438388.5076,Metanoia,2dPFQnktvPKZe3hjkstWfJx8tDrH8FW9xoiBjZMEpump,0.139218963,SOL,So11111111111111111111111111111111111111112,0.2986326,0.00126,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Metanoia,FpTruBXx6UcyBiVr2Rj7UMf9tNTpJgTyGpXynpuo9HzA,GhqrMrC4tY8zTDvWHiHG7ag4MWPqQ7qLZmeochY6aosY,29TPJAT4vP7rUmMETneBknCmH8UpzdeFaB6ZpJTmzVobE6kTou8KBgqvPPLb588ettce6HRVFUPcSbuSrRJpJnzG,620,5,4,true +2024-12-01 02:27:09.000 UTC,2024-12-01,2024-12-01,Tradewiz,solana,19.69581542709,Sell,0.083524089,SOL,So11111111111111111111111111111111111111112,313146.63071,GSC,F86FtKmg8vnaCfDoJF7U21NbmLprVazE3L2PshmCpump,0.26366694273,0.001118133,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GSC,7H6MS1nqg9Y6NN8srD9UCovDQwVxuZaozgXrMT7kcH61,35cx6mhp2Nsi33d7QfwnqVrYoEcyWpc17PtgMrCnwfxt,5K5isZKhK8mwM4rZ7akSejvwByViNxZ5GcCMcsqWENmT42WkJ4kUoYsAJQkeq73pwiEBgjwEoUEgRNB8jCt9Bugp,541,4,0,true +2024-12-04 17:48:52.000 UTC,2024-12-04,2024-12-01,Tradewiz,solana,29.19816884426,Sell,0.128270302,SOL,So11111111111111111111111111111111111111112,1239313.36229,Pretty,8pTpjjcq5pG7NXDm99NinoT1vRoH99X7s3C1Go3upump,0.30598138415,0.001344205,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Pretty,HBiCDY9CVNWENM8vTkHekJoNAeaLZAJoHr7EwvmAf5WY,GhqrMrC4tY8zTDvWHiHG7ag4MWPqQ7qLZmeochY6aosY,53KrU2unKY95pnSKSq3empC6woBXPxQShDv3EmpDNnn1knesLfxHtCetCkbuNPRWPXr3tj3WKuu8B63Ypawx8yvb,977,3,2,true +2024-12-04 04:50:48.000 UTC,2024-12-04,2024-12-01,Tradewiz,solana,77.97821513972,Buy,1428724.561159,eigenrobot,9qpaeuNMNWwHfa7keQ51D5VJNt2jessw8khxWis5pump,0.327832402,SOL,So11111111111111111111111111111111111111112,0.7135800000000001,0.003,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-eigenrobot,7Ys5V2wmjz3LihubgfnCdSZnwEUvBNBXXLvpAWZLRiS5,5q853P8PLeYiE5PjXg24kQ9nXrf5Qn6r55A9urS4hYF6,2XPV2bKhVjYGjuLGWhxjUK69mPFqyfFYcBM6HTSnErYLxHP1cNSksWp4i3VbF7QesYJtQnPvtt5aixEbkr137nz1,959,7,4,true +2024-12-04 13:04:31.000 UTC,2024-12-04,2024-12-01,Tradewiz,solana,0.00010256337,Sell,4.41e-7,SOL,So11111111111111111111111111111111111111112,117382.719111186,FCI,7s9q7HcfMhhf4XoFtHUzJWyAPAN3mSEmBDfB17iM67QJ,9.3028e-7,4e-9,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FCI,8AfbS9Ji49Jj4UkZmm4whYACyUN8QQQa1dsj9JuYQjFs,EcpfuULFUmouh7XU4uESrksywAwoND2PBu7T3LJwXJmM,4N2Fw8hjiaKSBHnkc7nbH4C6TbjFtpE5RHas2uQKkJvNVUSbwUuTST6GhTueLsrMJe4ho9Sr13xm3V1QQtNzGcSf,460,4,0,true +2024-12-04 20:35:20.000 UTC,2024-12-04,2024-12-01,Tradewiz,solana,46.792,Buy,66517.719092,convo,KnrmAHo1wW2fGqVXcSZSpo6kmiKam5vNbDByrURpump,0.2,SOL,So11111111111111111111111111111111111111112,0.46792,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-convo,Exy58ud7t8LHN1EBnrcRRwRQmz1uSUC3hQF9oHpM1Box,5CEbKj8WbqKkah7nhYxkXGvN97xvz5hxd8fQjYnW5NTV,rNFm4bvTJvPVtqQGoBhyFBPhFbJftPVxoasZccwEhaaJqBA4GMVurr3aJTexbDm4p673RSnpTHpnmxLshHmGEK6,38,5,0,true +2024-12-06 07:59:32.000 UTC,2024-12-06,2024-12-01,Tradewiz,solana,47.796,Buy,8548.498569,PIZZAGUY,5cN4dWZKqxrpwCCBcwHpsAgus1s7WWyRaguR8YbRiECY,0.2,SOL,So11111111111111111111111111111111111111112,0.47796,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-PIZZAGUY,FgYufZ6c7NpcMvRnbB6tYupsEoyiPCtBw9dZ3bVo8kHE,1EjTzPniAGBxzvi4TBfocfq4Zxuj2GCK42aWnNJEoSS,619fmC5opZwwwAnnci5snSqtVcp9cqPYW2SAiwq8DZQrPVS8F1cLDZW9fzwxdjBYbBKZtBTeMyA9rjeCkLU1U7uE,1700,6,0,true From fbbdd8ce00693d16db675610dafd7d10d7c77001 Mon Sep 17 00:00:00 2001 From: intensodefi Date: Fri, 13 Dec 2024 03:01:50 +0700 Subject: [PATCH 31/59] Add Levinswap to dex.trades (#7315) * Update dex_info.sql * Update _schema.yml * Update dex_gnosis_base_trades.sql * Create levinswap_gnosis_base_trades.sql * Update _schema.yml * Create levinswap_gnosis_base_trades_seed.csv * Update _sources.yml --------- Co-authored-by: Huang Geyang --- dbt_subprojects/dex/models/dex_info.sql | 1 + .../dex/models/trades/gnosis/_schema.yml | 16 ++++++++++++++ .../trades/gnosis/dex_gnosis_base_trades.sql | 1 + .../levinswap_gnosis_base_trades.sql | 21 +++++++++++++++++++ dbt_subprojects/dex/seeds/trades/_schema.yml | 15 +++++++++++++ .../levinswap_gnosis_base_trades_seed.csv | 2 ++ .../_sector/dex/trades/gnosis/_sources.yml | 4 ++++ 7 files changed, 60 insertions(+) create mode 100644 dbt_subprojects/dex/models/trades/gnosis/platforms/levinswap_gnosis_base_trades.sql create mode 100644 dbt_subprojects/dex/seeds/trades/levinswap_gnosis_base_trades_seed.csv diff --git a/dbt_subprojects/dex/models/dex_info.sql b/dbt_subprojects/dex/models/dex_info.sql index f5ffa82a7d7..2de3399cc3c 100644 --- a/dbt_subprojects/dex/models/dex_info.sql +++ b/dbt_subprojects/dex/models/dex_info.sql @@ -191,4 +191,5 @@ FROM (VALUES , ('akronswap', 'Akronswap', 'Direct', 'AkronFinance') , ('katana', 'Katana', 'Direct', 'AxieInfinity') , ('jetswap', 'JetSwap', 'Direct', 'Jetfuelfinance') + , ('levinswap', 'Levinswap', 'Direct', 'levinswap') ) AS temp_table (project, name, marketplace_type, x_username) diff --git a/dbt_subprojects/dex/models/trades/gnosis/_schema.yml b/dbt_subprojects/dex/models/trades/gnosis/_schema.yml index 1ac8909ff33..2dee59ac657 100644 --- a/dbt_subprojects/dex/models/trades/gnosis/_schema.yml +++ b/dbt_subprojects/dex/models/trades/gnosis/_schema.yml @@ -117,3 +117,19 @@ models: - check_dex_base_trades_seed: seed_file: ref('elk_finance_gnosis_base_trades_seed') + - name: levinswap_gnosis_base_trades + meta: + blockchain: gnosis + sector: dex + project: levinswap + contributors: intensodefi + config: + tags: ['gnosis', 'dex', 'trades', 'levinswap'] + description: "levinswap gnosis base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('levinswap_gnosis_base_trades_seed') diff --git a/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql b/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql index 70883d551bb..599b6cdceb4 100644 --- a/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/gnosis/dex_gnosis_base_trades.sql @@ -12,6 +12,7 @@ , ref('balancer_v3_gnosis_base_trades') , ref('honeyswap_v2_gnosis_base_trades') , ref('elk_finance_gnosis_base_trades') + , ref('levinswap_gnosis_base_trades') ] %} WITH base_union AS ( diff --git a/dbt_subprojects/dex/models/trades/gnosis/platforms/levinswap_gnosis_base_trades.sql b/dbt_subprojects/dex/models/trades/gnosis/platforms/levinswap_gnosis_base_trades.sql new file mode 100644 index 00000000000..48b6edd32f9 --- /dev/null +++ b/dbt_subprojects/dex/models/trades/gnosis/platforms/levinswap_gnosis_base_trades.sql @@ -0,0 +1,21 @@ +{{ + config( + schema = 'levinswap_gnosis', + alias = 'base_trades', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{{ + uniswap_compatible_v2_trades( + blockchain = 'gnosis', + project = 'levinswap', + version = '1', + Pair_evt_Swap = source('levinswap_gnosis', 'UniswapV2Pair_evt_Swap'), + Factory_evt_PairCreated = source('levinswap_gnosis', 'UniswapV2Factory_evt_PairCreated') + ) +}} diff --git a/dbt_subprojects/dex/seeds/trades/_schema.yml b/dbt_subprojects/dex/seeds/trades/_schema.yml index 5164c88beac..548192dab81 100644 --- a/dbt_subprojects/dex/seeds/trades/_schema.yml +++ b/dbt_subprojects/dex/seeds/trades/_schema.yml @@ -4559,3 +4559,18 @@ seeds: token_bought_amount_raw: uint256 token_sold_amount_raw: uint256 block_date: timestamp + + - name: levinswap_gnosis_base_trades_seed + config: + column_types: + blockchain: varchar + project: varchar + version: varchar + tx_hash: varbinary + evt_index: uint256 + block_number: uint256 + token_bought_address: varbinary + token_sold_address: varbinary + token_bought_amount_raw: uint256 + token_sold_amount_raw: uint256 + block_date: timestamp diff --git a/dbt_subprojects/dex/seeds/trades/levinswap_gnosis_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/levinswap_gnosis_base_trades_seed.csv new file mode 100644 index 00000000000..614606ee402 --- /dev/null +++ b/dbt_subprojects/dex/seeds/trades/levinswap_gnosis_base_trades_seed.csv @@ -0,0 +1,2 @@ +blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw +gnosis,levinswap,1,2024-11-22,0x92f310ece1e89b668d91ef9993a2806e4a0d2e4bb2b2803701c05cb455cff397,8,0xc1014b2ad0a731a28c0b74890ad3c50837faa309,0xddafbb505ad214d7b80b1f830fccc89b60fb7a83,37159382,10000000000000000,508965 diff --git a/sources/_sector/dex/trades/gnosis/_sources.yml b/sources/_sector/dex/trades/gnosis/_sources.yml index 821fa073bb9..c040eebe4a1 100644 --- a/sources/_sector/dex/trades/gnosis/_sources.yml +++ b/sources/_sector/dex/trades/gnosis/_sources.yml @@ -20,3 +20,7 @@ sources: tables: - name: ElkPair_evt_Swap - name: ElkFactory_evt_PairCreated + - name: levinswap_gnosis + tables: + - name: UniswapV2Pair_evt_Swap + - name: UniswapV2Factory_evt_PairCreated From ddb63e185f018c4850c4d1892fae8538ef5e254f Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:02:02 +0000 Subject: [PATCH 32/59] add erc4626 token pricing to balancer.trades (#7301) * add erc4626 token pricing to balancer.trades * fix copypasta * add missing comma * remove extra comma * remove extra comma * fix erc4626_token_prices * fix group by * fix wrong reference --- .../ethereum/balancer_v3_ethereum_trades.sql | 26 ++++++++++++++++--- .../gnosis/balancer_v3_gnosis_trades.sql | 26 ++++++++++++++++--- .../spellbook/_sources.yml | 3 +++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql index 71d7257a356..ad927febd4c 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/ethereum/balancer_v3_ethereum_trades.sql @@ -91,6 +91,18 @@ WITH AND bpt_prices.day <= DATE_TRUNC('day', dexs.block_time) AND bpt_prices.blockchain = 'ethereum' GROUP BY 1, 2, 3, 4, 5 + ), + + erc4626_prices AS( + SELECT + minute, + wrapped_token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + LEAD(minute, 1, NOW()) OVER (PARTITION BY wrapped_token ORDER BY minute) AS time_of_next_change + FROM {{ source('balancer_v3', 'erc4626_token_prices') }} + WHERE blockchain = 'ethereum' + GROUP BY 1, 2, 3 ) SELECT @@ -110,8 +122,8 @@ SELECT dexs.token_sold_amount_raw, COALESCE( dexs.amount_usd, - dexs.token_bought_amount_raw / POWER(10, COALESCE(erc20a.decimals, 18)) * bpa_bpt_prices.bpt_price, - dexs.token_sold_amount_raw / POWER(10, COALESCE(erc20b.decimals, 18)) * bpb_bpt_prices.bpt_price + dexs.token_bought_amount_raw / POWER(10, COALESCE(erc20a.decimals, erc4626a.decimals, 18)) * COALESCE(bpa_bpt_prices.bpt_price, erc4626a.price), + dexs.token_sold_amount_raw / POWER(10, COALESCE(erc20b.decimals, erc4626b.decimals, 18)) * COALESCE(bpb_bpt_prices.bpt_price, erc4626b.price) ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, @@ -148,4 +160,12 @@ FROM dexs LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpb_bpt_prices ON bpb_bpt_prices.contract_address = bpb.contract_address AND bpb_bpt_prices.day = bpb.bpb_max_block_date - AND bpb_bpt_prices.blockchain = 'ethereum' \ No newline at end of file + AND bpb_bpt_prices.blockchain = 'ethereum' + LEFT JOIN erc4626_prices erc4626a + ON erc4626a.wrapped_token = dexs.token_bought_address + AND erc4626a.minute <= dexs.block_time + AND dexs.block_time < erc4626a.time_of_next_change + LEFT JOIN erc4626_prices erc4626b + ON erc4626b.wrapped_token = dexs.token_sold_address + AND erc4626b.minute <= dexs.block_time + AND dexs.block_time < erc4626b.time_of_next_change \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql index 71b73f5fa60..7aea4bcc382 100644 --- a/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql +++ b/dbt_subprojects/dex/models/_projects/balancer/trades/gnosis/balancer_v3_gnosis_trades.sql @@ -91,6 +91,18 @@ WITH AND bpt_prices.day <= DATE_TRUNC('day', dexs.block_time) AND bpt_prices.blockchain = 'gnosis' GROUP BY 1, 2, 3, 4, 5 + ), + + erc4626_prices AS( + SELECT + minute, + wrapped_token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + LEAD(minute, 1, NOW()) OVER (PARTITION BY wrapped_token ORDER BY minute) AS time_of_next_change + FROM {{ source('balancer_v3', 'erc4626_token_prices') }} + WHERE blockchain = 'gnosis' + GROUP BY 1, 2, 3 ) SELECT @@ -110,8 +122,8 @@ SELECT dexs.token_sold_amount_raw, COALESCE( dexs.amount_usd, - dexs.token_bought_amount_raw / POWER(10, COALESCE(erc20a.decimals, 18)) * bpa_bpt_prices.bpt_price, - dexs.token_sold_amount_raw / POWER(10, COALESCE(erc20b.decimals, 18)) * bpb_bpt_prices.bpt_price + dexs.token_bought_amount_raw / POWER(10, COALESCE(erc20a.decimals, erc4626a.decimals, 18)) * COALESCE(bpa_bpt_prices.bpt_price, erc4626a.price), + dexs.token_sold_amount_raw / POWER(10, COALESCE(erc20b.decimals, erc4626b.decimals, 18)) * COALESCE(bpb_bpt_prices.bpt_price, erc4626b.price) ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, @@ -148,4 +160,12 @@ FROM dexs LEFT JOIN {{ source('balancer', 'bpt_prices') }} bpb_bpt_prices ON bpb_bpt_prices.contract_address = bpb.contract_address AND bpb_bpt_prices.day = bpb.bpb_max_block_date - AND bpb_bpt_prices.blockchain = 'gnosis' \ No newline at end of file + AND bpb_bpt_prices.blockchain = 'gnosis' + LEFT JOIN erc4626_prices erc4626a + ON erc4626a.wrapped_token = dexs.token_bought_address + AND erc4626a.minute <= dexs.block_time + AND dexs.block_time < erc4626a.time_of_next_change + LEFT JOIN erc4626_prices erc4626b + ON erc4626b.wrapped_token = dexs.token_sold_address + AND erc4626b.minute <= dexs.block_time + AND dexs.block_time < erc4626b.time_of_next_change \ No newline at end of file diff --git a/sources/_subprojects_outputs/spellbook/_sources.yml b/sources/_subprojects_outputs/spellbook/_sources.yml index 65c11b935de..9aa040b2036 100644 --- a/sources/_subprojects_outputs/spellbook/_sources.yml +++ b/sources/_subprojects_outputs/spellbook/_sources.yml @@ -116,6 +116,9 @@ sources: - name: balancer_v3_gnosis tables: - name: bpt_prices + - name: balancer_v3 + tables: + - name: erc4626_token_prices - name: addresses_ethereum tables: From 427f6748d2f84829a683df033d6acd1677f6f06a Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Thu, 12 Dec 2024 21:02:11 +0100 Subject: [PATCH 33/59] Cowprotocol/add base (#7295) * [cowprotocol] Add Base tables * add union in batches * remove comment * fix solver table reference * ad base sources * add another missing base reference * retrigger CI * fix solver query * remove batches changes --- .../models/_projects/cow_protocol/_schema.yml | 107 +++++++- .../base/cow_protocol_base_trades.sql | 257 ++++++++++++++++++ .../cow_protocol/cow_protocol_trades.sql | 30 ++ sources/_subprojects_outputs/dex/_sources.yml | 11 +- .../base/cow_protocol_base_sources.yml | 160 +++++++++++ 5 files changed, 550 insertions(+), 15 deletions(-) create mode 100644 dbt_subprojects/dex/models/_projects/cow_protocol/base/cow_protocol_base_trades.sql create mode 100644 sources/cow_protocol/base/cow_protocol_base_sources.yml diff --git a/dbt_subprojects/dex/models/_projects/cow_protocol/_schema.yml b/dbt_subprojects/dex/models/_projects/cow_protocol/_schema.yml index 958a9d028d5..9e0e203f3ef 100644 --- a/dbt_subprojects/dex/models/_projects/cow_protocol/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/cow_protocol/_schema.yml @@ -8,18 +8,26 @@ models: project: cow_protocol contributors: bh2smith, gentrexha config: - tags: ['ethereum','cow_protocol','trades','dex','aggregator','cross-chain'] + tags: + [ + "ethereum", + "cow_protocol", + "trades", + "dex", + "aggregator", + "cross-chain", + ] description: > - CoW Protocol trades on all chains across all contracts and versions. - This table will load dex trades downstream. - This particular version of the protocol trades table is intended to conform - with the columns as specified in dex.trades sector. - A more detailed version of the protocol trades table can be found under `cow_protocol_ethereum.trades` + CoW Protocol trades on all chains across all contracts and versions. + This table will load dex trades downstream. + This particular version of the protocol trades table is intended to conform + with the columns as specified in dex.trades sector. + A more detailed version of the protocol trades table can be found under `cow_protocol_ethereum.trades` columns: - &blockchain name: blockchain description: "Blockchain which the project is deployed" - - &project + - &project name: project description: "Project name" - &version @@ -92,7 +100,7 @@ models: project: cow_protocol contributors: olgafetisova config: - tags: ['arbitrum','cow_protocol','trades', 'dex', 'aggregator'] + tags: ["arbitrum", "cow_protocol", "trades", "dex", "aggregator"] description: > CoW Protocol enriched trades list on Arbitrum Chain data_tests: @@ -169,7 +177,7 @@ models: project: cow_protocol contributors: bh2smith, gentrexha config: - tags: ['ethereum','cow_protocol','trades', 'dex', 'aggregator'] + tags: ["ethereum", "cow_protocol", "trades", "dex", "aggregator"] description: > CoW Protocol enriched trades list on Ethereum data_tests: @@ -253,7 +261,7 @@ models: project: cow_protocol contributors: bh2smith config: - tags: ['gnosis','cow_protocol','trades', 'dex', 'aggregator'] + tags: ["gnosis", "cow_protocol", "trades", "dex", "aggregator"] description: > CoW Protocol enriched trades list on Gnosis Chain data_tests: @@ -322,4 +330,81 @@ models: - name: flags description: "bitmap with additional trade information (cf. https://github.com/cowprotocol/contracts/blob/main/src/contracts/libraries/GPv2Trade.sol#L58-L94)" - name: surplus_usd - description: "USD value of surplus (difference between limit and executed price)" \ No newline at end of file + description: "USD value of surplus (difference between limit and executed price)" + + - name: cow_protocol_base_trades + meta: + blockchain: base + project: cow_protocol + contributors: felix + config: + tags: ["base", "cow_protocol", "trades", "dex", "aggregator"] + description: > + CoW Protocol enriched trades list on Base + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - order_uid + - evt_index + columns: + - name: block_date + description: "UTC event block date of each trade" + - name: block_time + description: "Timestamp for block event time in UTC" + - name: tx_hash + description: "Transaction hash of trade" + - name: evt_index + description: "Index of the corresponding trade event" + - name: order_uid + description: "Unique identifier of order involved in trade. Note that partially fillable orders can be touched multiple times so this is not a unique ID for trade events." + - name: trader + description: "Owner of the order being traded (aka trader)" + - name: sell_token_address + description: "Ethereum address of sellToken" + - name: sell_token + description: "Symbol of sellToken" + - name: buy_token_address + description: "Ethereum address of buyToken" + - name: buy_token + description: "Symbol of buyToken" + - name: token_pair + description: "Ordered concatenation of buy/sell token symbols involved in trade" + - name: units_sold + description: "Units of sellToken sold (incorporates token decimals)." + - name: atoms_sold + description: "Amount (in atoms) of the sellToken sold" + - name: units_bought + description: "Units of buyToken bought (incorporates token decimals)." + - name: atoms_bought + description: "Amount (in atoms) of the buyToken bought" + - name: usd_value + description: "USD value of trade (taken as the max of buy value, sell value when both available, otherwise whichever is known)" + - name: buy_price + description: "USD price of buyToken at the time of trade" + - name: buy_value_usd + description: "USD value of buyToken amount" + - name: sell_price + description: "USD price of sellToken at the time of trade" + - name: sell_value_usd + description: "USD value of sellToken amount" + - name: fee + description: "Unit of fee taken (in sellToken) - incorporating token decimals" + - name: fee_atoms + description: "Amount (in atoms) of fee - taken in sellToken" + - name: fee_usd + description: "USD value of trade fee" + - name: app_data + description: "Hashed metadata related to trade event (full content available on IPFS)" + - name: receiver + description: "Recipient of trades buyToken" + - name: limit_sell_amount + description: "Sell amount side of the limit price" + - name: limit_buy_amount + description: "Buy amount side of the limit price" + - name: valid_to + description: "epoch timestamp (in seconds) until when the order was valid" + - name: flags + description: "bitmap with additional trade information (cf. https://github.com/cowprotocol/contracts/blob/main/src/contracts/libraries/GPv2Trade.sol#L58-L94)" + - name: surplus_usd + description: "USD value of surplus (difference between limit and executed price)" diff --git a/dbt_subprojects/dex/models/_projects/cow_protocol/base/cow_protocol_base_trades.sql b/dbt_subprojects/dex/models/_projects/cow_protocol/base/cow_protocol_base_trades.sql new file mode 100644 index 00000000000..449e1b0f3d5 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/cow_protocol/base/cow_protocol_base_trades.sql @@ -0,0 +1,257 @@ +{{ config( + schema='cow_protocol_base', + alias='trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['tx_hash', 'order_uid', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + post_hook='{{ expose_spells(\'["base"]\', + "project", + "cow_protocol", + \'["felix"]\') }}' + ) +}} + +WITH +-- First subquery joins buy and sell token prices from prices.usd. +-- Also deducts fee from sell amount. +trades_with_prices AS ( + SELECT cast(date_trunc('day', evt_block_time) as date) as block_date, + cast(date_trunc('month', evt_block_time) as date) as block_month, + evt_block_time as block_time, + evt_block_number as block_number, + evt_tx_hash as tx_hash, + evt_index, + trade.contract_address as project_contract_address, + owner as trader, + orderUid as order_uid, + sellToken as sell_token, + buyToken as buy_token, + sellAmount - feeAmount as sell_amount, + buyAmount as buy_amount, + feeAmount as fee_amount, + ps.price as sell_price, + pb.price as buy_price + FROM {{ source('gnosis_protocol_v2_base', 'GPv2Settlement_evt_Trade') }} trade + LEFT OUTER JOIN {{ source('prices', 'usd') }} as ps + ON sellToken = ps.contract_address + AND ps.minute = date_trunc('minute', evt_block_time) + AND ps.blockchain = 'base' + {% if is_incremental() %} + AND {{ incremental_predicate('ps.minute') }} + {% endif %} + LEFT OUTER JOIN {{ source('prices', 'usd') }} as pb + ON pb.contract_address = ( + CASE + WHEN buyToken = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + THEN 0x4200000000000000000000000000000000000006 + ELSE buyToken + END) + AND pb.minute = date_trunc('minute', evt_block_time) + AND pb.blockchain = 'base' + {% if is_incremental() %} + AND {{ incremental_predicate('pb.minute') }} + {% endif %} + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} +), +-- Second subquery gets token symbol and decimals from tokens.erc20 (to display units bought and sold) +trades_with_token_units as ( + SELECT block_date, + block_month, + block_time, + block_number, + tx_hash, + evt_index, + project_contract_address, + order_uid, + trader, + sell_token as sell_token_address, + (CASE + WHEN ts.symbol IS NULL THEN cast(sell_token as varchar) + ELSE ts.symbol + END) as sell_token, + buy_token as buy_token_address, + (CASE + WHEN tb.symbol IS NULL THEN cast(buy_token as varchar) + WHEN buy_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 'ETH' + ELSE tb.symbol + END) as buy_token, + sell_amount / pow(10, ts.decimals) as units_sold, + sell_amount as atoms_sold, + buy_amount / pow(10, tb.decimals) as units_bought, + buy_amount as atoms_bought, + -- We use sell value when possible and buy value when not + fee_amount / pow(10, ts.decimals) as fee, + fee_amount as fee_atoms, + sell_price, + buy_price + FROM trades_with_prices + LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ts + ON ts.blockchain='base' AND ts.contract_address = sell_token + LEFT OUTER JOIN {{ source('tokens', 'erc20') }} tb + ON tb.blockchain='base' AND tb.contract_address = + (CASE + WHEN buy_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + THEN 0x4200000000000000000000000000000000000006 + ELSE buy_token + END) +), +sorted_orders as ( + select + evt_tx_hash, + evt_block_number, + array_agg(orderUid order by evt_index) as order_ids + from ( + select + evt_tx_hash, + evt_index, + evt_block_number, + orderUid + from {{ source('gnosis_protocol_v2_base', 'GPv2Settlement_evt_Trade') }} + {% if is_incremental() %} + where {{ incremental_predicate('evt_block_time') }} + {% endif %} + ) + group by evt_tx_hash, evt_block_number +), + +orders_and_trades as ( + select + evt_tx_hash, + trades, + order_ids + from sorted_orders + inner join {{ source('gnosis_protocol_v2_base', 'GPv2Settlement_call_settle') }} + on evt_block_number = call_block_number + and evt_tx_hash = call_tx_hash +), + +uid_to_app_id as ( + SELECT + distinct uid, + evt_tx_hash as hash, + from_hex(JSON_EXTRACT_SCALAR(trade, '$.appData')) AS app_data, + from_hex(JSON_EXTRACT_SCALAR(trade, '$.receiver')) AS receiver, + cast(JSON_EXTRACT_SCALAR(trade, '$.sellAmount') as uint256) AS limit_sell_amount, + cast(JSON_EXTRACT_SCALAR(trade, '$.buyAmount') as uint256) AS limit_buy_amount, + date_format( + from_unixtime(cast(JSON_EXTRACT_SCALAR(trade, '$.validTo') as double)), + '%Y-%m-%d %T' + ) AS valid_to, + cast(JSON_EXTRACT_SCALAR(trade, '$.flags') as integer) AS flags + FROM + orders_and_trades + CROSS JOIN UNNEST (order_ids) + WITH + ORDINALITY AS o (uid, i) + CROSS JOIN UNNEST (trades) + WITH + ORDINALITY AS t (trade, j) + WHERE + i = j +), + +eth_flow_senders as ( + select + sender, + bytearray_concat( + bytearray_concat( + output_orderHash, + bytearray_substring(event.contract_address, 1, 20) + ), + 0xffffffff + ) AS order_uid + from {{ source('cow_protocol_base', 'CoWSwapEthFlow_evt_OrderPlacement') }} event + inner join {{ source('cow_protocol_base', 'CoWSwapEthFlow_call_createOrder') }} call + on call_block_number = evt_block_number + and call_tx_hash = evt_tx_hash + {% if is_incremental() %} + where {{ incremental_predicate('evt_block_time') }} + {% endif %} +), + +valued_trades as ( + SELECT block_date, + block_month, + block_time, + block_number, + tx_hash, + evt_index, + ARRAY[-1] as trace_address, + project_contract_address, + trades.order_uid, + -- ETH Flow orders have trader = sender of orderCreation. + case when sender is not null then sender else trader end as trader, + sell_token_address, + case when sender is not null then 'ETH' else sell_token end as sell_token, + buy_token_address, + buy_token, + case + when lower(buy_token) > lower(sell_token) then concat(sell_token, '-', buy_token) + else concat(buy_token, '-', sell_token) + end as token_pair, + units_sold, + atoms_sold, + units_bought, + atoms_bought, + (CASE + WHEN sell_price IS NOT NULL THEN + -- Choose the larger of two prices when both not null. + CASE + WHEN buy_price IS NOT NULL and buy_price * COALESCE(units_bought,0) > sell_price * COALESCE(units_sold,0) + then buy_price * units_bought + ELSE sell_price * units_sold + END + WHEN sell_price IS NULL AND buy_price IS NOT NULL THEN buy_price * units_bought + END) as usd_value, + buy_price, + buy_price * units_bought as buy_value_usd, + sell_price, + sell_price * units_sold as sell_value_usd, + fee, + fee_atoms, + (CASE + WHEN sell_price IS NOT NULL THEN + CASE + WHEN buy_price IS NOT NULL and buy_price * COALESCE(units_bought,0) > sell_price * COALESCE(units_sold,0) + then buy_price * units_bought * fee / units_sold + ELSE sell_price * fee + END + WHEN sell_price IS NULL AND buy_price IS NOT NULL + THEN buy_price * units_bought * fee / units_sold + END) as fee_usd, + app_data, + case + when receiver = 0x0000000000000000000000000000000000000000 + then trader + else receiver + end as receiver, + limit_sell_amount, + limit_buy_amount, + valid_to, + flags, + case when (flags % 2) = 0 then 'SELL' else 'BUY' end as order_type, + bitwise_and(flags, 2) != 0 as partial_fill, + (CASE + when (flags % 2) = 0 then atoms_sold / limit_sell_amount + else atoms_bought / limit_buy_amount + end + ) as fill_proportion + FROM trades_with_token_units trades + JOIN uid_to_app_id + ON uid = trades.order_uid + AND hash=tx_hash + LEFT OUTER JOIN eth_flow_senders efs + ON trades.order_uid = efs.order_uid +) + +select *, + -- Relative surplus (in %) is the difference between limit price and executed price as a ratio of the limit price. + -- Absolute surplus (in USD) is relative surplus multiplied with the value of the trade + usd_value * (atoms_bought * limit_sell_amount - atoms_sold * limit_buy_amount) / (atoms_bought * limit_sell_amount) as surplus_usd +from valued_trades diff --git a/dbt_subprojects/dex/models/_projects/cow_protocol/cow_protocol_trades.sql b/dbt_subprojects/dex/models/_projects/cow_protocol/cow_protocol_trades.sql index aa731eafcf3..1b76dbdc8f0 100644 --- a/dbt_subprojects/dex/models/_projects/cow_protocol/cow_protocol_trades.sql +++ b/dbt_subprojects/dex/models/_projects/cow_protocol/cow_protocol_trades.sql @@ -96,4 +96,34 @@ FROM trace_address, evt_index FROM {{ ref('cow_protocol_arbitrum_trades') }} + + UNION ALL + + + SELECT + 'base' AS blockchain, + 'cow_protocol' AS project, + '1' AS version, + block_date, + block_month, + block_time, + buy_token AS token_bought_symbol, + sell_token AS token_sold_symbol, + token_pair, + units_bought AS token_bought_amount, + units_sold AS token_sold_amount, + atoms_bought AS token_bought_amount_raw, + atoms_sold AS token_sold_amount_raw, + usd_value AS amount_usd, + buy_token_address AS token_bought_address, + sell_token_address AS token_sold_address, + trader AS taker, + CAST(NULL AS VARBINARY) AS maker, + project_contract_address, + tx_hash, + trader AS tx_from, + receiver AS tx_to, + trace_address, + evt_index + FROM {{ ref('cow_protocol_base_trades') }} ) diff --git a/sources/_subprojects_outputs/dex/_sources.yml b/sources/_subprojects_outputs/dex/_sources.yml index 82f13efac55..c99a0f0dd97 100644 --- a/sources/_subprojects_outputs/dex/_sources.yml +++ b/sources/_subprojects_outputs/dex/_sources.yml @@ -62,10 +62,10 @@ sources: - name: balancer_v2_pools_optimism - name: balancer_v2_pools_polygon - name: balancer_v2_pools_zkevm - - name: balancer_cowswap_amm_pools + - name: balancer_cowswap_amm_pools - name: balancer_v3_pools_ethereum - name: balancer_v3_pools_gnosis - - name: balancer_v3_pools + - name: balancer_v3_pools - name: uniswap_v3_optimism tables: - name: ovm1_pool_mapping @@ -78,9 +78,12 @@ sources: - name: cow_protocol_arbitrum tables: - name: trades + - name: cow_protocol_base + tables: + - name: trades - name: jelly_swap tables: - - name: trades + - name: trades - name: beethoven_x tables: - - name: trades + - name: trades diff --git a/sources/cow_protocol/base/cow_protocol_base_sources.yml b/sources/cow_protocol/base/cow_protocol_base_sources.yml new file mode 100644 index 00000000000..23b8894756e --- /dev/null +++ b/sources/cow_protocol/base/cow_protocol_base_sources.yml @@ -0,0 +1,160 @@ +version: 2 + +sources: + - name: gnosis_protocol_v2_base + description: "Base Chain decoded tables related to CoW Protocol contract" + tables: + - name: GPv2AllowListAuthentication_evt_SolverAdded + description: "GPv2AllowListAuthentication list of events when a Solver was added" + columns: + - &solver + name: solver + description: "Solver wallet address" + - &contract_address + name: contract_address + description: "Base address of contract emitting the event" + - &evt_block_number + name: evt_block_number + description: "Block number which processed the unique transaction hash" + - &evt_block_time + name: evt_block_time + description: "Timestamp for block event time in UTC" + - &evt_index + name: evt_index + description: "Index of the event within transaction" + - &evt_tx_hash + name: evt_tx_hash + description: "Primary key of the transaction" + data_tests: + - not_null + - name: GPv2AllowListAuthentication_evt_SolverRemoved + description: "GPv2AllowListAuthentication list of events when a solver was removed" + columns: + - *solver + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - name: GPv2Settlement_evt_Trade + description: "Trade events emitted by GPv2Settlement" + columns: + - &feeAmount + name: feeAmount + description: "Amount (in sell token) in atoms of the trade fee" + - &buyAmount + name: buyAmount + description: "Amount (in atoms) of the buyToken bought" + - &buyToken + name: buyToken + description: "Contract address of token obtained in trade. 0xeee...ee represents native asset ETH" + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - &orderUid + name: orderUid + description: "Unique identifier of order involved in trade. Note that partially fillable orders can be touched multiple times so this is not a unique ID for trade events." + - &owner + name: owner + description: "Owner of the order being traded (aka trader address)" + - &sellAmount + name: sellAmount + description: "Amount in atoms of the sellToken sold" + - &sellToken + name: sellToken + description: "Contract address of token sold in trade." + - name: GPv2Settlement_call_settle + description: "Contains all calls to the settle method of GPv2Settlement (both successful and failed)" + columns: + - &call_block_number + name: call_block_number + description: "Block number on which the method was called" + - &call_block_time + name: call_block_time + description: "UTC timestamp at which the method was called" + - &call_success + name: call_success + description: "Whether or not the call resulted in a successful transaction" + - &call_trace_address + name: call_trace_address + description: "Trace address of the call" + - &call_tx_hash + name: call_tx_hash + description: "Transaction hash of call (if successful, will result in a real transaction hash)." + - &clearingPrices + name: clearingPrices + description: "An array of relative token prices (matched with `tokens` by index)" + - *contract_address + - &interactions + name: interactions + description: "An array of internal contract interactions" + - &tokens + name: tokens + description: "An array of token addresses (to be matched by index on `clearingPrices`)" + - &trades + name: trades + description: "An array of order data for the trades being executed" + - name: GPv2Settlement_evt_Settlement + description: "Settlement events emitted by GPv2Settlement" + columns: + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - *solver + - name: GPv2Settlement_evt_Interaction + description: "Contract Interaction events emitted by GPv2Settlement" + columns: + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - &selector + name: selector + description: Method ID of the contract being interacted with + - &target + name: target + description: Target contract address of the interaction + - &value + name: value + description: ETH amount (in WEI) being passed into contract interaction + - name: cow_protocol_base + description: "Base decoded tables related to CoW Protocol contract's" + tables: + - name: CoWSwapEthFlow_evt_OrderPlacement + description: "CoWSwapEthFlow Order Placement Events" + columns: + - *contract_address + - *evt_block_number + - *evt_block_time + - *evt_index + - *evt_tx_hash + - &order + name: order + description: "Contains json encoded data for an EthFlowOrder: https://github.com/cowprotocol/ethflowcontract/blob/main/src/libraries/EthFlowOrder.sol#L19-L45" + - &data + name: data + description: "Hex Encoded data containing quoteId and validTo: https://github.com/cowprotocol/ethflowcontract/blob/9c74c8ba36ff9ff3e255172b02454f831c066865/src/CoWSwapEthFlow.sol#L110-L113" + - &sender + name: sender + description: "User who placed the order" + - &signature + name: signature + description: "Onchain EIP1271 signature" + - name: CoWSwapEthFlow_call_createOrder + description: "CoWSwapEthFlow createOrder function calls with input parameters and return value" + columns: + - *call_block_number + - *call_block_time + - *call_success + - *call_trace_address + - *call_tx_hash + - *contract_address + - *order + - &output_orderHash + name: output_orderHash + description: "ETH Flow Order Hash corresponds directly to a GPv2 Order Uid by concatenation with owner = contract_address and validTo = `ffffffff`" From 9f09723151a3c52f948f2acc20101ddf85d9616c Mon Sep 17 00:00:00 2001 From: Peter <157015397+safeintern@users.noreply.github.com> Date: Fri, 13 Dec 2024 04:03:46 +0800 Subject: [PATCH 34/59] Changed activity_start_date for Ronin Safes (#7313) --- .../models/_project/safe/ronin/safe_ronin_safes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/ronin/safe_ronin_safes.sql b/dbt_subprojects/hourly_spellbook/models/_project/safe/ronin/safe_ronin_safes.sql index ac8b5622cdf..1e16b748588 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/ronin/safe_ronin_safes.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/ronin/safe_ronin_safes.sql @@ -16,7 +16,7 @@ ) }} -{% set project_start_date = '2024-06-30' %} +{% set project_start_date = '2021-01-01' %} select 'ronin' as blockchain, From 0d95fe07cc9dba562daff16b33a38ab31403bb37 Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:04:22 +0000 Subject: [PATCH 35/59] fix next_change field on balancer_v3_erc4626_token_prices (#7303) * fix next_change field on balancer_v3_erc4626_token_prices * force run of gnosis liquidity * final * force run of gnosis liquidity * force rerun of balancer_v3_erc4626_token_prices.sql * remove unnecessary files --- .../ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql | 2 +- .../ethereum/balancer_v3_ethereum_erc4626_token_prices.sql | 2 +- .../gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql | 2 +- .../gnosis/balancer_v3_gnosis_erc4626_token_prices.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql index dddfbb06938..5d00d403603 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_mapping.sql @@ -1,7 +1,7 @@ {{ config( schema = 'balancer_v3_ethereum', - alias = 'erc4626_tokens_mapping', + alias = 'erc4626_token_mapping', materialized = 'table', file_format = 'delta' ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql index 99f9c1acac8..e00a572b011 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql @@ -60,6 +60,6 @@ SELECT underlying_token_symbol, decimals, APPROX_PERCENTILE(adjusted_price, 0.5) AS median_price, - LEAD(DATE_TRUNC('day', p.evt_block_time), 1, NOW()) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change + LEAD(p.evt_block_time, 1, CURRENT_DATE + INTERVAL '1' day) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change FROM price_join p GROUP BY 1, 2, 3, 4, 5, 6, 7 \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql index 697c6133ebb..66d94189050 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_mapping.sql @@ -1,7 +1,7 @@ {{ config( schema = 'balancer_v3_gnosis', - alias = 'erc4626_tokens_mapping', + alias = 'erc4626_token_mapping', materialized = 'table', file_format = 'delta' ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql index 9c19ed2da74..34fced964c3 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql @@ -60,6 +60,6 @@ SELECT underlying_token_symbol, decimals, APPROX_PERCENTILE(adjusted_price, 0.5) AS median_price, - LEAD(DATE_TRUNC('day', p.evt_block_time), 1, NOW()) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change + LEAD(p.evt_block_time, 1, CURRENT_DATE + INTERVAL '1' day) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change FROM price_join p GROUP BY 1, 2, 3, 4, 5, 6, 7 \ No newline at end of file From a4018d671e2fe67c7f264c1223dafadb78d26912 Mon Sep 17 00:00:00 2001 From: hildobby Date: Thu, 12 Dec 2024 21:04:41 +0100 Subject: [PATCH 36/59] add swell (#7300) Co-authored-by: Huang Geyang --- .../models/_sector/blobs/ethereum/blobs_submitters.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql b/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql index 765e3e7e0c0..4542d4d8391 100644 --- a/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql +++ b/dbt_subprojects/hourly_spellbook/models/_sector/blobs/ethereum/blobs_submitters.sql @@ -77,4 +77,5 @@ SELECT address , (0xb6cf39ee72e0127e6ea6059e38b8c197227a6ac7, 'Morph', 'MorphL2') , (0x2986bf308d0684ad77cd32ee1c60429e6573b5af, 'R0AR Chain', 'th3r0ar') , (0x2f60a5184c63ca94f82a27100643dbabe4f3f7fd, 'Unichain', 'unichain') + , (0xf854cd5b26bfd73d51236c0122798907ed65b1f2, 'Swell', 'swellnetworkio') ) AS x(address, entity, x_username) \ No newline at end of file From 4a768fedb81f330fef4c34c75b5134f2fd15753f Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:04:58 +0000 Subject: [PATCH 37/59] update balancer.token_whitelist (#7304) --- .../_project/balancer/support/balancer_token_whitelist.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/support/balancer_token_whitelist.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/support/balancer_token_whitelist.sql index 5727e1a0739..a063cb927cd 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/support/balancer_token_whitelist.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/support/balancer_token_whitelist.sql @@ -29,6 +29,8 @@ WITH whitelist_token as ( (0xe91d153e0b41518a2ce8dd3d7944fa863463a97d, 'WXDAI', 'gnosis'), (0x6a023ccd1ff6f2045c3309768ead9e68f978f6e1, 'WETH', 'gnosis'), (0x6c76971f98945ae98dd7d4dfca8711ebea946ea6, 'WSTETH', 'gnosis'), + (0xaf204776c7245bf4147c2612bf6e5972ee483701, 'SDAI', 'gnosis'), + (0x9c58bacc331c9aa871afd802db6379a98e80cedb, 'GNO', 'gnosis'), (0xdAC17F958D2ee523a2206206994597C13D831ec7, 'USDT', 'ethereum'), (0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48, 'USDC', 'ethereum'), (0x6B175474E89094C44Da98b954EedeAC495271d0F, 'DAI', 'ethereum'), From 92948fd8dac0d3fff80b1b6d25d4a52cbcc6a8e9 Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:05:13 +0000 Subject: [PATCH 38/59] Update tokens_ethereum_erc20.sql (#7306) * Update tokens_ethereum_erc20.sql * Update tokens_ethereum_erc20.sql --- .../tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql index 141f98c1e86..56a77d10618 100644 --- a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql @@ -4736,5 +4736,7 @@ FROM (VALUES , ( 0x358d94b5b2F147D741088803d932Acb566acB7B6, 'rSWELL', 18) , ( 0x9Ed15383940CC380fAEF0a75edacE507cC775f22, 'earnETH', 18) , ( 0x66E47E6957B85Cf62564610B76dD206BB04d831a, 'earnBTC', 8) - , (0x9Ba021B0a9b958B5E75cE9f6dff97C7eE52cb3E6, 'apxETH', 18) + , ( 0x9Ba021B0a9b958B5E75cE9f6dff97C7eE52cb3E6, 'apxETH', 18) + , ( 0xa4f6b2cdcb67bf8f48baa459a333d2667aa4b100, 'shezETH', 8) + , ( 0x7F4B66FF703336CfC35b901144614496Ae0b0D27, 'shezUSD', 18) ) AS temp_table (contract_address, symbol, decimals) From bf8b448233c1169ac2d62b22f3ce2c106f4d31c1 Mon Sep 17 00:00:00 2001 From: Phu <33654804+lequangphu@users.noreply.github.com> Date: Fri, 13 Dec 2024 03:05:43 +0700 Subject: [PATCH 39/59] Bridges of Bungee = {contract: SocketGateway, event: SocketBridge} (#7255) * Add Odos on Arbitrum to dex_aggregator * Change to WETH address on Arbitrum * Add sources of Odos on Arbitrum * Add odos/arbitrum seeds * Change data type * Add bridges of LiFi across chains * replace tests which is deprecated * fix name error of avalanche source * add index to unique test * fix concat issue * fix unique test of the main model * try to fix unique key * use surrogate key for data tests * use generate_surrogate_key inside the models * fix and remove redundant schema properties * data_tests * move generate_surrogate_key to schema.yml * 1. create macro lifi_extract_bridge_data_macro.sql and apply it in the models 2. update the models' config 3. flip materialization strategy * data_ again * fix naming of the main model * replace avalanche with avalanche_c * avalanche_c again * 1. add block_date column to the macro to use it in the add_tx_columns macro 2. update model of avalanche_c * rename columns to use add_tx_columns macro * remove evt_ * a few minor changes * add amount_usd column * data_tests again * map native token to wrapped token to avoid null price data * update the main model and schema * move type conversion from models to macro * fix source name * varbinary type doesn't need single quotes * correct collumn name * add post_hook * feat: add bungee bridges * create macro and models for each chain * add macro for bungee bridges * chore: ignore raw.sql * add amount_usd to bungee models * remove syntax error * add bungee sources * add post hook for bungee bridges * change eth address on the chains * change the main model * remove endfor * add for loop --------- Co-authored-by: Huang Geyang Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../bungee/bungee_SocketBridge_macro.sql | 23 ++++ .../arbitrum/bungee_arbitrum_bridges.sql | 42 +++++++ .../bungee_avalanche_c_bridges.sql | 42 +++++++ .../bungee/base/bungee_base_bridges.sql | 42 +++++++ .../bungee/blast/bungee_blast_bridges.sql | 42 +++++++ .../bungee/bnb/bungee_bnb_bridges.sql | 42 +++++++ .../_projects/bungee/bungee_bridges.sql | 46 +++++++ .../ethereum/bungee_ethereum_bridges.sql | 42 +++++++ .../bungee/fantom/bungee_fantom_bridges.sql | 42 +++++++ .../bungee/gnosis/bungee_gnosis_bridges.sql | 42 +++++++ .../bungee/linea/bungee_linea_bridges.sql | 42 +++++++ .../bungee/mantle/bungee_mantle_bridges.sql | 42 +++++++ .../optimism/bungee_optimism_bridges.sql | 42 +++++++ .../bungee/polygon/bungee_polygon_bridges.sql | 42 +++++++ .../models/_projects/bungee/schema.yml | 101 +++++++++++++++ .../bungee/scroll/bungee_scroll_bridges.sql | 42 +++++++ .../bungee/zkevm/bungee_zkevm_bridges.sql | 42 +++++++ .../bungee/zksync/bungee_zksync_bridges.sql | 42 +++++++ sources/bungee/sources.yml | 116 ++++++++++++++++++ 19 files changed, 916 insertions(+) create mode 100644 dbt_subprojects/daily_spellbook/macros/project/bungee/bungee_SocketBridge_macro.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/arbitrum/bungee_arbitrum_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/avalanche_c/bungee_avalanche_c_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/base/bungee_base_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/blast/bungee_blast_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/bnb/bungee_bnb_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/ethereum/bungee_ethereum_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/fantom/bungee_fantom_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/gnosis/bungee_gnosis_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/linea/bungee_linea_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/mantle/bungee_mantle_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/optimism/bungee_optimism_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/polygon/bungee_polygon_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/scroll/bungee_scroll_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/zkevm/bungee_zkevm_bridges.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/bungee/zksync/bungee_zksync_bridges.sql create mode 100644 sources/bungee/sources.yml diff --git a/dbt_subprojects/daily_spellbook/macros/project/bungee/bungee_SocketBridge_macro.sql b/dbt_subprojects/daily_spellbook/macros/project/bungee/bungee_SocketBridge_macro.sql new file mode 100644 index 00000000000..93a0a13dbd5 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/macros/project/bungee/bungee_SocketBridge_macro.sql @@ -0,0 +1,23 @@ +{% macro bungee_SocketBridge(blockchain) %} + +select + contract_address, + evt_tx_hash, + evt_index, + evt_block_time, + evt_block_number, + amount, + token, + toChainId, + bridgeName, + sender, + receiver, + metadata, + '{{ blockchain }}' as source_chain, + {{ dbt_utils.generate_surrogate_key(['evt_tx_hash', 'evt_index']) }} as transfer_id +from {{ source('socket_v2_' ~ blockchain, 'SocketGateway_evt_SocketBridge') }} +{% if is_incremental() %} +where {{ incremental_predicate('evt_block_time') }} +{% endif %} + +{% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/arbitrum/bungee_arbitrum_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/arbitrum/bungee_arbitrum_bridges.sql new file mode 100644 index 00000000000..f8ca6d45091 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/arbitrum/bungee_arbitrum_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_arbitrum', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('arbitrum') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x82af49447d8a07e3bd95bd0d56f35241523fbab1 -- WETH on Arbitrum + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'arbitrum' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/avalanche_c/bungee_avalanche_c_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/avalanche_c/bungee_avalanche_c_bridges.sql new file mode 100644 index 00000000000..ec26a230726 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/avalanche_c/bungee_avalanche_c_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_avalanche_c', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('avalanche_c') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7 -- WAVAX + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'avalanche_c' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/base/bungee_base_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/base/bungee_base_bridges.sql new file mode 100644 index 00000000000..7220bb9b57b --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/base/bungee_base_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_base', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('base') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x4200000000000000000000000000000000000006 -- WETH on Base + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'base' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/blast/bungee_blast_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/blast/bungee_blast_bridges.sql new file mode 100644 index 00000000000..b19a68e3771 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/blast/bungee_blast_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_blast', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('blast') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x4300000000000000000000000000000000000004 -- WETH on Blast + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'blast' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/bnb/bungee_bnb_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/bnb/bungee_bnb_bridges.sql new file mode 100644 index 00000000000..2feb8870be3 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/bnb/bungee_bnb_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_bnb', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('bnb') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c -- WBNB + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'bnb' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql new file mode 100644 index 00000000000..5e716870189 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql @@ -0,0 +1,46 @@ +{{ + config( + schema = 'bungee', + alias = 'bridges', + materialized = 'view', + post_hook = '{{ expose_spells(\'[ + "ethereum", "zkevm", "scroll", "blast", "linea", "mantle", "optimism", + "gnosis", "arbitrum", "zksync", "base", "bnb", "polygon", + "avalanche_c", "fantom" + ]\', + "project", "bungee", \'["lequangphu"]\') }}' + ) +}} + +{% set chains = [ + 'ethereum', 'zkevm', 'scroll', 'blast', 'linea', 'mantle', 'optimism', + 'gnosis', 'arbitrum', 'zksync', 'base', 'bnb', 'polygon', + 'avalanche_c', 'fantom' +] %} + +with bungee_bridges as ( + {% for chain in chains %} + select + contract_address, + evt_tx_hash, + evt_index, + evt_block_time, + evt_block_number, + amount, + token, + toChainId, + bridgeName, + sender, + receiver, + metadata, + source_chain, + transfer_id + from {{ ref( 'bungee_' ~ chain ~ '_bridges' ) }} + {% if not loop.last %} + union all + {% endif %} + {% endfor %} +) + +select * +from bungee_bridges \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/ethereum/bungee_ethereum_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/ethereum/bungee_ethereum_bridges.sql new file mode 100644 index 00000000000..31b632b2ae7 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/ethereum/bungee_ethereum_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_ethereum', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('ethereum') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 -- WETH + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'ethereum' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/fantom/bungee_fantom_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/fantom/bungee_fantom_bridges.sql new file mode 100644 index 00000000000..bb9698e6120 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/fantom/bungee_fantom_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_fantom', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('fantom') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83 -- WFTM + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'fantom' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/gnosis/bungee_gnosis_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/gnosis/bungee_gnosis_bridges.sql new file mode 100644 index 00000000000..262891ca52a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/gnosis/bungee_gnosis_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_gnosis', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('gnosis') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0xe91d153e0b41518a2ce8dd3d7944fa863463a97d -- WXDAI + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'gnosis' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/linea/bungee_linea_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/linea/bungee_linea_bridges.sql new file mode 100644 index 00000000000..4d22cc3b2ae --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/linea/bungee_linea_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_linea', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('linea') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0xe5d7c2a44ffddf6b295a15c148167daaaf5cf34f -- WETH on Linea + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'linea' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/mantle/bungee_mantle_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/mantle/bungee_mantle_bridges.sql new file mode 100644 index 00000000000..6f9cc1c609d --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/mantle/bungee_mantle_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_mantle', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('mantle') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x78c1b0c915c4faa5fffa6cabf0219da63d7f4cb8 -- WMNT + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'mantle' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/optimism/bungee_optimism_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/optimism/bungee_optimism_bridges.sql new file mode 100644 index 00000000000..5fd6c070d90 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/optimism/bungee_optimism_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_optimism', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('optimism') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x4200000000000000000000000000000000000006 -- WETH on Optimism + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'optimism' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/polygon/bungee_polygon_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/polygon/bungee_polygon_bridges.sql new file mode 100644 index 00000000000..e93d39768ee --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/polygon/bungee_polygon_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_polygon', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('polygon') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270 -- WMATIC + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'polygon' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/bungee/schema.yml new file mode 100644 index 00000000000..9788cf2079a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/schema.yml @@ -0,0 +1,101 @@ +version: 2 + +models: + - name: bungee_bridges + description: "Unified cross-chain bridges data from Bungee's Socket Bridge" + columns: &common_columns + - name: transfer_id + description: "Unique identifier for each transfer" + data_tests: + - unique + - not_null + - name: contract_address + description: "Address of the Socket Gateway contract" + - name: evt_tx_hash + description: "Transaction hash of the transfer event" + - name: evt_index + description: "Index of the event in the transaction" + - name: evt_block_time + description: "Timestamp of the block containing the transfer" + - name: evt_block_number + description: "Block number containing the transfer" + - name: amount + description: "Amount of tokens transferred" + - name: token + description: "Address of the transferred token" + - name: token_adjusted + description: "Adjusted token address (maps native token to wrapped version)" + - name: toChainId + description: "Destination chain ID" + - name: bridgeName + description: "Name of the bridge used" + - name: sender + description: "Address of the sender" + - name: receiver + description: "Address of the receiver" + - name: metadata + description: "Additional transfer metadata" + - name: source_chain + description: "Source blockchain of the transfer" + - name: amount_usd + description: "USD value of the transferred amount at the time of transfer" + + - name: bungee_ethereum_bridges + description: "Ethereum bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_zkevm_bridges + description: "zkEVM bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_scroll_bridges + description: "Scroll bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_blast_bridges + description: "Blast bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_linea_bridges + description: "Linea bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_mantle_bridges + description: "Mantle bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_optimism_bridges + description: "Optimism bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_gnosis_bridges + description: "Gnosis bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_arbitrum_bridges + description: "Arbitrum bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_zksync_bridges + description: "zkSync bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_base_bridges + description: "Base bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_bnb_bridges + description: "BNB Chain bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_polygon_bridges + description: "Polygon bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_avalanche_c_bridges + description: "Avalanche C-Chain bridges data from Bungee's Socket Bridge" + columns: *common_columns + + - name: bungee_fantom_bridges + description: "Fantom bridges data from Bungee's Socket Bridge" + columns: *common_columns \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/scroll/bungee_scroll_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/scroll/bungee_scroll_bridges.sql new file mode 100644 index 00000000000..9c3a4d3c1e3 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/scroll/bungee_scroll_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_scroll', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('scroll') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x5300000000000000000000000000000000000004 -- WETH on Scroll + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'scroll' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/zkevm/bungee_zkevm_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/zkevm/bungee_zkevm_bridges.sql new file mode 100644 index 00000000000..2759ebb201d --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/zkevm/bungee_zkevm_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_zkevm', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('zkevm') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x4F9A0e7FD2Bf6067db6994CF12E4495Df938E6e9 -- WETH on zkEVM + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'zkevm' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/zksync/bungee_zksync_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/zksync/bungee_zksync_bridges.sql new file mode 100644 index 00000000000..367a8cefcba --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/zksync/bungee_zksync_bridges.sql @@ -0,0 +1,42 @@ +{{ + config( + schema = 'bungee_zksync', + alias = 'bridges', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] + ) +}} + +with source_data as ( + {{ bungee_SocketBridge('zksync') }} +), + +tokens_mapped as ( + select + *, + case + when token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + then 0x5aea5775959fbc2557cc8789bc1bf90a239d9a91 -- WETH on zkSync + else token + end as token_adjusted + from source_data +), + +price_data as ( + select + tokens_mapped.*, + p.price * amount / power(10, p.decimals) as amount_usd + from tokens_mapped + left join {{ source('prices', 'usd') }} p + on p.contract_address = tokens_mapped.token_adjusted + and p.blockchain = 'zksync' + and p.minute = date_trunc('minute', tokens_mapped.evt_block_time) + {% if is_incremental() %} + and {{ incremental_predicate('p.minute') }} + {% endif %} +) + +select * from price_data \ No newline at end of file diff --git a/sources/bungee/sources.yml b/sources/bungee/sources.yml new file mode 100644 index 00000000000..c98f528f436 --- /dev/null +++ b/sources/bungee/sources.yml @@ -0,0 +1,116 @@ +version: 2 + +sources: + - name: socket_v2_ethereum + description: "Ethereum Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: &common_columns + - name: contract_address + description: "Contract address of the Socket Gateway" + - name: evt_tx_hash + description: "Transaction hash of the event" + - name: evt_index + description: "Event index in the transaction" + - name: evt_block_time + description: "Timestamp of the block when the event was emitted" + - name: evt_block_number + description: "Block number when the event was emitted" + - name: amount + description: "Amount of tokens being bridged" + - name: token + description: "Address of the token being bridged" + - name: toChainId + description: "Destination chain ID" + - name: bridgeName + description: "Name of the bridge used" + - name: sender + description: "Address of the sender" + - name: receiver + description: "Address of the receiver" + - name: metadata + description: "Additional bridge metadata" + + - name: socket_v2_arbitrum + description: "Arbitrum Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_optimism + description: "Optimism Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_avalanche_c + description: "Avalanche C-Chain Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_bnb + description: "BNB Chain Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_polygon + description: "Polygon Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_fantom + description: "Fantom Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_gnosis + description: "Gnosis Chain Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_zksync + description: "zkSync Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_base + description: "Base Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_zkevm + description: "zkEVM Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_scroll + description: "Scroll Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_linea + description: "Linea Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_mantle + description: "Mantle Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns + + - name: socket_v2_blast + description: "Blast Socket Gateway events" + tables: + - name: SocketGateway_evt_SocketBridge + columns: *common_columns \ No newline at end of file From 5ea5c4c3d3d2b69f5ecce4beced1f1dc9454ca3b Mon Sep 17 00:00:00 2001 From: Peter <49856481+peterrliem@users.noreply.github.com> Date: Fri, 13 Dec 2024 04:06:16 +0800 Subject: [PATCH 40/59] Create NFT Ronin Transfers (#7136) * Create nft_ronin_transfers table * Update nft_transfers.sql Added Ronin * Rename nft_ronin_transfers to nft_ronin_transfers.sql * Update _schema.yml --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../nft/models/_sector/transfers/_schema.yml | 22 +++++++++++++++++ .../transfers/chains/nft_ronin_transfers.sql | 24 +++++++++++++++++++ .../_sector/transfers/nft_transfers.sql | 3 ++- 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/nft/models/_sector/transfers/chains/nft_ronin_transfers.sql diff --git a/dbt_subprojects/nft/models/_sector/transfers/_schema.yml b/dbt_subprojects/nft/models/_sector/transfers/_schema.yml index e28d91c834d..1c470216126 100644 --- a/dbt_subprojects/nft/models/_sector/transfers/_schema.yml +++ b/dbt_subprojects/nft/models/_sector/transfers/_schema.yml @@ -471,3 +471,25 @@ models: - *token_standard - *transfer_type - *unique_transfer_id + + - name: nft_ronin_transfers + meta: + blockchain: ronin + sector: nft + contributors: petertherock + config: + tags: [ 'nft', 'transfers' ] + description: > + NFT transfers + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - token_id + - amount + columns: + # only listing columns that have tests here for simplicity + - *token_standard + - *transfer_type + - *unique_transfer_id diff --git a/dbt_subprojects/nft/models/_sector/transfers/chains/nft_ronin_transfers.sql b/dbt_subprojects/nft/models/_sector/transfers/chains/nft_ronin_transfers.sql new file mode 100644 index 00000000000..6bd78b2fd49 --- /dev/null +++ b/dbt_subprojects/nft/models/_sector/transfers/chains/nft_ronin_transfers.sql @@ -0,0 +1,24 @@ +{{ + config( + schema="nft_ronin", + alias="transfers", + partition_by=["block_month"], + materialized="incremental", + file_format="delta", + incremental_strategy="merge", + incremental_predicates=[ + incremental_predicate("DBT_INTERNAL_DEST.block_time") + ], + unique_key=["tx_hash", "evt_index", "token_id", "amount"], + ) +}} + +{{ + nft_transfers( + blockchain="ronin", + base_transactions=source("ronin", "transactions"), + erc721_transfers=source("erc721_ronin", "evt_transfer"), + erc1155_single=source("erc1155_ronin", "evt_transfersingle"), + erc1155_batch=source("erc1155_ronin", "evt_transferbatch"), + ) +}} diff --git a/dbt_subprojects/nft/models/_sector/transfers/nft_transfers.sql b/dbt_subprojects/nft/models/_sector/transfers/nft_transfers.sql index 721069b7ae1..7f4ffc9fd75 100644 --- a/dbt_subprojects/nft/models/_sector/transfers/nft_transfers.sql +++ b/dbt_subprojects/nft/models/_sector/transfers/nft_transfers.sql @@ -8,7 +8,7 @@ incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], unique_key = ['tx_hash', 'evt_index', 'token_id', 'amount'], - post_hook='{{ expose_spells(\'["ethereum", "bnb", "avalanche_c", "gnosis", "optimism", "arbitrum", "polygon", "fantom", "goerli", "base", "celo", "zksync", "zora", "scroll", "linea", "blast", "mantle"]\', + post_hook='{{ expose_spells(\'["ethereum", "bnb", "avalanche_c", "gnosis", "optimism", "arbitrum", "polygon", "fantom", "goerli", "base", "celo", "zksync", "zora", "scroll", "linea", "blast", "mantle", "sei", "ronin"]\', "sector", "nft", \'["hildobby", "0xRob", "rantum","petertherock"]\') }}' @@ -34,6 +34,7 @@ ,ref('nft_blast_transfers') ,ref('nft_mantle_transfers') ,ref('nft_sei_transfers') +,ref('nft_ronin_transfers') ] %} SELECT * From 5db57d23b933c811c16ea713d8ae0ee13a3be7f2 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:50:43 +0100 Subject: [PATCH 41/59] add flag (#7327) --- dbt_subprojects/daily_spellbook/README.md | 2 +- dbt_subprojects/dex/README.md | 2 +- dbt_subprojects/tokens/dbt_project.yml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/daily_spellbook/README.md b/dbt_subprojects/daily_spellbook/README.md index be76883e83d..284f462bfd6 100644 --- a/dbt_subprojects/daily_spellbook/README.md +++ b/dbt_subprojects/daily_spellbook/README.md @@ -1,3 +1,3 @@ ## Daily Spellbook -This is a DBT sub project for the the main models of Spellbook that runs on a daily candence. \ No newline at end of file +This is a DBT sub project for the the main models of Spellbook that runs on a daily candence. diff --git a/dbt_subprojects/dex/README.md b/dbt_subprojects/dex/README.md index 8aef6e6c3ac..f27f945ea18 100644 --- a/dbt_subprojects/dex/README.md +++ b/dbt_subprojects/dex/README.md @@ -8,4 +8,4 @@ This is a DBT subproject for the main lineages of the DEX sector. Included in th - `dex.sandwiched` - ...and more! -This subproject will be dedicated to building the above spells (and others in the future related to DEX) on an hourly cadence in production. All other spells not included within this subproject will treat these spells as sources. For example, labels spells which read from `dex.trades` will now treat the spell as a source, rather than reference within dbt. \ No newline at end of file +This subproject will be dedicated to building the above spells (and others in the future related to DEX) on an hourly cadence in production. All other spells not included within this subproject will treat these spells as sources. For example, labels spells which read from `dex.trades` will now treat the spell as a source, rather than reference within dbt. diff --git a/dbt_subprojects/tokens/dbt_project.yml b/dbt_subprojects/tokens/dbt_project.yml index 9ec35c4c768..4ab163deb41 100644 --- a/dbt_subprojects/tokens/dbt_project.yml +++ b/dbt_subprojects/tokens/dbt_project.yml @@ -14,6 +14,9 @@ quoting: # profile: "spellbook-poc-tokens" profile: "spellbook-local" +flags: + require_certificate_validation: true + vars: DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}" From 4774a181eeeeba30234dcea053c1c3bbd56ec1d0 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:09:25 +0100 Subject: [PATCH 42/59] add require_certificate_validation flags (#7328) --- dbt_subprojects/daily_spellbook/dbt_project.yml | 3 +++ dbt_subprojects/dex/dbt_project.yml | 3 +++ dbt_subprojects/hourly_spellbook/dbt_project.yml | 3 +++ dbt_subprojects/nft/dbt_project.yml | 3 +++ dbt_subprojects/solana/dbt_project.yml | 3 +++ 5 files changed, 15 insertions(+) diff --git a/dbt_subprojects/daily_spellbook/dbt_project.yml b/dbt_subprojects/daily_spellbook/dbt_project.yml index 6b78c227a00..3de0a51a609 100644 --- a/dbt_subprojects/daily_spellbook/dbt_project.yml +++ b/dbt_subprojects/daily_spellbook/dbt_project.yml @@ -14,6 +14,9 @@ quoting: # profile: "spellbook-poc-tokens" profile: "spellbook-local" +flags: + require_certificate_validation: true + vars: DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '3') }}" diff --git a/dbt_subprojects/dex/dbt_project.yml b/dbt_subprojects/dex/dbt_project.yml index e7ae73c9424..5e25bae64bd 100644 --- a/dbt_subprojects/dex/dbt_project.yml +++ b/dbt_subprojects/dex/dbt_project.yml @@ -13,6 +13,9 @@ quoting: # This setting configures which "profile" dbt uses for this project. profile: "spellbook-local" +flags: + require_certificate_validation: true + vars: DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}" diff --git a/dbt_subprojects/hourly_spellbook/dbt_project.yml b/dbt_subprojects/hourly_spellbook/dbt_project.yml index 401361c36c8..ce172a90410 100644 --- a/dbt_subprojects/hourly_spellbook/dbt_project.yml +++ b/dbt_subprojects/hourly_spellbook/dbt_project.yml @@ -14,6 +14,9 @@ quoting: # profile: "spellbook-poc-tokens" profile: "spellbook-local" +flags: + require_certificate_validation: true + vars: DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}" diff --git a/dbt_subprojects/nft/dbt_project.yml b/dbt_subprojects/nft/dbt_project.yml index f26a684eeb9..7fd0b4eb5c3 100644 --- a/dbt_subprojects/nft/dbt_project.yml +++ b/dbt_subprojects/nft/dbt_project.yml @@ -14,6 +14,9 @@ quoting: # profile: "spellbook-poc-tokens" profile: "spellbook-local" +flags: + require_certificate_validation: true + vars: DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}" diff --git a/dbt_subprojects/solana/dbt_project.yml b/dbt_subprojects/solana/dbt_project.yml index ed3a13af12f..9daa19b0fee 100644 --- a/dbt_subprojects/solana/dbt_project.yml +++ b/dbt_subprojects/solana/dbt_project.yml @@ -13,6 +13,9 @@ quoting: # This setting configures which "profile" dbt uses for this project. profile: "spellbook-local" +flags: + require_certificate_validation: true + vars: DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}" From 8bcac3953f4a40a850386d13d59072d082ec0a63 Mon Sep 17 00:00:00 2001 From: hinus <64959125+cxheng315@users.noreply.github.com> Date: Sat, 14 Dec 2024 01:45:14 +0800 Subject: [PATCH 43/59] Add apeiron token to ronin erc20 (#7323) * feat: add apeiron token to ronin erc20 * Update tokens_ronin_erc20.sql --- .../tokens/models/tokens/ronin/tokens_ronin_erc20.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql b/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql index 52bcf26685e..d384a32bd2f 100644 --- a/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ronin/tokens_ronin_erc20.sql @@ -20,4 +20,6 @@ FROM (VALUES , (0x18d2bdef572c67127e218c425f546fe64430a92c, 'LUAUSD', 18) , (0x7eae20d11ef8c779433eb24503def900b9d28ad7, 'PIXEL', 18) , (0xd61bbbb8369c46c15868ad9263a2710aced156c4, 'LUA', 18) -) AS temp_table (contract_address, symbol, decimals) \ No newline at end of file + , (0x7894b3088d069e70895effa4e8f7d2c243fd04c1, 'APRS', 18) + , (0xf80132fc0a86add011bffce3aedd60a86e3d704d, 'ANIMA', 18) +) AS temp_table (contract_address, symbol, decimals) From e13c06e1c671366c322853c0b60c91cd8b6b7266 Mon Sep 17 00:00:00 2001 From: Eduard Horch <40689054+grkhr@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:00:25 +0100 Subject: [PATCH 44/59] Update oneinch_mapped_contracts_macro.sql (#7330) --- .../project/oneinch/_meta/oneinch_mapped_contracts_macro.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql b/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql index fffe4792f2d..51d242c34ab 100644 --- a/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql +++ b/dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql @@ -79,7 +79,7 @@ , ('0xbf1fc29668e5f5eaa819948599c9ac1b1e03e75f', 'true', 'Cone' , 'ConeRouter01' , ['bnb']) , ('0x11984dc4465481512eb5b777e44061c158cf2259', 'true', 'Connext' , 'ConnextDiamond' , ['polygon']) , ('0xee9dec2712cce65174b561151701bf54b99c24c8', 'true', 'Connext' , 'ConnextDiamond' , ['arbitrum']) - , ('0x9008d19f58aabd9ed0d60971565aa8510560ab41', 'true', 'CoWSwap' , 'GPv2Settlement' , ['ethereum','gnosis','arbitrum']) + , ('0x9008d19f58aabd9ed0d60971565aa8510560ab41', 'true', 'CoWSwap' , 'GPv2Settlement' , ['ethereum','gnosis','arbitrum','base']) , ('0xfa43de785dd3cd0ef3dae0dd2b8be3f1b5112d1a', 'true', 'CrossCurve' , 'UnifiedRouterV2v1' , ['ethereum','bnb','polygon','gnosis','arbitrum','avalanche_c','optimism','base','fantom']) , ('0xa2a786ff9148f7c88ee93372db8cbe9e94585c74', 'true', 'CrossCurve' , 'UnifiedRouterV2v5' , ['ethereum','bnb','polygon','gnosis','arbitrum','avalanche_c','optimism','base','fantom','blast','linea','mantle']) , ('0xe7db62c7960183895190274f26925388db4a3be4', 'true', 'CrossCurve' , 'UnifiedRouterV2' , ['ethereum','bnb','polygon','gnosis','arbitrum','avalanche_c','optimism','base','fantom']) From 2d7e97a76135f0f7d2348b0a05ef39b8672c2836 Mon Sep 17 00:00:00 2001 From: Phu <33654804+lequangphu@users.noreply.github.com> Date: Sat, 14 Dec 2024 02:02:34 +0700 Subject: [PATCH 45/59] Continue work of PR #7255, add amount_usd to the main model (#7325) * Add Odos on Arbitrum to dex_aggregator * Change to WETH address on Arbitrum * Add sources of Odos on Arbitrum * Add odos/arbitrum seeds * Change data type * Add bridges of LiFi across chains * replace tests which is deprecated * fix name error of avalanche source * add index to unique test * fix concat issue * fix unique test of the main model * try to fix unique key * use surrogate key for data tests * use generate_surrogate_key inside the models * fix and remove redundant schema properties * data_tests * move generate_surrogate_key to schema.yml * 1. create macro lifi_extract_bridge_data_macro.sql and apply it in the models 2. update the models' config 3. flip materialization strategy * data_ again * fix naming of the main model * replace avalanche with avalanche_c * avalanche_c again * 1. add block_date column to the macro to use it in the add_tx_columns macro 2. update model of avalanche_c * rename columns to use add_tx_columns macro * remove evt_ * a few minor changes * add amount_usd column * data_tests again * map native token to wrapped token to avoid null price data * update the main model and schema * move type conversion from models to macro * fix source name * varbinary type doesn't need single quotes * correct collumn name * add post_hook * feat: add bungee bridges * create macro and models for each chain * add macro for bungee bridges * chore: ignore raw.sql * add amount_usd to bungee models * remove syntax error * add bungee sources * add post hook for bungee bridges * change eth address on the chains * change the main model * remove endfor * add for loop * add amount_usd to the main model --------- Co-authored-by: Huang Geyang Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../daily_spellbook/models/_projects/bungee/bungee_bridges.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql b/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql index 5e716870189..b2605abd834 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/bungee/bungee_bridges.sql @@ -34,7 +34,8 @@ with bungee_bridges as ( receiver, metadata, source_chain, - transfer_id + transfer_id, + amount_usd from {{ ref( 'bungee_' ~ chain ~ '_bridges' ) }} {% if not loop.last %} union all From 025892fb53df7fd30090fa625057f9643f590836 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:15:35 +0100 Subject: [PATCH 46/59] Balance improvements (#7290) * move safe balance models * add incremental to balances and use daily prices table --- .../balances_incremental_subset_daily.sql | 12 +- .../_projects/safe/balances/_schema.yml | 123 ++++++++++++++++++ .../safe/balances}/safe_arbitrum_balances.sql | 2 +- .../balances}/safe_avalanche_c_balances.sql | 2 +- .../safe/balances}/safe_base_balances.sql | 2 +- .../safe/balances}/safe_ethereum_balances.sql | 2 +- .../safe/balances}/safe_linea_balances.sql | 2 +- .../safe/balances}/safe_optimism_balances.sql | 2 +- .../safe/balances}/safe_polygon_balances.sql | 2 +- .../safe/balances}/safe_scroll_balances.sql | 2 +- .../safe/arbitrum/safe_arbitrum_schema.yml | 46 ------- .../avalanche_c/safe_avalanche_c_schema.yml | 48 +------ .../_project/safe/base/safe_base_schema.yml | 46 ------- .../models/_project/safe/ethereum/_schema.yml | 48 ------- .../_project/safe/linea/safe_linea_schema.yml | 48 +------ .../safe/optimism/safe_optimism_schema.yml | 45 ------- .../safe/polygon/safe_polygon_schema.yml | 49 +------ .../safe/scroll/safe_scroll_schema.yml | 46 ------- .../hourly_spellbook/_sources.yml | 25 +++- 19 files changed, 165 insertions(+), 387 deletions(-) create mode 100644 dbt_subprojects/daily_spellbook/models/_projects/safe/balances/_schema.yml rename dbt_subprojects/{hourly_spellbook/models/_project/safe/arbitrum => daily_spellbook/models/_projects/safe/balances}/safe_arbitrum_balances.sql (95%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/avalanche_c => daily_spellbook/models/_projects/safe/balances}/safe_avalanche_c_balances.sql (94%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/base => daily_spellbook/models/_projects/safe/balances}/safe_base_balances.sql (95%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/ethereum => daily_spellbook/models/_projects/safe/balances}/safe_ethereum_balances.sql (95%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/linea => daily_spellbook/models/_projects/safe/balances}/safe_linea_balances.sql (95%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/optimism => daily_spellbook/models/_projects/safe/balances}/safe_optimism_balances.sql (96%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/polygon => daily_spellbook/models/_projects/safe/balances}/safe_polygon_balances.sql (95%) rename dbt_subprojects/{hourly_spellbook/models/_project/safe/scroll => daily_spellbook/models/_projects/safe/balances}/safe_scroll_balances.sql (95%) diff --git a/dbt_macros/shared/balances_incremental_subset_daily.sql b/dbt_macros/shared/balances_incremental_subset_daily.sql index 9e330747b25..d397380b258 100644 --- a/dbt_macros/shared/balances_incremental_subset_daily.sql +++ b/dbt_macros/shared/balances_incremental_subset_daily.sql @@ -149,14 +149,18 @@ from( {% endif %} ) b -left join {{source('prices','usd')}} p - on (token_standard = 'erc20' +left join {{source('prices','usd_daily')}} p + on 1=1 + {% if is_incremental() %} + and {{ incremental_predicate('p.day') }} + {% endif %} + and ((token_standard = 'erc20' and p.blockchain = '{{blockchain}}' and b.token_address = p.contract_address - and b.day = p.minute) + and b.day = p.day) or (token_standard = 'native' and p.blockchain is null and p.contract_address is null and p.symbol = (select native_token_symbol from {{source('evms','info')}} where blockchain = '{{blockchain}}') - and b.day = p.minute) + and b.day = p.day)) {% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/_schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/_schema.yml new file mode 100644 index 00000000000..26d00f99e5b --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/_schema.yml @@ -0,0 +1,123 @@ +version: 2 + +models: + - name: safe_arbitrum_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: arbitrum + project: safe + contributors: safeintern + config: + tags: ['safe', 'arbitrum'] + description: “Safe addresses balances” + + - name: safe_avalanche_c_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: avalanche_c + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'avalanche_c' ] + description: “Safe addresses balances” + + - name: safe_base_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: base + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'base' ] + description: “Safe addresses balances” + + - name: safe_ethereum_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: ethereum + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'ethereum' ] + description: “Safe addresses balances” + + - name: safe_linea_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: linea + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'linea' ] + description: “Safe addresses balances” + + - name: safe_optimism_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: optimism + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'optimism' ] + description: “Safe addresses balances” + + - name: safe_polygon_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: polygon + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'polygon' ] + description: “Safe addresses balances” + + - name: safe_scroll_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - address + - token_address + meta: + blockchain: scroll + project: safe + contributors: safeintern + config: + tags: [ 'safe', 'scroll' ] + description: “Safe addresses balances” + diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_arbitrum_balances.sql similarity index 95% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_arbitrum_balances.sql index 56d1fe4895b..9c5b0f9816d 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_arbitrum_balances.sql @@ -15,7 +15,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_arbitrum_safes') }} + from {{ source('safe_arbitrum','safes') }} where blockchain = 'arbitrum' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_avalanche_c_balances.sql similarity index 94% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_avalanche_c_balances.sql index 432cd070a0a..33279643d65 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_avalanche_c_balances.sql @@ -15,7 +15,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_avalanche_c_safes') }} + from {{ source('safe_avalanche_c','safes') }} where blockchain = 'avalanche_c' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_base_balances.sql similarity index 95% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_base_balances.sql index 84aeba4f700..44e45138e77 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_base_balances.sql @@ -15,7 +15,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_base_safes') }} + from {{ source('safe_base','safes') }} where blockchain = 'base' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/safe_ethereum_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_ethereum_balances.sql similarity index 95% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/safe_ethereum_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_ethereum_balances.sql index ebc7a430771..e681bf730d2 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/safe_ethereum_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_ethereum_balances.sql @@ -14,7 +14,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_ethereum_safes') }} + from {{ source('safe_ethereum','safes') }} where blockchain = 'ethereum' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_linea_balances.sql similarity index 95% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_linea_balances.sql index 038ed2990d5..de62451a1e4 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_linea_balances.sql @@ -15,7 +15,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_linea_safes') }} + from {{ source('safe_linea','safes') }} where blockchain = 'linea' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_optimism_balances.sql similarity index 96% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_optimism_balances.sql index 2465f2a8d65..8338b3be28c 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_optimism_balances.sql @@ -20,7 +20,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_optimism_safes') }} + from {{ source('safe_optimism','safes') }} where blockchain = 'optimism' ), balances as ( diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_polygon_balances.sql similarity index 95% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_polygon_balances.sql index 5c62beac340..de7eb4a6d7a 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_polygon_balances.sql @@ -15,7 +15,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_polygon_safes') }} + from {{ source('safe_polygon','safes') }} where blockchain = 'polygon' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_scroll_balances.sql similarity index 95% rename from dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql rename to dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_scroll_balances.sql index 5c4389542bb..4cf40377e92 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_balances.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/safe/balances/safe_scroll_balances.sql @@ -15,7 +15,7 @@ with safes as ( select address, blockchain - from {{ ref('safe_scroll_safes') }} + from {{ source('safe_scroll','safes') }} where blockchain = 'scroll' ), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_schema.yml index bdda09eab40..269628b66e5 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/arbitrum/safe_arbitrum_schema.yml @@ -149,49 +149,3 @@ models: - &amount_usd name: amount_usd description: "USD amount of transferred ETH" - - - name: safe_arbitrum_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - address - - token_address - meta: - blockchain: arbitrum - project: safe - contributors: safeintern - config: - tags: ['safe', 'arbitrum'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: " unique key" diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml index 629b3491f2d..9cbdd4f86d7 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/avalanche_c/safe_avalanche_c_schema.yml @@ -80,7 +80,7 @@ models: name: value description: "Value of transaction" - &gas - name: gas + name: gas description: "Gas limit set for transaction" - &execution_gas_used name: execution_gas_used @@ -149,49 +149,3 @@ models: - &amount_usd name: amount_usd description: "USD amount of transferred AVAX" - - - name: safe_avalanche_c_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - address - - token_address - meta: - blockchain: avalanche_c - project: safe - contributors: safeintern - config: - tags: ['safe', 'avalanche_c'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: "unique key" diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml index fae0ee71228..4e73519cadc 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/base/safe_base_schema.yml @@ -141,49 +141,3 @@ models: - &output name: output description: "Output data" - - - name: safe_base_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - address - - token_address - meta: - blockchain: base - project: safe - contributors: safeintern - config: - tags: ['safe', 'base'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: " unique key" diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/_schema.yml index 501fd6d00ac..ea8bb76b964 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/ethereum/_schema.yml @@ -154,51 +154,3 @@ models: - &threshold name: threshold description: "Number of signer threshold" - - - name: safe_ethereum_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - blockchain - - address - - token_address - - meta: - blockchain: ethereum - project: safe - contributors: safeintern - config: - tags: ['safe', 'ethereum'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: " unique key" diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml index 2903c2ec94c..28ec16f2da1 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/linea/safe_linea_schema.yml @@ -147,50 +147,4 @@ models: - &output name: output description: "Output data" - - - - name: safe_linea_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - address - - token_address - meta: - blockchain: linea - project: safe - contributors: safeintern - config: - tags: ['safe', 'linea'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: " unique key" + diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_schema.yml index f7ea3fc9a8e..ab2bcca3a16 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/optimism/safe_optimism_schema.yml @@ -149,48 +149,3 @@ models: - &amount_usd name: amount_usd description: "USD amount of transferred ETH" - - - name: safe_optimism_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - blockchain - - address - - token_address - meta: - blockchain: optimism - project: safe - contributors: safeintern - config: - tags: ['safe', 'optimism'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_schema.yml index c65d6199675..2d30451c537 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/polygon/safe_polygon_schema.yml @@ -119,7 +119,7 @@ models: name: value description: "Value of transaction" - &gas - name: gas + name: gas description: "Gas limit set for transaction" - &execution_gas_used name: execution_gas_used @@ -151,50 +151,3 @@ models: - &output name: output description: "Output data" - - - name: safe_polygon_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - blockchain - - address - - token_address - meta: - blockchain: polygon - project: safe - contributors: safeintern - config: - tags: ['safe', 'polygon'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: " unique key" diff --git a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml index 0d1ad4706e2..f33f34b7dc2 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/safe/scroll/safe_scroll_schema.yml @@ -159,49 +159,3 @@ models: - &method name: method description: "Function method" - - - name: safe_scroll_balances - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day - - address - - token_address - meta: - blockchain: scroll - project: safe - contributors: safeintern - config: - tags: ['safe', 'scroll'] - description: “Safe addresses balances” - columns: - - name: day - - name: blockchain - - name: address - - &token_symbol - name: token_symbol - description: "Symbol for the token" - - &token_address - name: token_address - description: "Address for the token" - - &token_standard - name: token_standard - description: "Standard for the token" - - &token_id - name: token_id - description: "ID for the token" - - &balance - name: balance - description: "Balance for the user" - - &balance_usd - name: balance_usd - description: "USD value of balance for the user" - - &last_updated - name: last_updated - description: "UTC timestamp when data was last updated" - - &next_update - name: next_update - description: "UTC timestamp when data is next updated" - - &unique_key_id - name: unique_key_id - description: "unique key" diff --git a/sources/_subprojects_outputs/hourly_spellbook/_sources.yml b/sources/_subprojects_outputs/hourly_spellbook/_sources.yml index 7fd30baafb7..ecf970cd3af 100644 --- a/sources/_subprojects_outputs/hourly_spellbook/_sources.yml +++ b/sources/_subprojects_outputs/hourly_spellbook/_sources.yml @@ -38,6 +38,27 @@ sources: - name: safe_ethereum tables: - name: safes + - name: safe_arbitrum + tables: + - name: safes + - name: safe_avalanche_c + tables: + - name: safes + - name: safe_base + tables: + - name: safes + - name: safe_linea + tables: + - name: safes + - name: safe_optimism + tables: + - name: safes + - name: safe_polygon + tables: + - name: safes + - name: safe_scroll + tables: + - name: safes - name: perpetual tables: @@ -49,7 +70,7 @@ sources: - name: jelly_swap_sei tables: - - name: pools_metrics_daily + - name: pools_metrics_daily - name: account_abstraction_erc4337 tables: @@ -61,4 +82,4 @@ sources: - name: evms tables: - - name: transaction_metrics \ No newline at end of file + - name: transaction_metrics From f0b3045e1db00be7638a4cf3324ffa40748f0d74 Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Fri, 13 Dec 2024 21:52:06 +0000 Subject: [PATCH 47/59] Update _schema for balancer models (#7331) * Update _schema.yml * Update _schema.yml * Update _schema.yml * Update balancer_v3_ethereum_erc4626_token_prices.sql * Update balancer_v3_gnosis_erc4626_token_prices.sql * Update balancer_v3_erc4626_token_prices.sql --- .../models/_project/balancer/erc4626_tokens/_schema.yml | 6 +++--- .../erc4626_tokens/balancer_v3_erc4626_token_prices.sql | 1 + .../_project/balancer/erc4626_tokens/ethereum/_schema.yml | 6 +++--- .../ethereum/balancer_v3_ethereum_erc4626_token_prices.sql | 4 +++- .../_project/balancer/erc4626_tokens/gnosis/_schema.yml | 6 +++--- .../gnosis/balancer_v3_gnosis_erc4626_token_prices.sql | 3 ++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml index 6192f2b94b8..15203c33c6f 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/_schema.yml @@ -7,7 +7,7 @@ models: project: balancer contributors: viniabussafi config: - tags: ['ethereum', 'gnosis', 'static', 'token', 'mapping'] + tags: ['ethereum', 'gnosis', 'token', 'mapping'] description: > Mapping of ERC4626 tokens, including aTokens and MetaMorpho tokens, and their corresponding underlying tokens. data_tests: @@ -36,7 +36,7 @@ models: project: balancer contributors: viniabussafi config: - tags: ['ethereum', 'gnosis', 'static', 'token', 'pricing'] + tags: ['ethereum', 'gnosis', 'token', 'pricing'] description: > Pricing information for ERC4626 tokens. Prices are derived from the ratio of underlying tokens deposited or withdrawn to shares issued or burned, multiplied by the underlying token price. data_tests: @@ -62,4 +62,4 @@ models: - name: median_price description: "Median price of the static token, computed over the dataset." - name: next_change - description: "Timestamp of the next expected pricing change or event affecting the price." \ No newline at end of file + description: "Timestamp of the next expected pricing change or event affecting the price." diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql index d47484c2ad3..33553280603 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/balancer_v3_erc4626_token_prices.sql @@ -8,6 +8,7 @@ ) }} + {% set balancer_models = [ ref('balancer_v3_ethereum_erc4626_token_prices'), ref('balancer_v3_gnosis_erc4626_token_prices') diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml index ae19546932e..a12e45769be 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/_schema.yml @@ -7,7 +7,7 @@ models: project: balancer contributors: viniabussafi config: - tags: ['ethereum', 'static', 'token', 'mapping'] + tags: ['ethereum', 'token', 'mapping'] description: > Mapping of ERC4626 tokens on Ethereum, including aTokens and MetaMorpho tokens, and their corresponding underlying tokens. data_tests: @@ -36,7 +36,7 @@ models: project: balancer contributors: viniabussafi config: - tags: ['ethereum', 'static', 'token', 'prices'] + tags: ['ethereum', 'token', 'prices'] description: > Pricing information for ERC4626 tokens on Ethereum. Prices are derived from the ratio of underlying tokens deposited or withdrawn to shares issued or burned, multiplied by the underlying token price. data_tests: @@ -62,4 +62,4 @@ models: - name: median_price description: "Median price of the static token, computed over the dataset." - name: next_change - description: "Timestamp of the next expected pricing change or event affecting the price." \ No newline at end of file + description: "Timestamp of the next expected pricing change or event affecting the price." diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql index e00a572b011..02d989db3be 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/ethereum/balancer_v3_ethereum_erc4626_token_prices.sql @@ -9,6 +9,7 @@ ) }} + WITH wrap_unwrap AS( SELECT evt_block_time, @@ -62,4 +63,5 @@ SELECT APPROX_PERCENTILE(adjusted_price, 0.5) AS median_price, LEAD(p.evt_block_time, 1, CURRENT_DATE + INTERVAL '1' day) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change FROM price_join p -GROUP BY 1, 2, 3, 4, 5, 6, 7 \ No newline at end of file +GROUP BY 1, 2, 3, 4, 5, 6, 7 + diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml index 48cf7ead266..efb3ece0f2b 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/_schema.yml @@ -7,7 +7,7 @@ models: project: balancer contributors: viniabussafi config: - tags: ['gnosis', 'static', 'token', 'mapping'] + tags: ['gnosis', 'token', 'mapping'] description: > Mapping of ERC4626 tokens on gnosis, including aTokens and MetaMorpho tokens, and their corresponding underlying tokens. data_tests: @@ -36,7 +36,7 @@ models: project: balancer contributors: viniabussafi config: - tags: ['gnosis', 'static', 'token', 'prices'] + tags: ['gnosis', 'token', 'prices'] description: > Pricing information for ERC4626 tokens on gnosis. Prices are derived from the ratio of underlying tokens deposited or withdrawn to shares issued or burned, multiplied by the underlying token price. data_tests: @@ -62,4 +62,4 @@ models: - name: median_price description: "Median price of the static token, computed over the dataset." - name: next_change - description: "Timestamp of the next expected pricing change or event affecting the price." \ No newline at end of file + description: "Timestamp of the next expected pricing change or event affecting the price." diff --git a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql index 34fced964c3..f7166ea170f 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/balancer/erc4626_tokens/gnosis/balancer_v3_gnosis_erc4626_token_prices.sql @@ -9,6 +9,7 @@ ) }} + WITH wrap_unwrap AS( SELECT evt_block_time, @@ -62,4 +63,4 @@ SELECT APPROX_PERCENTILE(adjusted_price, 0.5) AS median_price, LEAD(p.evt_block_time, 1, CURRENT_DATE + INTERVAL '1' day) OVER (PARTITION BY wrappedToken ORDER BY p.evt_block_time) AS next_change FROM price_join p -GROUP BY 1, 2, 3, 4, 5, 6, 7 \ No newline at end of file +GROUP BY 1, 2, 3, 4, 5, 6, 7 From c7237e7c88cd59b8d2f1e59e346795218404b4f4 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:30:16 +0100 Subject: [PATCH 48/59] Auto labels and draft (#7321) * test auto labeler * fix * permissions * only do labels * try gh cli * try this * try this for labels * check repo * check auth status * set permissions at job level * syntax * simplify permissions * test auto project labels * make change in hourly * undo change in hourly * make change in solana * also set as draft/ready when labels are set * Revert "make change in solana" This reverts commit 66433397bd1acf2f89b933137157204f32ea8270. * ok this doesn't work, lets try this action * try gh cli * try this with gh api * try this * try this ID?? * enable content permissions * try gh CLI * syntax * final setup? * add readme changes to each subproject to check labels * add simple tokens readme --------- Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .github/labeler.yml | 23 +++++++++++ .github/workflows/pr_automation.yml | 46 ++++++++++++++++++++++ dbt_subprojects/dex/README.md | 1 + dbt_subprojects/hourly_spellbook/README.md | 2 +- dbt_subprojects/nft/README.md | 2 +- dbt_subprojects/solana/README.md | 2 +- dbt_subprojects/tokens/README.md | 3 ++ 7 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/pr_automation.yml create mode 100644 dbt_subprojects/tokens/README.md diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000000..688cbc3a965 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,23 @@ +"dbt: daily": +- changed-files: + - any-glob-to-any-file: 'dbt_subprojects/daily_spellbook/**' + +"dbt: dex": +- changed-files: + - any-glob-to-any-file: 'dbt_subprojects/dex/**' + +"dbt: hourly": +- changed-files: + - any-glob-to-any-file: 'dbt_subprojects/hourly_spellbook/**' + +"dbt: nft": +- changed-files: + - any-glob-to-any-file: 'dbt_subprojects/nft/**' + +"dbt: solana": +- changed-files: + - any-glob-to-any-file: 'dbt_subprojects/solana/**' + +"dbt: tokens": +- changed-files: + - any-glob-to-any-file: 'dbt_subprojects/tokens/**' diff --git a/.github/workflows/pr_automation.yml b/.github/workflows/pr_automation.yml new file mode 100644 index 00000000000..347e935773c --- /dev/null +++ b/.github/workflows/pr_automation.yml @@ -0,0 +1,46 @@ +name: PR automation + +on: + pull_request: + types: + - opened + - ready_for_review + - converted_to_draft + - synchronize + - labeled + +permissions: + pull-requests: write + contents: write + +jobs: + pr-automation: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + NUMBER: ${{ github.event.number }} + steps: + - name: Mark new PR as Draft + if: (github.event.action == 'opened' && github.event.pull_request.draft == false) + run: gh pr ready "$NUMBER" --undo + + - name: Set labels on Open + if: github.event.action == 'opened' + run: gh pr edit "$NUMBER" --add-label "WIP" + + - name: Set labels on Draft + if: github.event.action == 'converted_to_draft' + run: gh pr edit "$NUMBER" --add-label "WIP" --remove-label "ready-for-review" + + - name: Set labels on ready-for-review + if: github.event.action == 'ready_for_review' + run: gh pr edit "$NUMBER" --add-label "ready-for-review" --remove-label "WIP" + + - name: Add subproject labels + if: contains('synchronize,opened,reopened', github.event.action) + uses: actions/labeler@v5 + with: + sync-labels: true + + diff --git a/dbt_subprojects/dex/README.md b/dbt_subprojects/dex/README.md index f27f945ea18..a14c2b63f6a 100644 --- a/dbt_subprojects/dex/README.md +++ b/dbt_subprojects/dex/README.md @@ -1,5 +1,6 @@ ## DEX subproject + This is a DBT subproject for the main lineages of the DEX sector. Included in this subproject, but not limited to over time: - `dex.trades` - `dex_aggreagtor.trades` diff --git a/dbt_subprojects/hourly_spellbook/README.md b/dbt_subprojects/hourly_spellbook/README.md index ae104440a3c..3c685b43f50 100644 --- a/dbt_subprojects/hourly_spellbook/README.md +++ b/dbt_subprojects/hourly_spellbook/README.md @@ -1,3 +1,3 @@ ## Hourly Spellbook -This is a DBT sub project for the all hourly models in spellbook +This is a DBT sub project for the all hourly models in spellbook. diff --git a/dbt_subprojects/nft/README.md b/dbt_subprojects/nft/README.md index 0b38ebded01..eb055932640 100644 --- a/dbt_subprojects/nft/README.md +++ b/dbt_subprojects/nft/README.md @@ -1,3 +1,3 @@ ## Daily Spellbook -This is a DBT sub project for the all the NFT related models in spellbook. +This is a DBT sub project for the all the NFT related models in spellbook diff --git a/dbt_subprojects/solana/README.md b/dbt_subprojects/solana/README.md index 2fee156a2d1..9b3ec39f43f 100644 --- a/dbt_subprojects/solana/README.md +++ b/dbt_subprojects/solana/README.md @@ -1,3 +1,3 @@ ## Solana subproject -This is a DBT subproject for the main lineages of Solana data. \ No newline at end of file +This is a DBT subproject for the main lineages of Solana data diff --git a/dbt_subprojects/tokens/README.md b/dbt_subprojects/tokens/README.md new file mode 100644 index 00000000000..f5a7e541b34 --- /dev/null +++ b/dbt_subprojects/tokens/README.md @@ -0,0 +1,3 @@ +## Tokens subproject + +This is a DBT subproject for the main lineages of tokens data From 080521873e7fbb305fd4d0ca87772091b7d58839 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:37:41 +0100 Subject: [PATCH 49/59] set up as pul_request_target (#7338) --- .github/workflows/pr_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_automation.yml b/.github/workflows/pr_automation.yml index 347e935773c..e43f95343f5 100644 --- a/.github/workflows/pr_automation.yml +++ b/.github/workflows/pr_automation.yml @@ -1,7 +1,7 @@ name: PR automation on: - pull_request: + pull_request_target: types: - opened - ready_for_review From aa33d8d412e4daf26bceb87f96d87b0801d29342 Mon Sep 17 00:00:00 2001 From: maybeyonas Date: Mon, 16 Dec 2024 16:52:54 +0530 Subject: [PATCH 50/59] [branch repeat] [swell] adding daily balances table of core swell assets (#7246) * fuck it we ball ? * dont think the previous incremental balled * trying to fix seed testing * lets try this * adding core asset balances * fix unique key * testing single seed * adding more seeds * [to be reverted] try testing only missing tokens * revert * adding missing swell assets to tokens_ethereum_erc20 * fix earnBTC decimals * add dependancy warning * add seed for earnAssets and rSWELL --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../balances_incremental_subset_daily.sql | 3 + .../swell_balances_ethereum_core_assets.sql | 58 +++++++++++++++++ .../swell_balances_ethereum_schema.yml | 63 +++++++++++++++++++ .../seeds/swell/balances/ethereum/schema.yml | 12 ++++ ...ell_balances_ethereum_core_assets_seed.csv | 8 +++ 5 files changed, 144 insertions(+) create mode 100644 dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_core_assets.sql create mode 100644 dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/schema.yml create mode 100644 dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/swell_balances_ethereum_core_assets_seed.csv diff --git a/dbt_macros/shared/balances_incremental_subset_daily.sql b/dbt_macros/shared/balances_incremental_subset_daily.sql index d397380b258..a8566504dd6 100644 --- a/dbt_macros/shared/balances_incremental_subset_daily.sql +++ b/dbt_macros/shared/balances_incremental_subset_daily.sql @@ -2,6 +2,9 @@ @NOTICE this macro constructs the address level token balances table for given input table @NOTICE aka, you give lists of tokens and/or address, it generates table with daily balances of the address-token pair + + @WARN this macro has a dependancy on erc20.tokens. + @WARN if your token is not in the default list, manually add it via spellbook/dbt_subprojects/tokens/models/tokens//tokens__erc20.sql @PARAM blockchain -- blockchain name @PARAM address_list -- must have an address column, can be none if only filtering on tokens diff --git a/dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_core_assets.sql b/dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_core_assets.sql new file mode 100644 index 00000000000..66bf403601c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_core_assets.sql @@ -0,0 +1,58 @@ +{{ + config( + schema = 'swell_balances_ethereum', + alias = 'core_assets', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['day', 'wallet_address', 'token_address'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.day')], + post_hook='{{ expose_spells(\'["ethereum"]\', + "project", + "swell", + \'["maybeYonas"]\') }}' + ) +}} + +with +tokens as ( + select * from (values + (0xf951E335afb289353dc249e82926178EaC7DEd78, 'swETH', 'Swell LRT'), + (0xFAe103DC9cf190eD75350761e95403b7b8aFa6c0, 'rswETH', 'Swell LRT'), + (0x0a6E7Ba5042B38349e437ec6Db6214AEC7B35676, 'SWELL', 'Swell LRT'), + (0x358d94b5b2F147D741088803d932Acb566acB7B6, 'rSWELL', 'Swell LRT'), + (0x9Ed15383940CC380fAEF0a75edacE507cC775f22, 'earnETH', 'Swell LRT'), + (0x66E47E6957B85Cf62564610B76dD206BB04d831a, 'earnBTC', 'Swell LRT'), + (0x8DB2350D78aBc13f5673A411D4700BCF87864dDE, 'swBTC', 'Swell LRT') + ) as t( + token_address, + symbol, + name + ) +), +balances as ( + {{ + balances_incremental_subset_daily( + blockchain = 'ethereum', + token_list = 'tokens', + start_date = '2023-04-12' + ) + }} +) + +select + -- t.name, + b.blockchain, + b.day, + b.address as wallet_address, + b.token_symbol, + b.token_address, + b.token_standard, + b.token_id, + b.balance, + b.balance_usd, + b.last_updated, + b.next_update +from balances b +-- left join tokens t +-- on b.token_address = t.token_address diff --git a/dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_schema.yml b/dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_schema.yml new file mode 100644 index 00000000000..733fd566ef0 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/swell/balances/ethereum/swell_balances_ethereum_schema.yml @@ -0,0 +1,63 @@ +version: 2 + +models: + - name: swell_balances_ethereum_core_assets + + meta: + blockchain: ethereum + project: swell + contributors: maybeYonas + + config: + tags: ['swell', 'restaking', 'lrt', 'lst', 'vaults', 'balances'] + description: "balances of swell asset holders" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - wallet_address + - token_address + - check_seed: + seed_file: ref('swell_balances_ethereum_core_assets_seed') + match_columns: + - blockchain + - day + - wallet_address + - token_address + check_columns: + - balance + - last_updated + columns: + - &blockchain + name: blockchain + description: "blockchain" + - &day + name: day + description: "date on which the token balance is logged" + - &wallet_address + name: wallet_address + description: "wallet address of the holder" + - &token_symbol + name: token_symbol + description: "token symbol" + - &token_address + name: token_address + description: "token address" + - &token_standard + name: token_standard + description: "standard of the token (erc20)" + - &token_id + name: token_id + description: "ID of the token" + - &balance + name: balance + description: "asset balance of the wallet" + - &balance_usd + name: balance_usd + description: "usd value of token balance of the wallet" + - &last_updated + name: last_updated + description: "UTC timestamp when balance was last updated" + - &next_update + name: next_update + description: "UTC timestamp when balance is next updated" diff --git a/dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/schema.yml b/dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/schema.yml new file mode 100644 index 00000000000..5bf343416bd --- /dev/null +++ b/dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/schema.yml @@ -0,0 +1,12 @@ +version: 2 + +seeds: + - name: swell_balances_ethereum_core_assets_seed + config: + column_types: + blockchain: varchar + day: timestamp + wallet_address: varbinary + token_address: varbinary + balance: double + last_updated: timestamp \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/swell_balances_ethereum_core_assets_seed.csv b/dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/swell_balances_ethereum_core_assets_seed.csv new file mode 100644 index 00000000000..56bd9d1d361 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/seeds/swell/balances/ethereum/swell_balances_ethereum_core_assets_seed.csv @@ -0,0 +1,8 @@ +blockchain,day,wallet_address,token_address,balance,last_updated +ethereum,2024-08-13 00:00:00.000 UTC,0x4589Ef5d52a152C25Daa513cF43b1389Bd1f3C00,0xf951E335afb289353dc249e82926178EaC7DEd78,0.004752378895635013,2024-02-29 00:00:00.000 UTC +ethereum,2024-12-04 00:00:00.000 UTC,0xaf1ec97ea59e9052ee563e4265c39291c020e3dd,0x0a6e7ba5042b38349e437ec6db6214aec7b35676,1074.616281,2024-11-08 00:00:00.000 UTC +ethereum,2024-12-05 00:00:00.000 UTC,0x841ea52b74c94420a61d41ba790e710e1af8ee0a,0xFAe103DC9cf190eD75350761e95403b7b8aFa6c0,0.050121577326075153,2024-11-24 00:00:00.000 UTC +ethereum,2024-08-13 00:00:00.000 UTC,0xfee2d4498085581dde097b9924e4e3544682d767,0x8db2350d78abc13f5673a411d4700bcf87864dde,0.04096165,2024-08-13 00:00:00.000 UTC +ethereum,2024-12-04 00:00:00.000 UTC,0xbdD5d655ad781FB9929BFe515EB4A50090444C21,0x9Ed15383940CC380fAEF0a75edacE507cC775f22,89.828941195874851394,2024-11-08 00:00:00.000 UTC +ethereum,2024-12-01 00:00:00.000 UTC,0x84a6a7c0674A3AA03e09c026600cb46181821f07,0x66E47E6957B85Cf62564610B76dD206BB04d831a,1,2024-11-25 00:00:00.000 UTC +ethereum,2024-12-04 00:00:00.000 UTC,0x22aA3f5D1daFfe1a9DF298e79a0CF2f98C1b92FF,0x358d94b5b2F147D741088803d932Acb566acB7B6,278707.960865665406065408,2024-12-02 00:00:00.000 UTC \ No newline at end of file From 2db18af7428c35bffb1122ce9b633c897c71010c Mon Sep 17 00:00:00 2001 From: viniabussafi <131974393+viniabussafi@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:55:35 +0000 Subject: [PATCH 51/59] update tokens_ethereum_erc20 (#7339) * update tokens_ethereum_erc20 * remove extra line * fix --- .../tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql | 1 + .../tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql b/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql index b185e8e88f4..edf8630b902 100644 --- a/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/arbitrum/tokens_arbitrum_erc20.sql @@ -123,4 +123,5 @@ FROM (VALUES , (0x764bfc309090e7f93edce53e5befa374cdcb7b8e, 'GRIMACE', 18) , (0xB0BDE111812EAC913b392D80D51966eC977bE3A2, 'jUSDC', 18) , (0xf3b7994e4dA53E04155057Fd61dc501599d57877, 'farmdWETHV3', 18) + , (0x521598765b0E5a4CD743f2769C06787b84617252, 'TLP', 18) ) AS temp_table (contract_address, symbol, decimals) diff --git a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql index 56a77d10618..96bf183c18c 100644 --- a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql @@ -4739,4 +4739,5 @@ FROM (VALUES , ( 0x9Ba021B0a9b958B5E75cE9f6dff97C7eE52cb3E6, 'apxETH', 18) , ( 0xa4f6b2cdcb67bf8f48baa459a333d2667aa4b100, 'shezETH', 8) , ( 0x7F4B66FF703336CfC35b901144614496Ae0b0D27, 'shezUSD', 18) + , ( 0xa7b66dda0d7c7c1e43d3130491216172993d163d, 'EVS', 6) ) AS temp_table (contract_address, symbol, decimals) From c134f358a522bab024ecc8e2992eb8aa10c2855f Mon Sep 17 00:00:00 2001 From: hinus <64959125+cxheng315@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:55:49 +0800 Subject: [PATCH 52/59] feat: add wstETH to trusted tokens (#7312) * feat: add wstETH to trusted tokens * Add test --------- Co-authored-by: Huang Geyang --- dbt_subprojects/tokens/models/prices/_schema.yml | 5 +++++ .../tokens/models/prices/prices_trusted_tokens.sql | 1 + 2 files changed, 6 insertions(+) diff --git a/dbt_subprojects/tokens/models/prices/_schema.yml b/dbt_subprojects/tokens/models/prices/_schema.yml index 3e50acf4bd5..0774a3dfa3f 100644 --- a/dbt_subprojects/tokens/models/prices/_schema.yml +++ b/dbt_subprojects/tokens/models/prices/_schema.yml @@ -195,6 +195,11 @@ models: config: tags: [ 'prices', 'stability' ] description: "List of trusted tokens across blockchains" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - contract_address - name: prices_usd_trusted_tokens meta: diff --git a/dbt_subprojects/tokens/models/prices/prices_trusted_tokens.sql b/dbt_subprojects/tokens/models/prices/prices_trusted_tokens.sql index 93aacaa0d63..cb20a76eda0 100644 --- a/dbt_subprojects/tokens/models/prices/prices_trusted_tokens.sql +++ b/dbt_subprojects/tokens/models/prices/prices_trusted_tokens.sql @@ -67,6 +67,7 @@ WITH trusted_tokens AS ( , ('ethereum', 0x6b175474e89094c44da98b954eedeac495271d0f) , ('ethereum', 0x2260fac5e5542a773aa44fbcfedf7c193bc2c599) , ('ethereum', 0xf939e0a03fb07f59a73314e73794be0e57ac1b4e) + , ('ethereum', 0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0) -- wstETH , ('fantom', 0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83) , ('flare', 0x1D80c49BbBCd1C0911346656B529DF9E5c2F783d) , ('fantom', 0x04068da6c83afcfa0e13ba15a6696662335d5b75) From 643d7b9fb12fbcdf7c5ec231bed9e33fa09770e1 Mon Sep 17 00:00:00 2001 From: maybeyonas Date: Mon, 16 Dec 2024 18:02:16 +0530 Subject: [PATCH 53/59] [fluid] adding fluid to dex.trades (#7291) * add fluid pools * minor commet * add block number filter * pool typo fixes and incremental patches * fix source name * adding fluid base trades * add post_hook * add dex source * add schema and info * fix boolean * undo dex info * re add dex info * add seeds * wel wel wel * fix seed * fix seed * incremental fix * adding version * Update dbt_subprojects/dex/models/trades/ethereum/platforms/fluid_v1_ethereum_base_trades.sql Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --------- Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../ethereum/fluid_v1_ethereum_pools.sql | 53 ++++++++++++++++ .../_projects/fluid/ethereum/schema.yml | 54 +++++++++++++++++ dbt_subprojects/dex/models/dex_info.sql | 1 + .../dex/models/trades/ethereum/_schema.yml | 19 ++++++ .../ethereum/dex_ethereum_base_trades.sql | 1 + .../fluid_v1_ethereum_base_trades.sql | 60 +++++++++++++++++++ dbt_subprojects/dex/seeds/trades/_schema.yml | 17 +++++- .../fluid_ethereum_base_trades_seed.csv | 3 + .../_sector/dex/trades/ethereum/_sources.yml | 3 + 9 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/dex/models/_projects/fluid/ethereum/fluid_v1_ethereum_pools.sql create mode 100644 dbt_subprojects/dex/models/_projects/fluid/ethereum/schema.yml create mode 100644 dbt_subprojects/dex/models/trades/ethereum/platforms/fluid_v1_ethereum_base_trades.sql create mode 100644 dbt_subprojects/dex/seeds/trades/fluid_ethereum_base_trades_seed.csv diff --git a/dbt_subprojects/dex/models/_projects/fluid/ethereum/fluid_v1_ethereum_pools.sql b/dbt_subprojects/dex/models/_projects/fluid/ethereum/fluid_v1_ethereum_pools.sql new file mode 100644 index 00000000000..f716aad3990 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/fluid/ethereum/fluid_v1_ethereum_pools.sql @@ -0,0 +1,53 @@ +{{ + config( + schema = 'fluid_ethereum', + alias = 'pools', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['dex'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + post_hook='{{ expose_spells(blockchains = \'["ethereum"]\', + spell_type = "project", + spell_name = "fluid", + contributors = \'["maybeYonas", "pyor_xyz"]\') }}' + ) +}} + +{% set weth_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' %} + +with +decoded_events as ( + select + block_time, + block_number, + index as evt_index, + tx_hash, + contract_address as factory, + substr(topic1, 13) as dex, + substr(topic2, 13) as supplyToken, + substr(topic3, 13) as borrowToken, + bytearray_to_uint256(data) as dexId + from {{ source('ethereum', 'logs')}} + where topic0 = 0x3fecd5f7aca6136a20a999e7d11ff5dcea4bd675cb125f93ccd7d53f98ec57e4 + -- DexT1Deployed -> sample tx: https://etherscan.io/tx/0xabf5c0e676e69de941c283400d7ac5f47b17a09d870f225b5240522f95da501c#eventlog + and block_number > 20776998 + {% if is_incremental() %} + and {{ incremental_predicate('block_time') }} + {% endif %} +) + +select + 'ethereum' as blockchain, + 'fluid' as project, + '1' as version, + block_time, + block_number, + evt_index, + tx_hash, + factory, + dex, + case supplyToken when 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee then {{weth_address}} else supplyToken end as supply_token, + case borrowToken when 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee then {{weth_address}} else borrowToken end as borrow_token, + dexId as dex_id +from decoded_events \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/fluid/ethereum/schema.yml b/dbt_subprojects/dex/models/_projects/fluid/ethereum/schema.yml new file mode 100644 index 00000000000..8a955decee1 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/fluid/ethereum/schema.yml @@ -0,0 +1,54 @@ +version: 2 + +models: + - name: fluid_v1_ethereum_pools + meta: + blockchain: ethereum + sector: dex + project: fluid + contributor: maybeYonas + config: + tags: ['ethereum', 'trades', 'fluid', 'dex'] + description: "fluid pools" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - dex + columns: + - &blockchain + name: blockchain + description: "blockchain where the pool is deployed" + - &project + name: project + description: "project name" + - &version + name: version + description: "project version" + - &block_time + name: block_time + description: "utc block timestamp when the pool is deployed" + - &block_number + name: block_number + description: "block number at which the pool is deployed" + - &evt_index + name: evt_index + description: "index of log emitted" + - &tx_hash + name: tx_hash + description: "hash of tx in which pool is deployed" + - &factory + name: factory + description: "factory contract deploying the pool" + - &dex + name: dex + description: "pool contract address" + - &supply_token + name: supply_token + description: "supply token" + - &borrow_token + name: borrow_token + description: "borrow token" + - &dex_id + name: dex_id + description: "nonce unique to factory and pool" + diff --git a/dbt_subprojects/dex/models/dex_info.sql b/dbt_subprojects/dex/models/dex_info.sql index 2de3399cc3c..678dbf2100d 100644 --- a/dbt_subprojects/dex/models/dex_info.sql +++ b/dbt_subprojects/dex/models/dex_info.sql @@ -192,4 +192,5 @@ FROM (VALUES , ('katana', 'Katana', 'Direct', 'AxieInfinity') , ('jetswap', 'JetSwap', 'Direct', 'Jetfuelfinance') , ('levinswap', 'Levinswap', 'Direct', 'levinswap') + , ('fluid', 'Fluid DEX', 'Direct', '0xfluid') ) AS temp_table (project, name, marketplace_type, x_username) diff --git a/dbt_subprojects/dex/models/trades/ethereum/_schema.yml b/dbt_subprojects/dex/models/trades/ethereum/_schema.yml index 9290c868bf6..5b33a0992c0 100644 --- a/dbt_subprojects/dex/models/trades/ethereum/_schema.yml +++ b/dbt_subprojects/dex/models/trades/ethereum/_schema.yml @@ -645,3 +645,22 @@ models: combination_of_columns: - tx_hash - evt_index + + - name: fluid_v1_ethereum_base_trades + meta: + blockchain: ethereum + sector: dex + project: fluid + contributors: maybeYonas, pyor_xyz + config: + tags: ["ethereum", "dex", "trades", "fluid"] + description: "fluid dex ethereum base trades" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + - evt_index + - check_dex_base_trades_seed: + seed_file: ref('fluid_ethereum_base_trades_seed') + filter: + version: 1 diff --git a/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql b/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql index 25e4ae0e1b8..7dc42461e0d 100644 --- a/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql +++ b/dbt_subprojects/dex/models/trades/ethereum/dex_ethereum_base_trades.sql @@ -40,6 +40,7 @@ , ref('swaap_v2_ethereum_base_trades') , ref('valantis_hot_ethereum_base_trades') , ref('xchange_v2_ethereum_base_trades') + , ref('fluid_v1_ethereum_base_trades') ] %} WITH base_union AS ( diff --git a/dbt_subprojects/dex/models/trades/ethereum/platforms/fluid_v1_ethereum_base_trades.sql b/dbt_subprojects/dex/models/trades/ethereum/platforms/fluid_v1_ethereum_base_trades.sql new file mode 100644 index 00000000000..2c6008d451b --- /dev/null +++ b/dbt_subprojects/dex/models/trades/ethereum/platforms/fluid_v1_ethereum_base_trades.sql @@ -0,0 +1,60 @@ +{{ + config( + schema = 'fluid_v1_ethereum', + alias = 'base_trades', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['tx_hash', 'evt_index'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +with +decoded_events as ( + select + '1' as version, + t.evt_block_number as block_number, + t.evt_block_time as block_time, + t.to as taker, + cast(null as varbinary) as maker, + t.amountOut as token_bought_amount_raw, + t.amountIn as token_sold_amount_raw, + case when swap0to1 + then p.borrow_token + else p.supply_token + end as token_bought_address, + case when not(swap0to1) + then p.borrow_token + else p.supply_token + end as token_sold_address, + t.contract_address as project_contract_address, + t.evt_tx_hash as tx_hash, + t.evt_index + from {{ source('fluid_ethereum', 'FluidDexT1_evt_Swap') }} t + inner join {{ ref('fluid_v1_ethereum_pools') }} p + on t.contract_address = p.dex + {% if is_incremental() %} + where {{ incremental_predicate('t.evt_block_time') }} + {% endif %} +) + + +SELECT + 'ethereum' as blockchain, + 'fluid' as project, + dexs.version, + cast(date_trunc('month', dexs.block_time) as date) as block_month, + cast(date_trunc('day', dexs.block_time) as date) as block_date, + dexs.block_time, + dexs.block_number, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + dexs.token_bought_address, + dexs.token_sold_address, + dexs.taker, + dexs.maker, + dexs.project_contract_address, + dexs.tx_hash, + dexs.evt_index +FROM decoded_events dexs diff --git a/dbt_subprojects/dex/seeds/trades/_schema.yml b/dbt_subprojects/dex/seeds/trades/_schema.yml index 548192dab81..1bc3dd58d97 100644 --- a/dbt_subprojects/dex/seeds/trades/_schema.yml +++ b/dbt_subprojects/dex/seeds/trades/_schema.yml @@ -4559,7 +4559,7 @@ seeds: token_bought_amount_raw: uint256 token_sold_amount_raw: uint256 block_date: timestamp - + - name: levinswap_gnosis_base_trades_seed config: column_types: @@ -4574,3 +4574,18 @@ seeds: token_bought_amount_raw: uint256 token_sold_amount_raw: uint256 block_date: timestamp + + - name: fluid_ethereum_base_trades_seed + config: + column_types: + blockchain: varchar + project: varchar + version: varchar + tx_hash: varbinary + evt_index: uint256 + block_number: uint256 + token_bought_address: varbinary + token_sold_address: varbinary + token_bought_amount_raw: uint256 + token_sold_amount_raw: uint256 + block_date: timestamp diff --git a/dbt_subprojects/dex/seeds/trades/fluid_ethereum_base_trades_seed.csv b/dbt_subprojects/dex/seeds/trades/fluid_ethereum_base_trades_seed.csv new file mode 100644 index 00000000000..79406267b36 --- /dev/null +++ b/dbt_subprojects/dex/seeds/trades/fluid_ethereum_base_trades_seed.csv @@ -0,0 +1,3 @@ +blockchain,project,version,block_date,tx_hash,evt_index,token_bought_address,token_sold_address,block_number,token_bought_amount_raw,token_sold_amount_raw +ethereum,fluid,1,2024-12-10,0x6d3da988fdce7dec28e2e61cca6704cda8c9a42faf0a62d35f9a10765b59799c,350,0xdac17f958d2ee523a2206206994597c13d831ec7,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,21374285,99976540470,100000000000 +ethereum,fluid,1,2024-10-30,0x700f7f91ce515c42872e70ce896241c6feb4cdd3390ff40853e6131cfd9361bc,400,0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,21074822,144552151975089000000,171000000000000000000 \ No newline at end of file diff --git a/sources/_sector/dex/trades/ethereum/_sources.yml b/sources/_sector/dex/trades/ethereum/_sources.yml index d95c3bb0c06..5c63d9043bc 100644 --- a/sources/_sector/dex/trades/ethereum/_sources.yml +++ b/sources/_sector/dex/trades/ethereum/_sources.yml @@ -129,3 +129,6 @@ sources: - name: SovereignPool_evt_Swap - name: ProtocolFactory_evt_SovereignPoolDeployed - name: HOT_evt_HotSwap + - name: fluid_ethereum + tables: + - name: FluidDexT1_evt_Swap From cec63b1b1d011dc0d26f84dfba1224e2c9144e53 Mon Sep 17 00:00:00 2001 From: whale_hunter <143016036+whalehunting@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:32:23 +0100 Subject: [PATCH 54/59] fix: flokibot spellbook missing trades (#7335) * chore: add openocean aggregator trades * fix: fix table name * fix: add missing sources --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../ethereum/flokibot_ethereum_bot_trades.sql | 13 +++++++++++++ .../ethereum/openocean_ethereum_sources.yml | 6 ++++++ 2 files changed, 19 insertions(+) create mode 100644 sources/openocean/ethereum/openocean_ethereum_sources.yml diff --git a/dbt_subprojects/dex/models/bot_trades/flokibot/ethereum/flokibot_ethereum_bot_trades.sql b/dbt_subprojects/dex/models/bot_trades/flokibot/ethereum/flokibot_ethereum_bot_trades.sql index d8991b7cbfd..767d546a6c9 100644 --- a/dbt_subprojects/dex/models/bot_trades/flokibot/ethereum/flokibot_ethereum_bot_trades.sql +++ b/dbt_subprojects/dex/models/bot_trades/flokibot/ethereum/flokibot_ethereum_bot_trades.sql @@ -64,12 +64,25 @@ with {% else %} and call_block_time >= timestamp '{{project_start_date}}' {% endif %} ), + openocean_aggregator_trades as ( + select evt_block_time as block_time, evt_tx_hash as tx_hash + from {{ source('openocean_v2_ethereum', 'OpenOceanExchangeProxy_evt_Swapped') }} + where + referrer = {{ treasury_fee_wallet_2 }} + {% if is_incremental() %} + and {{ incremental_predicate('evt_block_time') }} + {% else %} and evt_block_time >= timestamp '{{project_start_date}}' + {% endif %} + ), trade_transactions as ( select block_time, address, null as tx_hash from bot_contracts union all select block_time, null as address, tx_hash from oneinch_aggregator_trades + union all + select block_time, null as address, tx_hash + from openocean_aggregator_trades ), bot_trades as ( select diff --git a/sources/openocean/ethereum/openocean_ethereum_sources.yml b/sources/openocean/ethereum/openocean_ethereum_sources.yml new file mode 100644 index 00000000000..7058039ffd1 --- /dev/null +++ b/sources/openocean/ethereum/openocean_ethereum_sources.yml @@ -0,0 +1,6 @@ +version: 2 + +sources: + - name: openocean_v2_ethereum + tables: + - name: OpenOceanExchangeProxy_evt_Swapped From a22be37f72c1531eb0e4aa5806b2554d674ab000 Mon Sep 17 00:00:00 2001 From: Haris Angelidakis <64154020+harisang@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:34:15 +0200 Subject: [PATCH 55/59] update furucombo name on gc and arbitrum (#7333) Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql | 4 ++-- .../cow_protocol/gnosis/cow_protocol_gnosis_solvers.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql index 22feb72c40a..cfda80c9661 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_solvers.sql @@ -45,7 +45,7 @@ known_solver_metadata (address, environment, name) as ( (0xc46Ac109FDe084192BE59C24C3680D818763b0fd, 'prod', 'Gnosis_ParaSwap'), (0xD31E0CE8154Da6b8086d961eB3068Ef74C4322b6, 'prod', 'Gnosis_0x'), (0xAa224676d096B6Fc257F8C386C67d9e96e53AD59, 'prod', 'Gnosis_BalancerSOR'), - (0x5932b2c05172aAfE097CE0Fbd27d18a7d5A45eE1, 'prod', 'Furucombo'), + (0x5932b2c05172aAfE097CE0Fbd27d18a7d5A45eE1, 'prod', 'Portus'), (0x3A485742Bd85e660e72dE0f49cC27AD7a62911B5, 'prod', 'Seasolver'), (0x059aefdF9d9F47def37cF7066DA83fEB91fDd089, 'prod', 'Barter'), (0x40798d2261f8b7F11BFa73623c99C876844dD05A, 'prod', 'OpenOcean_Aggregator'), @@ -64,7 +64,7 @@ known_solver_metadata (address, environment, name) as ( (0x9C803d345615aDe1e5ae07A789968403fAc9171a, 'barn', 'Gnosis_ParaSwap'), (0x69433b336952e84Db44EF40b16B338F139B8baA1, 'barn', 'Gnosis_0x'), (0xCED55FC88186f672105712fe177374cce4861FDF, 'barn', 'Gnosis_BalancerSOR'), - (0xE376a730037D8B495864FD5ed6373BE89643cD06, 'barn', 'Furucombo'), + (0xE376a730037D8B495864FD5ed6373BE89643cD06, 'barn', 'Portus'), (0x2633bd8e5FDf7C72Aca1d291CA11bdB717A6aa3d, 'barn', 'Seasolver'), (0x7B0211286d8Dfdb717f4A2E5Fa5131eD911097e1, 'barn', 'Barter'), (0xc8371B2898FBaEeAe658f9FaeE8ddeDA24e37012, 'barn', 'OpenOcean_Aggregator'), diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_solvers.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_solvers.sql index dd7b75e11b0..3710ace1cd4 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_solvers.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_solvers.sql @@ -61,14 +61,14 @@ known_solver_metadata (address, environment, name) as ( (0x4930a9012e8677ae764e44f2b46af8087a1f9f8e, 'barn', 'Gnosis_BalancerSOR'), (0x8e600b399Da9c46255ccac98764987cF81837a66, 'barn', 'Enso'), (0xC8D2f12a9505a82C4f6994204f4BbF095183E63A, 'barn', 'Seasolver'), - (0x67be9614C4E0FCdA95AFC66a95B5BDAFb55fa362, 'barn', 'Furucombo'), + (0x67be9614C4E0FCdA95AFC66a95B5BDAFb55fa362, 'barn', 'Portus'), (0x53F5378A6f8bb24333aD8D68FD28816504a467b2, 'barn', 'Copium_Capital'), (0xC4dd6355Fbc6Eb108FD1C100389789C5B1A9A980, 'barn', 'Barter'), (0x4398129426Cb1377E9E10395b8dfBDa153c7Fe7D, 'barn', 'Fractal'), (0x727EB77c6f84ef148403f641aA32d75b7f6902A7, 'prod', 'Fractal'), (0x0a360134553feED49FE5eb273074d80B6e45941F, 'prod', 'Barter'), (0xb4694FE6590acd1281Dc34a966bbAE224559BaD4, 'prod', 'Copium_Capital'), - (0x227FDA1D5970dF605D785Bf5F2F8899d5fdF8624, 'prod', 'Furucombo'), + (0x227FDA1D5970dF605D785Bf5F2F8899d5fdF8624, 'prod', 'Portus'), (0xE3068acB5b5672408eADaD4417e7d3BA41D4FEBe, 'prod', 'Seasolver'), (0x12c53cdD1ef150E1cd291DD210b761acFADA6B9C, 'prod', 'Enso'), (0xf671d28fef15e5eafc21898c2814b1b4cd88bc9a, 'prod', 'Gnosis_BalancerSOR'), From 1ca58e95a657839dfe58f350593cefd607b3a615 Mon Sep 17 00:00:00 2001 From: clizzard <145153355+clizzard7@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:55:14 +0100 Subject: [PATCH 56/59] add sanji (#7337) * add sanji * fixed seed --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../_sector/dex/bot_trades/solana/_schema.yml | 22 +++ .../solana/dex_solana_bot_trades.sql | 1 + .../platforms/sanji_solana_bot_trades.sql | 152 ++++++++++++++++++ .../seeds/sanji/sanji_solana_trades_seed.csv | 21 +++ 4 files changed, 196 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/sanji_solana_bot_trades.sql create mode 100644 dbt_subprojects/solana/seeds/sanji/sanji_solana_trades_seed.csv diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml index bdf23c8aba1..413d474f8a0 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml @@ -733,3 +733,25 @@ models: - check_bot_trades_seed: seed_file: ref('prophetbots_solana_trades_seed') blockchain: solana + + - name: sanji_solana_bot_trades + meta: + blockchain: solana + sector: dex + project: sanji + contributors: whale_hunter + config: + tags: ["solana", "dex", "sanji", "trades"] + description: > + Sanji trades on Solana + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_id + - tx_index + - outer_instruction_index + - inner_instruction_index + - check_bot_trades_seed: + seed_file: ref('sanji_solana_trades_seed') + blockchain: solana diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql index 1432bf2fbb6..a047ad493be 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql @@ -38,6 +38,7 @@ , ref('bitfoot_solana_bot_trades') , ref('tradewiz_solana_bot_trades') , ref('prophetbots_solana_bot_trades') + , ref('sanji_solana_bot_trades') ] %} {% for bot in solana_trading_bot %} diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/sanji_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/sanji_solana_bot_trades.sql new file mode 100644 index 00000000000..03cc106c17b --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/sanji_solana_bot_trades.sql @@ -0,0 +1,152 @@ +{{ config( + alias = 'bot_trades', + schema = 'sanji_solana', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] + ) +}} + +{% set project_start_date = '2024-11-21' %} +{% set fee_receiver_1 = '4E64WX4EARRMfHsvL4ZXbrbpiPcBUyrC62uawGofhdNN' %} +{% set wsol_token = 'So11111111111111111111111111111111111111112' %} + +WITH + allFeePayments AS ( + SELECT + tx_id, + 'SOL' AS feeTokenType, + balance_change / 1e9 AS fee_token_amount, + '{{wsol_token}}' AS fee_token_mint_address + FROM + {{ source('solana','account_activity') }} + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND tx_success + AND balance_change > 0 + AND ( + address = '{{fee_receiver_1}}' + ) + ), + botTrades AS ( + SELECT + trades.block_time, + CAST(date_trunc('day', trades.block_time) AS date) AS block_date, + CAST(date_trunc('month', trades.block_time) AS date) AS block_month, + 'solana' AS blockchain, + amount_usd, + IF( + token_sold_mint_address = '{{wsol_token}}', + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_mint_address AS token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_mint_address AS token_sold_address, + fee_token_amount * price AS fee_usd, + fee_token_amount, + IF(feeTokenType = 'SOL', 'SOL', symbol) AS fee_token_symbol, + fee_token_mint_address AS fee_token_address, + project, + version, + token_pair, + project_program_id AS project_contract_address, + trader_id AS user, + trades.tx_id, + tx_index, + outer_instruction_index, + inner_instruction_index + FROM + {{ ref('dex_solana_trades') }} AS trades + JOIN allFeePayments AS feePayments ON trades.tx_id = feePayments.tx_id + LEFT JOIN {{ source('prices', 'usd') }} AS feeTokenPrices ON ( + feeTokenPrices.blockchain = 'solana' + AND fee_token_mint_address = toBase58 (feeTokenPrices.contract_address) + AND date_trunc('minute', block_time) = minute + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + JOIN {{ source('solana','transactions') }} AS transactions ON ( + trades.tx_id = id + {% if is_incremental() %} + AND {{ incremental_predicate('transactions.block_time') }} + {% else %} + AND transactions.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + WHERE + trades.trader_id != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestInnerInstructionIndexForEachTrade AS ( + SELECT + tx_id, + outer_instruction_index, + MAX(inner_instruction_index) AS highestInnerInstructionIndex + FROM + botTrades + GROUP BY + tx_id, + outer_instruction_index + ) +SELECT + block_time, + block_date, + block_month, + 'Sanji' as bot, + blockchain, + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + project, + version, + token_pair, + project_contract_address, + user, + botTrades.tx_id, + tx_index, + botTrades.outer_instruction_index, + COALESCE(inner_instruction_index, 0) AS inner_instruction_index, + IF( + inner_instruction_index = highestInnerInstructionIndex, + true, + false + ) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestInnerInstructionIndexForEachTrade ON ( + botTrades.tx_id = highestInnerInstructionIndexForEachTrade.tx_id + AND botTrades.outer_instruction_index = highestInnerInstructionIndexForEachTrade.outer_instruction_index + ) +ORDER BY + block_time DESC, + tx_index DESC, + outer_instruction_index DESC, + inner_instruction_index DESC diff --git a/dbt_subprojects/solana/seeds/sanji/sanji_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/sanji/sanji_solana_trades_seed.csv new file mode 100644 index 00000000000..379c07d13cc --- /dev/null +++ b/dbt_subprojects/solana/seeds/sanji/sanji_solana_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction +2024-12-15 07:06:47.000 UTC,2024-12-15,2024-12-01,Sanji,solana,2.12730476036,Sell,0.009734612,SOL,So11111111111111111111111111111111111111112,632.120372,timmy,4MH3cmHGELMSWeWsYVQJFgb9tmed26ZTUVQ3jkANpump,0.014851298800000001,0.00006796,SOL,So11111111111111111111111111111111111111112,raydium,4,timmy-SOL,a5cEyYEG1RaBTpsY1QapYKDv6ixvbD6CVkisSvT5Ufo,7rqre26G9XjaEjbwtSHCU97LoU1KY7LGAthXNDyygLUD,4GdbkGaaMWoPM14vJsNWCMQ4vpnMKzJAow2ZZEXe1BnGtzoMyBq5GfLBK8X6W1XihZBPV7x677z47DAtYvv7Qwt4,1263,5,0,true +2024-12-14 21:57:46.000 UTC,2024-12-14,2024-12-01,Sanji,solana,10.8765,Buy,108399.066862,ROJO,HANuXquoXxPQuUdWq2pSonH3GUwjfwg9wwxeP8do6bWV,0.05,SOL,So11111111111111111111111111111111111111112,0.0761355,0.00035,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-ROJO,27gMxh6F68GoctWNp3YyLhrMwaZsLzwK6WDLyVQCQUmd,FAZuZgyYsYQjkJD6o3YaVwz4MdvitFDWoA4DvWmR8mS5,62qqXEMGk6uiSLBfscjbcLP7aAPHtK7uvmUmxq5akQ7FE6eHeM5axrZk6J18Wc11GstxEdgoMEKn8akZwApXjmDM,1632,6,0,true +2024-12-13 23:30:55.000 UTC,2024-12-13,2024-12-01,Sanji,solana,1.25854035633,Sell,0.005622751,SOL,So11111111111111111111111111111111111111112,19013.576389,ipod,5wQPBazUaVtMTqtAGXYPdDSenxDqM5vJiDsEfov7pump,0.007536356100000001,0.00003367,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-ipod,A7LAumFoyJFTR8V9ChoSP8iSpquVacpZHva1oZ2JqS16,8Z4zvnXqR7bgJttHKpAP2SUa1TuzmXjqLj7pEbEzMmkd,2kgteAdXaZYthVsUoxHGmQGJKBGbgEahA4nGSaYh3eqzYVGKm6ovtm3UynpcKqSPX21Zdz6GfyEnGtUDdShRS2FB,2946,3,2,true +2024-12-07 16:58:50.000 UTC,2024-12-07,2024-12-01,Sanji,solana,190.39411413672,Sell,0.786817564,SOL,So11111111111111111111111111111111111111112,856847975790.57,DNA,DiSetnR7k57wmfvywJhUVjPwWfg54SdQKxQdJEBYW23B,1.3498394538,0.00557831,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-DNA,45L3EuUJKyqhkNDfocrd3yAZu8nzMPXiHtwhjCvidgua,B273T7o6o1Cn9uy4uByzKxpHGqqALAPo9JuZBp9ctZbK,5vBFyx35NGM1Z8r8iB8TLDpVA3mK7LhqqUu5E2iTbJU6UNxPxmMFTTwspqa2Zh6FUKizddQVH2vtEQELWz3rLXbS,617,5,0,true +2024-12-06 01:33:22.000 UTC,2024-12-06,2024-12-01,Sanji,solana,2.48840085109,Sell,0.010124917,SOL,So11111111111111111111111111111111111111112,2.5,USDT,Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB,0.0174201776,0.00007088,SOL,So11111111111111111111111111111111111111112,raydium,4,USDT-SOL,7XawhbbxtsRcQA8KTkHT9f9nc6d69UwqCDh6U5EEbEmX,EYzmfW4iqDGNhgTiyA71ETMLnSRPJEKykYzDwNWBQFYv,5sY1pA9d1NY7byu4pK6FAtNGjASU51zv5guRb31HNws5pGVg88XEpMNbPULSCo3GbYhbzgRgKynyejhxfWNodBHg,895,5,0,true +2024-12-07 06:41:44.000 UTC,2024-12-07,2024-12-01,Sanji,solana,0.0774845692,Sell,0.00032884,SOL,So11111111111111111111111111111111111111112,6503.817475,Trump47,PfzhpjdDd2rHLRLWyTQBbqB6KBJfx8yVVGqSYkq5bqC,0.000541949,0.0000023,SOL,So11111111111111111111111111111111111111112,raydium,4,Trump47-SOL,2kcuRFMvJvqe82E7byJBCDMn4LT4zNFb2FpPhVbdqCw2,BLZ2UfxRoJ8arGKgAtArQ1SDFDttFA8c6v4qyPGpZLaq,4MNycd2wXRvzR8hikbS5Wsudn3tmf2vHjzxsuXSk5fvBzWW5oYFGLfv7pH1Pdo1eTL2kno7jwsDbVEFz5izKRboM,1480,5,0,true +2024-12-12 22:57:10.000 UTC,2024-12-12,2024-12-01,Sanji,solana,45.932176694949995,Sell,0.201342115,SOL,So11111111111111111111111111111111111111112,7151571.722822,JAILFORJAY,3KSkAdFbPHfGREVEo3DcGDBHEdZ8Ey3zGgFSajYVpump,0.1618422659,0.00070943,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-JAILFORJAY,Exj9V36qy1esrknFo4SAi8728BEGcrXPLwquDHfLtNqZ,8ChErCicUoNVmKbMuzVzoSoVn9s9Fkrsf8f2PoFKWrbZ,5un9ZXyeZfrYrEutX3dVG2dY76GWViTmVR4KVhfKBJQoqixsW2aZisRr31mEz86pfhoUoTWnVsqBqW5qrpyL8RPX,164,3,2,true +2024-12-12 03:19:52.000 UTC,2024-12-12,2024-12-01,Sanji,solana,67.28554651878,Sell,0.293643827,SOL,So11111111111111111111111111111111111111112,629457.273508496,CHILLSLUT,4F33vogWv87XcpGYL4pPXBfmXc5XBEKqPTVYrJrFpump,0.4643155476,0.00202634,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-CHILLSLUT,CYHPMpq6paZHLe24um42WPQMNRGNGTK4Q2LgDFWPZEdS,Ghcb5NXNE8KwvRYob3b8oGy6g5GF9z3Gd4rfrujCRAuL,TckxHyY7orvMD2Aidin1yWhpZM6L614TP8wV4zJTxN1nwgpSmS9AXyzrhG5i7vNp7DJvuurs1aQwPXyqwKEvHc1,757,5,0,true +2024-12-08 02:52:58.000 UTC,2024-12-08,2024-12-01,Sanji,solana,200.38093529355,Sell,0.831697735,SOL,So11111111111111111111111111111111111111112,1748801.699025,T51,4zGCj2Rq16sp3fFdzZHbVUdAqeS4p9buThjKKRQGpump,1.4026679577,0.00582189,SOL,So11111111111111111111111111111111111111112,raydium,4,T51-SOL,GAjizQ2pXjkT962BU9ge7iKkaJtVYvQyHvPfV4vVR9YN,4o3DSxj7rj6mJUNYYCy4vm3dKDwteZVR27K1XT8eSrqu,4BwNnddS7SJpi2oUGH4F9KWo4sQp7RoHERbJtn7eevAshAToZagzR4neQHc867ueKvFa2aU7PmYS3L3xJtEJnbyN,1722,5,0,true +2024-12-07 16:58:51.000 UTC,2024-12-07,2024-12-01,Sanji,solana,7.07179338996,Sell,0.029224702,SOL,So11111111111111111111111111111111111111112,3750224.749119,USA,69kdRLyP5DTRkpHraaSZAQbWmAwzF9guKjZfzMXzcbAs,0.0494800704,0.00020448,SOL,So11111111111111111111111111111111111111112,raydium,4,USA-SOL,HKprCtGbnh1j8xeQggzWhhVd3kwDUdphqPqDP8vMay8b,7wyuQVaYkPxu7oTp1juBXCpmjXM1JvjtBH2bDmWWjddp,xXHQ2v53E2fDymBmad8vQtQZCapEX3MFMr2k5vLgB3vjZV5sdKFtxnGaL8e7JrnYLLcA4cT5t2BbnYZGfhdaDqP,391,5,0,true +2024-12-01 04:48:17.000 UTC,2024-12-01,2024-12-01,Sanji,solana,1.5498037895799999,Sell,0.006555299,SOL,So11111111111111111111111111111111111111112,2707.23956,SNOOPY,CSh9t2fx2TAE25AiZ9oagdcuHomTQoCE1PGxaMjfpump,0.0112039438,0.00004739,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-SNOOPY,6NZ8USQQCiy2dAe6Qg7GZGMRS1z86qq2iH8bokD271ht,BcGqkk3QVi4XmN4BN4deJSvGL1ss8AwiM3n1L7BouFr1,47fBahJ11QJTSkcn2qqdMTc69nQ53yZW5yqu48DzTiLckYu2hwMJATMmQsjhoAGVUDSsUphocrPsFmH7VgL6nEUb,987,5,0,true +2024-12-12 06:28:07.000 UTC,2024-12-12,2024-12-01,Sanji,solana,93.99705161856001,Sell,0.408540732,SOL,So11111111111111111111111111111111111111112,2598.06085,APPLE,H33XL6HHDReCVRgSApZpsXM7Hy7JGyLztRJaGxjapump,0.6579804832,0.00285979,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-APPLE,BEMi9yKdXLL9rNY43cU8CVxZZP4hmr7w7odwtxAMKAzD,vv7iSWi5CFuP9GCv1ufASHBDz6kFNDEWq5nhnpKaLUt,3xS9c2bGTWoTkXfFtRriP8DYTUTm29tqdkpEvCWLe6KJtMDdqvJjgmer3QWffWVYijwkp9ouDTBVVfvjKLmeRnZc,1708,5,0,true +2024-12-04 03:46:15.000 UTC,2024-12-04,2024-12-01,Sanji,solana,7.153499999999999,Buy,99989.752594,JIZZ,Ggx1nVeY23M16r2YqgeJfXMqA5WCGmTQN8RqhaXWPbTL,0.03,SOL,So11111111111111111111111111111111111111112,0.0500745,0.00021,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-JIZZ,EhY8JFCUyPuqJpdNiU9shKjToQb1b6qycmMwdx1o2LSv,2fBwpC4QGeExBWDfpGSqTNMC8xqWMu4Q8gSJYDYiSxta,3QUbFCvFCRDcarhnprUfwDpMFd5mJc7Mp9RXAyqRmmgJWydnQgaBe8i9VgyjKDp3JbL2JwUjqBQhN8qzgd65ugGa,1605,6,0,true +2024-12-12 14:40:20.000 UTC,2024-12-12,2024-12-01,Sanji,solana,4.125138145739999,Sell,0.017878638,SOL,So11111111111111111111111111111111111111112,234.904987,TRUST,EqeEBGHQhQy6SqeaJcnqAsNs3qaG19sdF89Xsarpump,0.02884125,0.000125,SOL,So11111111111111111111111111111111111111112,raydium,4,TRUST-SOL,9dkWXEmCvQGLazzbWZiSDExoPKiw7HkwPSM5bJW8qymL,BtZ1iVeH6h98fgi88pF6qj9oG41v42n3yndpSMDif4kh,5fx2StwS691CMAcJgk7jSPwuUVg1nuWtyKbdfrTNVg62r3uqzejo8yH3o4vYGUGht2yG84HXTJEEFSFtMkcgcqk7,3037,5,0,true +2024-12-12 00:13:26.000 UTC,2024-12-12,2024-12-01,Sanji,solana,3.06566549352,Sell,0.013495622,SOL,So11111111111111111111111111111111111111112,476869.372056,AWWTIC,BPcCs1fer87KzqgkHHCvR36SkfDqaEcLGxt62fkkpump,0.0214620768,0.00009448,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-AWWTIC,4icnNdfDN3FyzTpXBBJWZ5BRHnjvLRAkgK5QhhxDHSD7,8UWDczTGjzjnwmQ3Ah6jcN5qPxCT9oWB3BKhfWyW9G6R,59nJmeny6UzHEeRdUMG1UhVY1xE74yJrR7o9rLauHfPUAsg2yR38oK3mM8sDGtYV8uxXAFmFtmTqH7zmW4FQtLuj,1926,5,0,true +2024-12-12 20:06:07.000 UTC,2024-12-12,2024-12-01,Sanji,solana,6.8361,Buy,37511.994306,CHAINSAW,7nFvyQr2mwHLjBBECj3MsUxeAbP2D89p7Tx6bQtBpump,0.03,SOL,So11111111111111111111111111111111111111112,0.047852700000000005,0.00021,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-CHAINSAW,Eg4hFsUc2MwiBumykUe5dnQws37M3P3WzdGFKpQuqp5t,AQLntiBeqNvynXjrWJv4MSmMFURSfrSbasjUbqwHyWN1,uhCyivWoQUksTryEegnTYxRjrXAp3aeBWRyc8q8Kr99FPj3m1rGZm8tq67fRsbqBSkUs6y4NL7jFGhMEoG9Ke5v,2138,6,0,true +2024-12-13 20:05:17.000 UTC,2024-12-13,2024-12-01,Sanji,solana,0.67938346075,Sell,0.003043151,SOL,So11111111111111111111111111111111111111112,2200.998928,EVOLVE,HQ7NoKrgxcep1hSnNKGbWRtpMf59As5eVKeuwyCzpump,0.0043690025,0.00001957,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-EVOLVE,6J3DXzGB9Ndw795pAfH65HQSDo4MwgRbFDpSF27L3g4S,9zxuSu9cp5euiHaB9NAZ1Y6PhtASWmpetDhvrAUhZBZm,44UWzYjiCcsG6RS81CdCbzQjFaih8ppkkxedMUoZJ2qroSn8Tqov1EXzQfiPvh8cPABK3sCXCdmBppoHSXMFxTFx,1962,5,0,true +2024-12-12 20:47:31.000 UTC,2024-12-12,2024-12-01,Sanji,solana,0.01325559392,Sell,0.000057814,SOL,So11111111111111111111111111111111111111112,0.00000341,WETH,7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs,0.0000940048,4.1e-7,SOL,So11111111111111111111111111111111111111112,raydium,4,WETH-SOL,4yrHms7ekgTBgJg77zJ33TsWrraqHsCXDtuSZqUsuGHb,6VvTvxNoZzHVgivqeFrzBcKK8N9ckMGbMTxcNgbB8MVo,2YjFHGk3YnWdLukUzUmZBZhtQbh7rhoVtSxDDEKtuCVqun85E2DG7tE13D8S9XuiEs58PbfHcBF7BJ61DKWY8wyP,1634,5,0,true +2024-12-12 13:26:56.000 UTC,2024-12-12,2024-12-01,Sanji,solana,0.0017850892199999999,Sell,0.000007698,SOL,So11111111111111111111111111111111111111112,4.5e-7,WETH,7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs,0.000013681509999999998,5.9e-8,SOL,So11111111111111111111111111111111111111112,raydium,4,WETH-SOL,4yrHms7ekgTBgJg77zJ33TsWrraqHsCXDtuSZqUsuGHb,6VvTvxNoZzHVgivqeFrzBcKK8N9ckMGbMTxcNgbB8MVo,FQBE9S8LZTiSvtMCU6qYwtRbvDCG2syeLGXUMq2whe4ALcYHNixAYxSsNY67F3QLACHYjhM1HojoeU7nPvko9J8,1019,5,0,true +2024-12-01 00:19:25.000 UTC,2024-12-01,2024-12-01,Sanji,solana,11.9225,Buy,345972.31649,GRINCHWIF,BoQPkPXdNqRuwyPpkC47Ac9mCk2WwhbgbDG5dt7spump,0.05,SOL,So11111111111111111111111111111111111111112,0.08345749999999999,0.00035,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-GRINCHWIF,ALkGqJ7inoN4n2wW6exur43YQRSHZ5dP3DpQ879DwM85,77PPF9tCqn3PsrYs7WhgKEpy76b3Uzh1i4B93BmoSq2G,4caUnR34GpNUDzeYWDWJZQEP7B6rycCLAoL6sDnYuzDF8ti7wXqsxDqoYuocumX4B7Nmzup8RyZxschkB9ZAQ2tu,1180,6,0,true From 39d698d8fcfad8121a0fe5b57196bbfe23608240 Mon Sep 17 00:00:00 2001 From: clizzard <145153355+clizzard7@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:58:01 +0100 Subject: [PATCH 57/59] add nova (#7307) * add nova * fixed seed * rename schema --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../_sector/dex/bot_trades/solana/_schema.yml | 22 +++ .../platforms/nova_solana_bot_trades.sql | 152 ++++++++++++++++++ .../seeds/nova/nova_solana_trades_seed.csv | 21 +++ 3 files changed, 195 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/nova_solana_bot_trades.sql create mode 100644 dbt_subprojects/solana/seeds/nova/nova_solana_trades_seed.csv diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml index 413d474f8a0..fb74d1ec172 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml @@ -734,6 +734,28 @@ models: seed_file: ref('prophetbots_solana_trades_seed') blockchain: solana + - name: nova_solana_bot_trades + meta: + blockchain: solana + sector: dex + project: nova + contributors: whale_hunter + config: + tags: ["solana", "dex", "nova", "trades"] + description: > + Nova trades on Solana + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_id + - tx_index + - outer_instruction_index + - inner_instruction_index + - check_bot_trades_seed: + seed_file: ref('nova_solana_trades_seed') + blockchain: solana + - name: sanji_solana_bot_trades meta: blockchain: solana diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/nova_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/nova_solana_bot_trades.sql new file mode 100644 index 00000000000..2508b0dfedc --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/nova_solana_bot_trades.sql @@ -0,0 +1,152 @@ +{{ config( + alias = 'bot_trades', + schema = 'nova_solana', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] + ) +}} + +{% set project_start_date = '2024-11-30' %} +{% set fee_receiver_1 = 'noVaE91mUL5jTb8e9Vf6dqJdNPzJpEQ3uAdnQ8h4nVz' %} +{% set wsol_token = 'So11111111111111111111111111111111111111112' %} + +WITH + allFeePayments AS ( + SELECT + tx_id, + 'SOL' AS feeTokenType, + balance_change / 1e9 AS fee_token_amount, + '{{wsol_token}}' AS fee_token_mint_address + FROM + {{ source('solana','account_activity') }} + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND tx_success + AND balance_change > 0 + AND ( + address = '{{fee_receiver_1}}' + ) + ), + botTrades AS ( + SELECT + trades.block_time, + CAST(date_trunc('day', trades.block_time) AS date) AS block_date, + CAST(date_trunc('month', trades.block_time) AS date) AS block_month, + 'solana' AS blockchain, + amount_usd, + IF( + token_sold_mint_address = '{{wsol_token}}', + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_mint_address AS token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_mint_address AS token_sold_address, + fee_token_amount * price AS fee_usd, + fee_token_amount, + IF(feeTokenType = 'SOL', 'SOL', symbol) AS fee_token_symbol, + fee_token_mint_address AS fee_token_address, + project, + version, + token_pair, + project_program_id AS project_contract_address, + trader_id AS user, + trades.tx_id, + tx_index, + outer_instruction_index, + inner_instruction_index + FROM + {{ ref('dex_solana_trades') }} AS trades + JOIN allFeePayments AS feePayments ON trades.tx_id = feePayments.tx_id + LEFT JOIN {{ source('prices', 'usd') }} AS feeTokenPrices ON ( + feeTokenPrices.blockchain = 'solana' + AND fee_token_mint_address = toBase58 (feeTokenPrices.contract_address) + AND date_trunc('minute', block_time) = minute + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + JOIN {{ source('solana','transactions') }} AS transactions ON ( + trades.tx_id = id + {% if is_incremental() %} + AND {{ incremental_predicate('transactions.block_time') }} + {% else %} + AND transactions.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + WHERE + trades.trader_id != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestInnerInstructionIndexForEachTrade AS ( + SELECT + tx_id, + outer_instruction_index, + MAX(inner_instruction_index) AS highestInnerInstructionIndex + FROM + botTrades + GROUP BY + tx_id, + outer_instruction_index + ) +SELECT + block_time, + block_date, + block_month, + 'Nova' as bot, + blockchain, + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + project, + version, + token_pair, + project_contract_address, + user, + botTrades.tx_id, + tx_index, + botTrades.outer_instruction_index, + COALESCE(inner_instruction_index, 0) AS inner_instruction_index, + IF( + inner_instruction_index = highestInnerInstructionIndex, + true, + false + ) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestInnerInstructionIndexForEachTrade ON ( + botTrades.tx_id = highestInnerInstructionIndexForEachTrade.tx_id + AND botTrades.outer_instruction_index = highestInnerInstructionIndexForEachTrade.outer_instruction_index + ) +ORDER BY + block_time DESC, + tx_index DESC, + outer_instruction_index DESC, + inner_instruction_index DESC diff --git a/dbt_subprojects/solana/seeds/nova/nova_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/nova/nova_solana_trades_seed.csv new file mode 100644 index 00000000000..e71fe551768 --- /dev/null +++ b/dbt_subprojects/solana/seeds/nova/nova_solana_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction +2024-12-06 10:52:09.000 UTC,2024-12-06,2024-12-01,Nova,solana,2.336895,Buy,29376.95995,Queef,6sBcBA44i4g1m6UBMfkyn4rQKv1wag7UqB5yng7Cpump,0.0099,SOL,So11111111111111111111111111111111111111112,0.023605,0.0001,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Queef,CCCSXX9vERWwdpT7tGKWjUiYJjzhYd4wJqMeFo2j1ZB5,6UmQuA1koenTdJ6WRtraYneA7WhJC9YopVY8deUhfhR4,5tZLEggJ183v2X1fvJm7VhrAzJ7LBMVEKR3cT9YFoHyFFzuamyMgwsWTQ1M2FoKgPDdkavNoXVqooNVhaQx2542N,832,6,1,true +2024-12-04 19:50:20.000 UTC,2024-12-04,2024-12-01,Nova,solana,126.45952852578002,Sell,0.544005543,SOL,So11111111111111111111111111111111111111112,2290980.376444,Int,3rq9pxbn9b3pBmQZEwyDieh5QgwF77wVSZprwYhqpump,1.25194912884,0.005385654,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Int,BYDqrvBUWN9bEHHS5gRYpvxmht6GssAa2nFRA7cTcuCu,3daw5AUQXuQE8BwWevn9CSE5uKby28yu2oHJasB3dQLr,DxaqUyVNMhGXnb1SPrhb59F2bDKhuiZeJZseegjA8jcoU2eEKgZS3ZdWeoZjqsXAWsLar2WyFuDvVDfUSm4vnm4,1626,4,3,true +2024-12-06 12:54:28.000 UTC,2024-12-06,2024-12-01,Nova,solana,11.511921556419999,Buy,364304.938888,entry,3uZPvXaHH3ky2Vv9w1tkwEDtKYWqDCJ12WuE7FEppump,0.049537078,SOL,So11111111111111111111111111111111111111112,0.11619499999999999,0.0005,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-entry,DzEyLM8zFnqq3XNSVbyNxHNYAWpm8FmP3VnAJ51Nj8ye,4EGs99ECwUkwE4nxxapRyC4hdjX8aP9CdWwoJTV4KUX1,5r4wH4jMEofRzP9Sai7YoBATFhVxHhixoEfQUfoPhBninZbyMaNMj6jVUujrrG9NT2e2i62VzAa5vXKTz5q7WbNT,828,4,5,true +2024-12-11 08:26:32.000 UTC,2024-12-11,2024-12-01,Nova,solana,240.96468838129,Sell,1.091671673,SOL,So11111111111111111111111111111111111111112,17017539.00197,Fuji,2xV8eMGgZtLuiGE8gkpQzHLwoJt1PEZ1y5k7hU15pump,2.38555029077,0.010807549,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Fuji,AoJzpdNa5msoNtAzPjBvDXGBjDLnMT7W3uL6ziUztrvB,D7RRsBqadacKN2iHwRPhAwmxkAegekmU6e5EJTXksEPH,2Qei2PVABjF9kwLkKMWqqPviCXFn5WxaQQXKdPkwuuQXM6bSbtdzuQ5zEqEjGVxu9F1Z1yQx5b9U2SGnheeybsyZ,1614,4,3,true +2024-12-11 09:06:42.000 UTC,2024-12-11,2024-12-01,Nova,solana,3.4589988938399996,Sell,0.015617658,SOL,So11111111111111111111111111111111111111112,12162.93127,o1,2wPCNPVRat4ZVFLK2cWFf7FxNKQJP1SUC9rv3FRXpump,0.03458986048,0.000156176,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-o1,9ji9mLEJXf8BYVs4fMFoTLHhxpyamGZYJRUETPczx1Dh,EkN1h43qTCbGhmCmL9qM6NCsXLXyinFjuuVQ1n6bmPsT,3c5GnTmKH5oWh7wdZd3j6mpKMAe213SvMstxH872pCgYm4Z7utiFjrpzBRMrdpbA42yGUpTw3AZLZY2LWTSb6yRt,567,4,1,true +2024-12-06 03:34:54.000 UTC,2024-12-06,2024-12-01,Nova,solana,239.75219680254,Buy,2766765.95187,MPIGGY,DEj74N1Zfs2qWJ4QdsUj2dBUGXTJvgSM2rGnvPnZJMzQ,0.999217291,SOL,So11111111111111111111111111111111111111112,2.3994,0.01,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-MPIGGY,,EF3Xy3e9zEhsucRjYixgBkPhohdJqwQ5PnbqskoRS5tH,4aR72MtDtnwGi99va776UEM4d4Y1ERLq4XsURXjTXM2PSEfLdmZN3jDkH7vwnGRMgyMmWUixwtn19mf1rTv8HMhS,63,4,5,true +2024-12-08 03:16:53.000 UTC,2024-12-08,2024-12-01,Nova,solana,238.54049999999998,Buy,177792.733025,BRAH,EjNADFFKpKJHqEPp1GLzx2ECjFL2d65J3pcGhbP4pump,0.99,SOL,So11111111111111111111111111111111111111112,2.4095,0.01,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-BRAH,4xSk39qGUvkf3H7JusDcVRfwV8enoGnswS8a8G91e6bT,4BxjftWJs2pWeEjMNiJeiB2wXKSapnxfZMWnDbj9ZmdX,5mkESD7H2Z1Djg1YnMJFBD2DH7eFRhGtWNsZsxo1neNFhDnVn3DpEMf4JmaRs9TYonZLZQRpDMdFFQr7g9EjDyP8,171,7,1,true +2024-12-09 02:52:56.000 UTC,2024-12-09,2024-12-01,Nova,solana,2.3050340978399997,Sell,0.009967284,SOL,So11111111111111111111111111111111111111112,3882.064707008,SDOGE,2ARPkfifeN9cQyPAmEcNie5KUHApMykCQHb7R1ib5txe,0.02305014672,0.000099672,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-SDOGE,Hj1bx6VEQU7bRkaZLJpoNWgpofg3nLExKrH53e8zYnqd,BcSgJcT2GdkxQcibP8uvj8gGUj8yQREp57AXZvZXDB43,F5dwis6oXVHFWCGVjznZsoMDjhSbbnjSUMPopdjx1G2bucZ5VDAmDZ93BKTCtoyLFhTWgXfRY5w3Af2YnwECiAj,602,4,1,true +2024-12-07 14:06:49.000 UTC,2024-12-07,2024-12-01,Nova,solana,124.16807477799999,Sell,0.5194013,SOL,So11111111111111111111111111111111111111112,87590.427802,SOLDIER,GGUxJxZqUDfNbnzQqdCnrEzUX7LuCA9usdAkkSfApump,1.24168074778,0.005194013,SOL,So11111111111111111111111111111111111111112,raydium,4,SOLDIER-SOL,3wReKYqb76AqStpVTZHGefrqAMH1yTv8nk4PqZyFwyeW,EWhcbXDKx2gTEXxiYuEnJ7YtfRgna4XMwdW7ELzEiQNs,21NuyTNeNhXjVPNFkEN7RVFDjFUg2uESqzVxQeS9BHGoq1dQuTFfxEvDrXEDifswMxwt5FY5SPNKBpg4p8om6DX8,801,4,1,true +2024-12-11 17:09:29.000 UTC,2024-12-11,2024-12-01,Nova,solana,166.62935509779,Sell,0.724822111,SOL,So11111111111111111111111111111111111111112,163580.068884,FKT,EHziSfC9bfpCGWpiYHSkYhBQTXsKpBJTsn5z5EyKpump,1.66629352569,0.007248221,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FKT,55Vyv9QAKxSVX4nmufZQCKXk5WA1VnEgGcvtyav8R5ey,CtZtEjTVVbPnoHwfyxMSfei9uTLe7ts7yVKKKDkejeno,GCTUUUNMXRJ7xEU2rxFciEQGAadqUS8owS4pH6SD84KmESPFxEDBiTPurBiqsstCEpuKP6utxrnvRFNWovEH17x,1214,4,1,true +2024-12-11 04:15:16.000 UTC,2024-12-11,2024-12-01,Nova,solana,65.15147218416,Sell,0.297876153,SOL,So11111111111111111111111111111111111111112,653440.556033,BERNZY,DvgkpHBUfH5UE9ed4A7PfaBdE2yKpQ9pscyFrQwEpump,0.65151460592,0.002978761,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-BERNZY,3NBi4gcQZBj4mvG3GVhmoPaY1i4Pg4MvsgdR7g5AKNnV,6Eo8a9FxjpFNL8R6HZ2Rp6v4ZUustoLaeVehahjY817j,2Bwom76L4yFpEuEWAfdaG8THkUhQQck3b4i7arHAHFTDHhck2LEgZnPoSTHN9XPQs94oDPpFiFs1jgYr3ub8p4MH,1197,4,1,true +2024-12-11 06:57:55.000 UTC,2024-12-11,2024-12-01,Nova,solana,186.61399539676,Sell,0.847551982,SOL,So11111111111111111111111111111111111111112,161919.644767,BrettAI,CHLiroKnCmzoAJ37236F2Y5nGmpEkC1cQoa8L1AGpump,1.8661397734200003,0.008475519,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-BrettAI,4YorehZjDUSVcXfs222Y1DrbBVvpRMq17ZFNpgXd45Bb,8coN8Dq9R71tFoKjXVo1hjnH1gFRkRgvrjzcg4vkwx3E,54H2coEwVA5VWEhxa1o6DXY7ccZV9J4G1kxtLKLZ5wLnne9MpbPBZWoT8KTozUX1pPZATD8VnqDwdkEajs3SxTih,241,4,1,true +2024-12-11 13:39:47.000 UTC,2024-12-11,2024-12-01,Nova,solana,67.34178,Buy,276547.749475,KOVU,Acud16jqA9j1Hc83JrHYeiN2BpK8LYYQ1h6FGnYrpump,0.297,SOL,So11111111111111111111111111111111111111112,0.68022,0.003,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-KOVU,EFp41LWtiEzaVFiYhMema8ETHwJfx3fvdMJ2EAbUq37E,92v3iTUKvr4NZ9MSeQitRdu7BkifeCMXVJiuggtkE8uu,4FknfwQaCiHskhDbUTzramrFvSZjgTHR8RfYMeYg6F9JkDN5DRsxygpJpzQMbKHHWj9YEoKXwTiCgXnDsQ6vcguY,1283,7,1,true +2024-12-11 17:54:24.000 UTC,2024-12-11,2024-12-01,Nova,solana,40.909980354720005,Sell,0.178771108,SOL,So11111111111111111111111111111111111111112,1664280.240719,LilysBed,HKrM8f2DCGqVa2mgPwJb5oAevbqmursDEMnA4hszpump,0.40500858372,0.001769833,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-LilysBed,GMBSFRKSzvg61egswdhLPCS2GVcDPkregSSzTLzBRMpk,GnEuZhJp51vf23kwYNktiHmTjbHVNNGYNorxwi9uY5Q2,4v78BzGfCW7BPE5aSM1ir6B3rYBhiPWkFbvBj3g2ehVWN24Uqfa2DhnbZqFgd1KVE8jMhpX9gndAseHTLjVy1BRt,655,4,3,true +2024-12-12 05:09:56.000 UTC,2024-12-12,2024-12-01,Nova,solana,45.46674,Buy,318421.086978,Frog,8si8aCqGyzW2faCEHMbwKxLGMzbPQdjccSWhrYUVpump,0.198,SOL,So11111111111111111111111111111111111111112,0.45926,0.002,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Frog,21Bnj82WEr4ZQzmSHmARYZeU6Mgm3ipFTVU7JhD8XbDz,DbczwqpdHtALaWHpDWCYRzuTaowbPdRX6vgg6oBMJLFe,4JwzNgbWwDmXV7F717t4xZ7MGyoEXFhtW1HYgWKi28oqmjbHLTDvfxWgAYNJAsUkJbBBr8Ps5qe2rs5iK23ZFBNs,1755,7,1,true +2024-12-08 22:05:34.000 UTC,2024-12-08,2024-12-01,Nova,solana,117.54818121,Buy,4480224.629267,FART,2gcS7doQQb3jisCaPeAtcA6xFEXkY9s12j1157xbpump,0.49914302,SOL,So11111111111111111111111111111111111111112,1.1775,0.005,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-FART,Esc6woR2gGtBzeswbWWuZasGF2RNb8yESG2jFEFdBaPv,2vz8yVLLS6QFboK9xtpKv24tu7GUo7SHgXZDiBzSPby8,3f9QmZUeuHToi6u1gmudS92DvmjJk1LMY1oXJvuuxSHsNA9t3cUbqNvh2uvWP8ZSk7ER9QBzY5NZ1TiyVmDg31Ws,1088,4,5,true +2024-12-09 00:22:55.000 UTC,2024-12-09,2024-12-01,Nova,solana,11.660715,Buy,104953.750318,Aaliyah,BkAUZGUSfUtxPgRJJ2JDDk1XdM72zEsf4ozsmB1Spump,0.0495,SOL,So11111111111111111111111111111111111111112,0.117785,0.0005,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Aaliyah,CzW4ZoCyA5oDkxL8qhH9n9rKfaPZrVjJp6yuDX6LdHkC,DbczwqpdHtALaWHpDWCYRzuTaowbPdRX6vgg6oBMJLFe,5UxvyhvrKKezVbpVbQFLN2eeAB4uqrBGzaUbWLzcgdfZzpxAchGegviV1159VWyBPVTaVpCKbCVRr3zHXG6gHvkB,1853,7,1,true +2024-12-05 23:26:17.000 UTC,2024-12-05,2024-12-01,Nova,solana,237.2832,Buy,413016.040553,TIME,8oLsaCFngYG6vaRDCbFZTqWuBiz3swAL3RH4unXZpump,0.99,SOL,So11111111111111111111111111111111111111112,2.3968000000000003,0.01,SOL,So11111111111111111111111111111111111111112,raydium,4,TIME-SOL,G5pWpEtvAQCQAxEnyHDWrg8W1JFSp5hwPqYjtCAyrgdM,5URGB13Skm7oKyJgCgMpPU755veJajtHHuovQKMHrrrY,5T8GbHUH7C8JxGqCvM4miEzPYfT2SGhcC9iMsuKWxso4BwZZxLWxLGikw99ckFhKrsz9nwwsp7Csmpmdj9ijuGVv,962,7,1,true +2024-12-04 03:25:55.000 UTC,2024-12-04,2024-12-01,Nova,solana,118.7494441513,Buy,2884988.076315,Jarritos,chVWsF9er1J2umikgjrCxcMojdVn3yeszEuTjiEpump,0.498319111,SOL,So11111111111111111111111111111111111111112,1.1915,0.005,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Jarritos,,DcpvApMJBJY7UnxxNpiFo8s6V97jBg1usvPcAcroAKQR,3UG11MTv2UabeSPkn9abzj6QM4RvKeyHqQLKBDLQ1MNJGBHjzyodBbpHYz6CDNGTQMNEBJwh3urx1ztFU6Ki68SH,2241,4,5,true +2024-12-11 23:26:18.000 UTC,2024-12-11,2024-12-01,Nova,solana,131.57564682496,Sell,0.578024192,SOL,So11111111111111111111111111111111111111112,14727345.799435,XENA,3Yxasmk23oCReSebfhkjMHvEQyVPnSr4fvBMVMKgpump,1.30259878957,0.005722439,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,XENA-SOL,6oQreXqmsB6twE8f9phvMoDdequxjVigF2sHAjMbSXmM,3ENRG3FZp3nwcGDv7hMJ9VPTEAo1qVEdPgrkL9NoknNP,31PX7xburMDvWiieFyUhnQ7cf9LTLxk5MeKrudhVii3jVD9NQTYxZzbciiA6gWHZLhwNjcHP82Q39Pfh1opeR3bZ,1314,4,3,true From d2be350cb2f37c79f9466c70098da8d78217013d Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Mon, 16 Dec 2024 22:24:49 +0100 Subject: [PATCH 58/59] [CoW Protocol] Base changes dependent on #7295 (#7319) * [CoW Protocol] Base changes dependent on #7295 * add extra join condition for performance * select evt_block_number on gnosis view * more block numbers * group by --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../cow_protocol_arbitrum_batches.sql | 1 + .../base/cow_protocol_base_batches.sql | 99 +++++++++++++++++++ .../base/cow_protocol_base_schema.yml | 25 +++++ .../base/cow_protocol_base_solvers.sql | 64 ++++++++++++ .../cow_protocol/cow_protocol_batches.sql | 22 +++++ .../cow_protocol_ethereum_batches.sql | 1 + .../gnosis/cow_protocol_gnosis_batches.sql | 5 +- 7 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_batches.sql create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_schema.yml create mode 100644 dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_solvers.sql diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql index 46c51e364c8..5b69f8d0065 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/arbitrum/cow_protocol_arbitrum_batches.sql @@ -89,6 +89,7 @@ combined_batch_info as ( on b.evt_tx_hash = t.tx_hash inner join {{ source('arbitrum', 'transactions') }} tx on evt_tx_hash = hash + and evt_block_number = block_number {% if is_incremental() %} AND {{ incremental_predicate('block_time') }} {% endif %} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_batches.sql new file mode 100644 index 00000000000..89c3a314de9 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_batches.sql @@ -0,0 +1,99 @@ +{{ config( + schema = 'cow_protocol_base', + alias = 'batches', + materialized='incremental', + partition_by = ['block_date'], + unique_key = ['tx_hash'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + post_hook='{{ expose_spells(blockchains = \'["base"]\', + spell_type = "project", + spell_name = "cow_protocol", + contributors = \'["felix"]\') }}' + ) +}} + +WITH +batch_counts as ( + select try_cast(date_trunc('day', s.evt_block_time) as date) as block_date, + s.evt_block_number, + s.evt_block_time, + s.evt_tx_hash, + solver, + name, + sum( + case + when selector != 0x2e1a7d4d -- unwrap + and selector != 0x095ea7b3 -- approval + then 1 + else 0 + end) as dex_swaps, + sum(case when selector = 0x2e1a7d4d then 1 else 0 end) as unwraps, + sum(case when selector = 0x095ea7b3 then 1 else 0 end) as token_approvals + from {{ source('gnosis_protocol_v2_base', 'GPv2Settlement_evt_Settlement') }} s + left outer join {{ source('gnosis_protocol_v2_base', 'GPv2Settlement_evt_Interaction') }} i + on i.evt_tx_hash = s.evt_tx_hash + {% if is_incremental() %} + AND {{ incremental_predicate('i.evt_block_time') }} + {% endif %} + join {{ ref('cow_protocol_base_solvers') }} + on solver = address + {% if is_incremental() %} + WHERE {{ incremental_predicate('s.evt_block_time') }} + {% endif %} + group by s.evt_block_number, s.evt_block_time, s.evt_tx_hash, solver, name +), + +batch_values as ( + select + tx_hash, + count(*) as num_trades, + sum(usd_value) as batch_value, + sum(fee_usd) as fee_value, + price as eth_price + from {{ source('cow_protocol_base', 'trades') }} + left outer join {{ source('prices', 'usd') }} as p + on p.contract_address = 0x4200000000000000000000000000000000000006 + {% if is_incremental() %} + and {{ incremental_predicate('minute') }} + {% endif %} + and p.minute = date_trunc('minute', block_time) + and blockchain = 'base' + {% if is_incremental() %} + WHERE {{ incremental_predicate('block_time') }} + {% endif %} + group by tx_hash, price +), + +combined_batch_info as ( + select + b.block_date, + evt_block_number as block_number, + evt_block_time as block_time, + num_trades, + dex_swaps, + batch_value, + solver as solver_address, + evt_tx_hash as tx_hash, + gas_price, + gas_used, + ((gas_price / pow(10, 9)) * gas_used * eth_price) / pow(10, 9) as tx_cost_usd, + fee_value, + 2 * bytearray_length(data) / 1024 as call_data_size, + unwraps, + token_approvals + from batch_counts b + join batch_values t + on b.evt_tx_hash = t.tx_hash + inner join {{ source('base', 'transactions') }} tx + on evt_tx_hash = hash + and evt_block_number = block_number + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + where num_trades > 0 --! Exclude Withdraw Batches +) + +select * from combined_batch_info diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_schema.yml b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_schema.yml new file mode 100644 index 00000000000..ecc9d21b4f1 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_schema.yml @@ -0,0 +1,25 @@ +version: 2 + +models: + - name: cow_protocol_base_solvers + meta: + blockchain: base + project: cow_protocol + contributors: felix + config: + tags: ["base", "cow_protocol", "solver"] + description: > + CoW Protocol solvers list on Base Chain + + - name: cow_protocol_base_batches + meta: + blockchain: base + project: cow_protocol + contributors: felix + config: + tags: ["base", "cow_protocol", "trades", "dex", "aggregator", "auction"] + description: > + CoW Protocol enriched batches table on Base Chain + data_tests: + - unique: + column_name: tx_hash diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_solvers.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_solvers.sql new file mode 100644 index 00000000000..70375160bd4 --- /dev/null +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/base/cow_protocol_base_solvers.sql @@ -0,0 +1,64 @@ +{{ config( + schema = 'cow_protocol_base', + alias='solvers', + post_hook='{{ expose_spells(blockchains = \'["base"]\', + spell_type = "project", + spell_name = "cow_protocol", + contributors = \'["felix"]\') }}' +)}} + +WITH +-- Aggregate the solver added and removed events into a single table +-- with true/false for adds/removes respectively +solver_activation_events as ( + select solver, evt_block_number, evt_index, True as activated + from {{ source('gnosis_protocol_v2_base', 'GPv2AllowListAuthentication_evt_SolverAdded') }} + union + select solver, evt_block_number, evt_index, False as activated + from {{ source('gnosis_protocol_v2_base', 'GPv2AllowListAuthentication_evt_SolverRemoved') }} +), +-- Sorting by (evt_block_number, evt_index) allows us to pick the most recent activation status of each unique solver +ranked_solver_events as ( + select + rank() over (partition by solver order by evt_block_number desc, evt_index desc) as rk, + solver, + evt_block_number, + evt_index, + activated as active + from solver_activation_events +), +registered_solvers as ( + select solver, active + from ranked_solver_events + where rk = 1 +), +-- Manually inserting environment and name for each "known" solver +known_solver_metadata (address, environment, name) as ( + select * + from ( + VALUES + (0x8d98057b8c3d6c7cB02f1C1BE7E37D416F2D3e96, 'barn', 'Baseline'), + (0x351DfD19DfA0e3BfD3E7D9B22658C09e66Fd14AD, 'barn', 'Seasolver'), + (0xcE55eD17ddD7ED6E7eC396eaC772959A6D7252EA, 'barn', 'Naive'), + (0x8982b03D56de1F6f173d04A940B20A69F6A59239, 'barn', 'Gnosis_1inch'), + (0x5951400dE8fA58DA26Ab9402D2603ec0bD788273, 'barn', 'Gnosis_ParaSwap'), + (0x147d05987f3008A6C9Ec3E93A4ead430907ac3E1, 'barn', 'Gnosis_0x'), + (0x9451D27C993f7a61096BFC33e0241644a7566F66, 'barn', 'Gnosis_BalancerSOR'), + (0x0AC9287C83C2386A6a0bb27F847Ce59a0034183C, 'barn', 'Laita'), + (0x69d7F96dFD091652f317D0734A5F2B492ACcbE07, 'prod', 'Baseline'), + (0x4cb862E4821fea2dabBD1f0A69c17d52da2A58f6, 'prod', 'Seasolver'), + (0xF401ceF222F1CA2fE84a8C7BFC75A636A4542A74, 'prod', 'Naive'), + (0x8F7f754300B1ccfa37eA25fD48FB059af0F19e12, 'prod', 'Gnosis_1inch'), + (0xe321609c56aD89711EfB69c248ebe94922902F81, 'prod', 'Gnosis_ParaSwap'), + (0xbBcCE072fb1Bd2C096667E257322f47693D3dc96, 'prod', 'Gnosis_0x'), + (0x983aC485620E265730e367B2C7BCBf6Eb9d62A21, 'prod', 'Gnosis_BalancerSOR'), + (0x1A422923290fd16C2ED00ED16B4203cF4bb35d82, 'prod', 'Laita') + ) as _ +) +-- Combining the metadata with current activation status for final table +select solver as address, + case when environment is not null then environment else 'new' end as environment, + case when name is not null then name else 'Uncatalogued' end as name, + active +from registered_solvers +left outer join known_solver_metadata on solver = address diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/cow_protocol_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/cow_protocol_batches.sql index 5a13b752fa1..a45d21db499 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/cow_protocol_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/cow_protocol_batches.sql @@ -73,4 +73,26 @@ FROM unwraps, token_approvals FROM {{ ref('cow_protocol_arbitrum_batches') }} + + UNION ALL + + SELECT + 'base' AS blockchain, + 'cow_protocol' AS project, + '1' AS version, + block_date, + block_time, + num_trades, + dex_swaps, + batch_value, + solver_address, + tx_hash, + gas_price, + gas_used, + tx_cost_usd, + fee_value, + call_data_size, + unwraps, + token_approvals + FROM {{ ref('cow_protocol_base_batches') }} ) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql index 27849f7bd64..5a37703d2ef 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_batches.sql @@ -90,6 +90,7 @@ combined_batch_info as ( on b.evt_tx_hash = t.tx_hash inner join {{ source('ethereum', 'transactions') }} tx on evt_tx_hash = hash + and evt_block_number = block_number {% if is_incremental() %} AND {{ incremental_predicate('block_time') }} {% endif %} diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql index dd63b3cafad..354ef3d9446 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/gnosis/cow_protocol_gnosis_batches.sql @@ -19,6 +19,7 @@ WITH -- Find the PoC Query here: https://dune.com/queries/1722419 batch_counts as ( select try_cast(date_trunc('day', s.evt_block_time) as date) as block_date, + s.evt_block_number, s.evt_block_time, s.evt_tx_hash, solver, @@ -43,7 +44,7 @@ batch_counts as ( {% if is_incremental() %} WHERE {{ incremental_predicate('s.evt_block_time') }} {% endif %} - group by s.evt_tx_hash, solver, s.evt_block_time, name + group by s.evt_block_number, s.evt_tx_hash, solver, s.evt_block_time, name ), batch_values as ( @@ -70,6 +71,7 @@ batch_values as ( combined_batch_info as ( select b.block_date, + evt_block_number as block_number, evt_block_time as block_time, num_trades, dex_swaps, @@ -88,6 +90,7 @@ combined_batch_info as ( on b.evt_tx_hash = t.tx_hash inner join {{ source('gnosis', 'transactions') }} tx on evt_tx_hash = hash + and evt_block_number = block_number {% if is_incremental() %} AND {{ incremental_predicate('tx.block_time') }} {% endif %} From cccc1a238a2cfba777db9c11f7664da14bc57440 Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:25:33 -0500 Subject: [PATCH 59/59] Update prices_avalanche_c_tokens.sql (#7336) * Update prices_avalanche_c_tokens.sql Added missing prices for Avalanche tokens * Update prices_avalanche_c_tokens.sql * Update prices_avalanche_c_tokens.sql TUS and CRA not active anymore * remove inactive token --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../avalanche_c/prices_avalanche_c_tokens.sql | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/tokens/models/prices/avalanche_c/prices_avalanche_c_tokens.sql b/dbt_subprojects/tokens/models/prices/avalanche_c/prices_avalanche_c_tokens.sql index b32b0651bd6..6fa223a52c3 100644 --- a/dbt_subprojects/tokens/models/prices/avalanche_c/prices_avalanche_c_tokens.sql +++ b/dbt_subprojects/tokens/models/prices/avalanche_c/prices_avalanche_c_tokens.sql @@ -92,5 +92,19 @@ FROM ('hon-heroes-of-nft','avalanche_c','HON',0xed2b42d3c9c6e97e11755bb37df29b6375ede3eb,18), ('bawls-bawls-onu','avalanche_c','BAWLS',0x2da8312e2c08b79104c6b18ba26bc7065abec704,18), ('shrap-shrapnel', 'avalanche_c', 'SHRAP', 0xd402298a793948698b9a63311404fbbee944eafd, 18), - ('bal-balancer', 'avalanche_c', 'BAL', 0xe15bcb9e0ea69e6ab9fa080c4c4a5632896298c3, 18) + ('bal-balancer', 'avalanche_c', 'BAL', 0xe15bcb9e0ea69e6ab9fa080c4c4a5632896298c3, 18), + ('ausd-agora-dollar', 'avalanche_c', 'AUSD', 0x00000000eFE302BEAA2b3e6e1b18d08D69a9012a, 6), + ('stg-stargatetoken', 'avalanche_c', 'STG', 0x2F6F07CDcf3588944Bf4C42aC74ff24bF56e7590, 18), + --('tus-treasure-under-sea', 'avalanche_c', 'TUS', 0xf693248f96fe03422fea95ac0afbbbc4a8fdd172, 18), + --('cra-crabada', 'avalanche_c', 'CRA', 0xa32608e873f9ddef944b24798db69d80bbb4d1ed, 18), + ('gmx-gmx', 'avalanche_c', 'GMX', 0x62edc0692bd897d2295872a9ffcac5425011c661, 18), + ('klo-kalao', 'avalanche_c', 'KLO', 0xb27c8941a7df8958a1778c0259f76d1f8b711c35, 18), + ('solvbtc-solv-protocol-solvbtc', 'avalanche_c', 'SolvBTC', 0xbc78d84ba0c46dfe32cf2895a19939c86b81a777, 18), + ('sb-snowbank', 'avalanche_c', 'SB', 0x7d1232b90d3f809a54eeaeebc639c62df8a8942f, 9), + ('euroc-euro-coin', 'avalanche_c', 'EURC', 0xC891EB4cbdEFf6e073e859e987815Ed1505c2ACD, 6), + ('renbtc-renbtc', 'avalanche_c', 'renBTC', 0xdbf31df14b66535af65aac99c32e9ea844e14501, 8), + ('arena-the-arena', 'avalanche_c', 'ARENA', 0xb8d7710f7d8349a506b75dd184f05777c82dad0c, 18), + ('zro-layerzero', 'avalanche_c', 'ZRO', 0x6985884c4392d348587b19cb9eaaf157f13271cd, 18), + ('beam-beam-eth', 'avalanche_c', 'BEAM', 0x62d0a8458ed7719fdaf978fe5929c6d342b0bfce, 18) + ) as temp (token_id, blockchain, symbol, contract_address, decimals)