Add multiple price feeds option

cowprotocol/accounting/price_feed/raw_price_feed_query_4252674.sql

@@ -1,56 +1,45 @@
 -- This query parses the raw price_feed table that we sync on Dune,
--- and returns a view that is based on hourly prices, similar to
--- how the Dune price feed is used for slippage accounting
+-- and returns a view that is based on minute prices, quite similar to
+-- how the Dune price.usd looks like
 --
--- Parameters:
---  {{blockchain}} - network to run the analysis on; currently it only works for mainnet
 --
 -- The columns of the result are
--- - token_address: a token
--- - hour: the hour for which a price is computed
+-- - minute: the minute for which a price is computed
+-- - blockchain: the corresponding chain
+-- - contract_address: a token
 -- - decimals: the decimals of the token
+-- - price: the price, in USD, of one unit of the token
 -- - source: the source from which this price was computed,
---   price_unit: the price, in USD, of one unit of the token
--- - price_atom: the price, in USD, of one atom of the token
 
 with imported_prices as (
     select
-        token_address,
         cast(replace(time, 'T', ' ') as timestamp) as time, --noqa: RF04
-        cast(price as double) as price_unit_eth,
+        'ethereum' as blockchain,
+        token_address as contract_address,
         decimals,
-        source
-    from dune.cowprotocol.dataset_price_feed_{{blockchain}}
+        source,
+        cast(price as double) as price_unit_eth
+    from dune.cowprotocol.dataset_price_feed_ethereum
 ),
 
 imported_prices_per_minute as (
     select -- noqa: ST06
-        token_address,
         date_trunc('minute', time) as minute, --noqa: RF04
+        blockchain,
+        contract_address,
         decimals,
         source,
         avg(price_unit_eth) as price_unit_eth
-    from imported_prices group by 1, 2, 3, 4
-),
-
-imported_prices_per_minute_with_usd_prices as (
-    select -- noqa: ST06
-        ippm.token_address,
-        ippm.minute,
-        ippm.decimals,
-        ippm.price_unit_eth,
-        ippm.price_unit_eth * p.price as price_unit,
-        ippm.price_unit_eth * p.price / pow(10, ippm.decimals) as price_atom,
-        source
-    from imported_prices_per_minute as ippm inner join prices.usd as p on ippm.minute = p.minute
-    where p.contract_address = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 and blockchain = 'ethereum'
+    from imported_prices group by 1, 2, 3, 4, 5
 )
 
 select -- noqa: ST06
-    token_address,
-    date_trunc('hour', minute) as hour, --noqa: RF04
-    decimals,
-    source,
-    avg(price_unit) as price_unit,
-    avg(price_atom) as price_atom
-from imported_prices_per_minute_with_usd_prices group by 1, 2, 3, 4
+    ippm.minute,
+    ippm.blockchain,
+    ippm.contract_address,
+    ippm.decimals,
+    ippm.source,
+    ippm.price_unit_eth * p.price as price
+from imported_prices_per_minute as ippm
+inner join prices.usd as p on ippm.minute = p.minute
+where p.contract_address = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 and p.blockchain = 'ethereum'

cowprotocol/accounting/rewards/.sqlfluff

@@ -13,3 +13,4 @@ time='2024-08-27 00:00:00'
 blockchain=ethereum
 upper_cap=0.012
 lower_cap=-0.01
+price_feed=dune_price_feed,multiple_price_feeds

cowprotocol/accounting/rewards/main_rewards_dashboard_query_2510345.sql

@@ -8,7 +8,7 @@ solver_slippage as (
     select
         solver_address as solver,
         slippage_wei * 1.0 / pow(10, 18) as slippage
-    from "query_4070065(blockchain='{{blockchain}}',start_time='{{start_time}}',end_time='{{end_time}}',slippage_table_name='slippage_per_solver')"
+    from "query_4070065(blockchain='{{blockchain}}',price_feed='dune_price_feed',start_time='{{start_time}}',end_time='{{end_time}}',slippage_table_name='slippage_per_solver')"
 ),
 
 named_results as (

cowprotocol/accounting/rewards/unusual_slippage_query_2332678.sql

@@ -2,12 +2,13 @@
 -- Parameters:
 --  {{start_time}} - the start date timestamp for the accounting period  (inclusively)
 --  {{end_time}} - the end date timestamp for the accounting period (exclusively)
--- {{min_absolute_slippage_tolerance}} -- the minimum absolute threshold above which a tx is may be flagged as high-slippage
--- {{relative_slippage_tolerance}} -- the minimum relative threshold (wrt batch value) above which a tx may be flagged as high-slippage
+--  {{min_absolute_slippage_tolerance}} -- the minimum absolute threshold above which a tx is may be flagged as high-slippage
+--  {{relative_slippage_tolerance}} -- the minimum relative threshold (wrt batch value) above which a tx may be flagged as high-slippage
 --  {{significant_slippage_value}} -- the absolute threshold above which a tx is always flagged as high-slippage
+--  {{price_feed}} -- option to user either the dune_price_feed (which has been used up till now) or the median_price_feed
 with
 results_per_tx as (
-    select * from "query_4070065(blockchain='{{blockchain}}',start_time='{{start_time}}',end_time='{{end_time}}',slippage_table_name='slippage_per_transaction')"
+    select * from "query_4070065(blockchain='{{blockchain}}',price_feed='dune_price_feed',start_time='{{start_time}}',end_time='{{end_time}}',slippage_table_name='slippage_per_transaction')"
 ),
 
 url_helper as (
@@ -24,6 +25,7 @@ select  --noqa: ST06
     concat(
         '<a href="https://dune.com/queries/4059683',
         '?blockchain={{blockchain}}',
+        '&price_feed=dune_price_feed'
         '&start_time={{start_time}}',
         '&end_time={{end_time}}',
         '&slippage_table_name=raw_slippage_breakdown',

cowprotocol/accounting/slippage/.sqlfluff

@@ -4,3 +4,4 @@ end_time='2024-08-02 12:00'
 blockchain='ethereum'
 slippage_table_name=slippage_per_solver,slippage_per_transaction
 raw_slippage_table_name=raw_slippage_breakdown,raw_slippage_per_transaction
+price_feed=dune_price_feed,median_price_feed

cowprotocol/accounting/slippage/raw_slippage_4059683.sql

@@ -6,6 +6,8 @@
 --  {{blockchain}} - network to run the analysis on
 --  {{raw_slippage_table_name}} - raw_slippage_breakdown for a detailed per token breakdown of
 --    slippage; raw_slippage_per_transaction for aggregated values per transaction
+--  {{price_feed}} -- option to user either the dune_price_feed (which has been used up till now) or the median_price_feed, which
+--    considers the median over multiple price feeds
 --
 -- The columns of raw_slippage_breakdown are
 -- - block_time: time of settlement transaction
@@ -56,7 +58,7 @@ imbalances as (
 ),
 
 prices as (
-    select * from "query_4064601(blockchain='{{blockchain}}',start_time='{{start_time}}',end_time='{{end_time}}')"
+    select * from "query_4064601(blockchain='{{blockchain}}',price_feed='{{price_feed}}',start_time='{{start_time}}',end_time='{{end_time}}')"
 ),
 
 raw_slippage_breakdown as (

cowprotocol/accounting/slippage/slippage_4070065.sql

@@ -7,6 +7,7 @@
 --  {{blockchain}} - network to run the analysis on
 --  {{slippage_table_name}} - slippage_per_transaction for aggregated values per transaction;
 --    slippage_per_solver for aggregated values per transaction
+--  {{price_feed}} -- option to user either the dune_price_feed (which has been used up till now) or the median_price_feed
 --
 -- The columns of slippage_per_transaction are
 -- - block_time: time of settlement transaction
@@ -40,7 +41,7 @@ slippage_per_transaction as (
         sum(if(slippage_type = 'raw_imbalance', slippage_usd, 0)) as imbalance_usd,
         sum(if(slippage_type = 'protocol_fee', -slippage_usd, 0)) as protocol_fee_usd,
         sum(if(slippage_type = 'network_fee', -slippage_usd, 0)) as network_fee_usd
-    from "query_4059683(blockchain='{{blockchain}}',start_time='{{start_time}}',end_time='{{end_time}}',raw_slippage_table_name='raw_slippage_breakdown')" as rs
+    from "query_4059683(blockchain='{{blockchain}}',price_feed='{{price_feed}}',start_time='{{start_time}}',end_time='{{end_time}}',raw_slippage_table_name='raw_slippage_breakdown')" as rs
     inner join cow_protocol_{{blockchain}}.batches as b
         on rs.tx_hash = b.tx_hash
     where rs.tx_hash not in (select tx_hash from excluded_batches)

cowprotocol/accounting/slippage/slippage_prices_4064601.sql

@@ -6,6 +6,7 @@
 --  {{start_time}} - the timestamp for which the analysis should start (inclusively)
 --  {{end_time}} - the timestamp for which the analysis should end (exclusively)
 --  {{blockchain}} - network to run the analysis on
+--  {{price_feed}} -- option to user either the dune_price_feed (which has been used up till now) or the median_price_feed
 --
 -- The columns of the result are
 -- - hour: hour for which a price is valid
@@ -25,24 +26,82 @@ with token_times as (
     group by 1, 2
 ),
 
--- Precise prices are prices from the Dune price feed.
-precise_prices as (
+-- Fetching all additional price feeds that are synced to Dune
+imported_price_feeds as (
+    select --noqa: ST06
+        a.source as price_source,
+        date_trunc('hour', a.minute) as hour, --noqa: RF04
+        a.contract_address as token_address,
+        a.decimals,
+        avg(a.price) as price_unit
+    from "query_4252674" as a inner join token_times as tt
+        on
+            date_trunc('hour', a.minute) = tt.hour
+            and a.contract_address = tt.token_address
+            and a.blockchain = '{{blockchain}}'
+    group by 1, 2, 3, 4
+),
+
+-- the Dune price feed; note that this is computed on Dune and is not part of the imported_price_feeds_raw table.
+dune_price_feed as (
     select -- noqa: ST06
-        date_trunc('hour', minute) as hour, --noqa: RF04
+        'dune' as price_source,
+        date_trunc('hour', a.minute) as hour, --noqa: RF04
+        a.contract_address as token_address,
+        a.decimals,
+        avg(a.price) as price_unit
+    from prices.usd as a inner join token_times as tt
+        on
+            date_trunc('hour', a.minute) = tt.hour
+            and a.contract_address = tt.token_address
+            and a.blockchain = '{{blockchain}}'
+    group by 1, 2, 3, 4
+),
+
+-- we now collect together all different price feeds that we have
+all_price_feeds as (
+    select *
+    from imported_price_feeds
+    union all
+    select *
+    from dune_price_feed
+),
+
+-- we are now ready to define a new price feed that is the median of all price feeds defined above
+-- there is an intermediate table to help with the calculation,
+-- and the code for the median is based on the No.2 section
+-- of this article: https://medium.com/learning-sql/how-to-calculate-median-the-right-way-in-postgresql-f7b84e9e2df7
+intermediate_compute_median_table as (
+    select
+        hour,
         token_address,
         decimals,
-        avg(price) as price_unit,
-        avg(price) / pow(10, decimals) as price_atom
-    from
-        prices.usd
-    inner join token_times
-        on
-            date_trunc('hour', minute) = hour
-            and contract_address = token_address
-            and blockchain = '{{blockchain}}'
+        price_unit,
+        row_number() over (partition by hour, token_address, decimals order by price_unit asc) as rn_asc,
+        count(*) over (partition by hour, token_address, decimals) as ct
+    from all_price_feeds
+),
+
+-- this is the final table generated, that uses the median of all price feeds
+-- to compute a final price.
+median_price_feed as (
+    select
+        hour,
+        token_address,
+        decimals,
+        avg(price_unit) as price_unit
+    from intermediate_compute_median_table
+    where rn_asc between ct / 2.0 and ct / 2.0 + 1
     group by 1, 2, 3
 ),
 
+-- We now define the precise_prices table, and there are 2 options to choose from,
+-- either the median_price_feed table or the dune_price_feed
+precise_prices as (
+    select *
+    from {{price_feed}}
+),
+
 -- Intrinsic prices are prices reconstructed from exchange rates from within the auction
 -- A price can be reconstructed if there was a trade with another token which did have a Dune price.
 -- If there a multiple prices reconstructed in this way, an average is taken.
@@ -52,15 +111,13 @@ intrinsic_prices as (
         hour,
         token_address,
         decimals,
-        avg(price_unit) as price_unit,
-        avg(price_atom) as price_atom
+        avg(price_unit) as price_unit
     from (
         select
             date_trunc('hour', block_time) as hour, --noqa: RF04
             buy_token_address as token_address,
             round(log(10, atoms_bought / units_bought)) as decimals,
-            usd_value / units_bought as price_unit,
-            usd_value / atoms_bought as price_atom
+            usd_value / units_bought as price_unit
         from cow_protocol_{{blockchain}}.trades
         where
             block_time >= cast('{{start_time}}' as timestamp) and block_time < cast('{{end_time}}' as timestamp)
@@ -70,8 +127,7 @@ intrinsic_prices as (
             date_trunc('hour', block_time) as hour, --noqa: RF04
             sell_token_address as token_address,
             round(log(10, atoms_sold / units_sold)) as decimals,
-            usd_value / units_sold as price_unit,
-            usd_value / atoms_sold as price_atom
+            usd_value / units_sold as price_unit
         from cow_protocol_{{blockchain}}.trades
         where
             block_time >= cast('{{start_time}}' as timestamp) and block_time < cast('{{end_time}}' as timestamp)
@@ -80,8 +136,8 @@ intrinsic_prices as (
     group by 1, 2, 3
 ),
 
--- The final price is the Dune price if it exists and the intrinsic price otherwise. If both prices
--- are not available, the price is null.
+-- The final price is the precise price if it exists and the intrinsic price otherwise.
+-- If both prices are not available, the price is null.
 prices as (
     select
         tt.hour,
@@ -95,9 +151,9 @@ prices as (
             intrinsic.price_unit
         ) as price_unit,
         coalesce(
-            precise.price_atom,
-            intrinsic.price_atom
-        ) as price_atom
+            precise.price_unit,
+            intrinsic.price_unit
+        ) / pow(10, coalesce(precise.decimals, intrinsic.decimals)) as price_atom
     from token_times as tt
     left join precise_prices as precise
         on
@@ -109,6 +165,8 @@ prices as (
             and tt.token_address = intrinsic.token_address
 ),
 
+-- We also want to have the prices of the native token of each chain
+-- so we define this intermediate table to help with that
 wrapped_native_token as (
     select
         case '{{blockchain}}'
@@ -119,7 +177,7 @@ wrapped_native_token as (
         end as native_token_address
 ),
 
--- The price of the native token is reconstructed from it chain-dependent wrapped version.
+-- The price of the native token is reconstructed from its chain-dependent wrapped version.
 native_token_prices as (
     select -- noqa: ST06
         date_trunc('hour', minute) as hour, --noqa: RF04
@@ -136,5 +194,5 @@ native_token_prices as (
 )
 
 select * from prices
-union all
+union distinct -- new price feeds might already entries for the native token
 select * from native_token_prices

cowprotocol/accounting/slippage/slippage_query_3427730.sql

@@ -6,11 +6,12 @@ select
     concat(
         '<a href="https://dune.com/queries/4070065',
         '&blockchain=ethereum',
+        '&price_feed=dune_price_feed'
         '&start_time={{start_time}}',
         '&end_time={{end_time}}',
         '&slippage_table_name=slippage_per_transaction',
         '" target="_blank">link</a>'
     ) as slippage_per_transaction
-from "query_4070065(blockchain='{{blockchain}}',start_time='{{start_time}}',end_time='{{end_time}}',slippage_table_name='slippage_per_solver')"
+from "query_4070065(blockchain='{{blockchain}}',price_feed='dune_price_feed',start_time='{{start_time}}',end_time='{{end_time}}',slippage_table_name='slippage_per_solver')"
 inner join cow_protocol_{{blockchain}}.solvers
     on solver_address = address