diff --git a/auction-server/src/api/rest.rs b/auction-server/src/api/rest.rs index 4bfdc92c..679ba96c 100644 --- a/auction-server/src/api/rest.rs +++ b/auction-server/src/api/rest.rs @@ -2,8 +2,8 @@ use { crate::{ api::RestError, auction::{ - per::MulticallStatus, simulate_bids, + MulticallStatus, }, state::{ SimulatedBid, diff --git a/beacon/README.md b/beacon/README.md new file mode 100644 index 00000000..4afa8ac0 --- /dev/null +++ b/beacon/README.md @@ -0,0 +1,9 @@ +# Beacon + +The Beacon is the off-chain service that exposes liquidation opportunities on integrated protocols to searchers. Protocols surface their liquidatable vaults/accounts along with the calldata and the token denominations and amounts involved in the transaction. Searchers can query these opportunities from the Beacon server. If they wish to act on an opportunity, they can simply construct a transaction based off the information in the opportunity. + +The LiquidationAdapter contract that is part of the Express Relay on-chain stack allows searchers to perform liquidations across different protocols without needing to deploy their own contracts or perform bespoke engineering work. The Beacon service is important in enabling this, as it disseminates the calldata that searchers need to include in the transactions they construct. + +Each protocol that integrates with Express Relay and the LiquidationAdapter workflow must provide code that handles getting liquidatable accounts; the example file for the TokenVault dummy contract is found in `/protocols`. Some common types are defined in `utils/types_liquidation_adapter.py`, and standard functions for accessing Pyth Hermes prices can be found in `utils/pyth_prices.py`. The exact interface of the methods in the protocol's file is not important, but it should have a similar `main()` interface: the same command line arguments and general behavior of sending liquidatable vaults to the Beacon server when specified. + +The party that runs the beacon can run the protocol-provided file to get and surface liquidatable accounts to the Beacon server. diff --git a/beacon/protocols/beacon_TokenVault.py b/beacon/protocols/beacon_TokenVault.py index 71bc3d88..5246fede 100644 --- a/beacon/protocols/beacon_TokenVault.py +++ b/beacon/protocols/beacon_TokenVault.py @@ -2,15 +2,23 @@ from eth_abi import encode import json from typing import TypedDict +import argparse +import logging +import asyncio +import httpx -from beacon.utils.pyth_prices import * -from beacon.utils.types_liquidation_adapter import * +from beacon.utils.pyth_prices import PriceFeedClient, PriceFeed +from beacon.utils.types_liquidation_adapter import LiquidationOpportunity TOKEN_VAULT_ADDRESS = "0x72A22FfcAfa6684d4EE449620270ac05afE963d0" -CHAIN_RPC_ENDPOINT = "http://localhost:8545" -class LiquidationAccount(TypedDict): +class ProtocolAccount(TypedDict): + """ + ProtocolAccount is a TypedDict that represents an account/vault in the protocol. + + This class contains all the relevant information about a vault/account on this protocol that is necessary for identifying whether it is eligible for liquidation and constructing a LiquidationOpportunity object. + """ account_number: int token_address_collateral: str token_address_debt: str @@ -30,15 +38,17 @@ def get_vault_abi(): return data['abi'] -""" -get_accounts() is the first method that the protocol should implement. It should take no arguments and return all the open accounts in the protocol in the form of a list of objects of type LiquidationAccount (defined above). Each LiquidationAccount object represents an account/vault in the protocol. -This function can be implemented in any way, but it should be able to return all the open accounts in the protocol. For some protocols, this may be easily doable by just querying on-chain state; however, most protocols will likely need to maintain or access an off-chain indexer to get the list of all open accounts. -""" - +async def get_accounts(rpc_url: str) -> list[ProtocolAccount]: + """ + Returns all the open accounts in the protocol in the form of a list of type ProtocolAccount. -async def get_accounts() -> list[LiquidationAccount]: + Args: + rpc_url (str): The RPC URL of the chain + Returns: + List of objects of type ProtocolAccount (defined above). Each ProtocolAccount object represents an account/vault in the protocol. + """ abi = get_vault_abi() - w3 = web3.AsyncWeb3(web3.AsyncHTTPProvider(CHAIN_RPC_ENDPOINT)) + w3 = web3.AsyncWeb3(web3.AsyncHTTPProvider(rpc_url)) token_vault = w3.eth.contract( address=TOKEN_VAULT_ADDRESS, abi=abi) @@ -64,7 +74,7 @@ async def get_accounts() -> list[LiquidationAccount]: 16) == 0: done = True else: - account: LiquidationAccount = { + account: ProtocolAccount = { "account_number": account_number, "token_address_collateral": vault_dict['tokenCollateral'], "token_id_collateral": vault_dict['tokenIDCollateral'].hex(), @@ -81,8 +91,18 @@ async def get_accounts() -> list[LiquidationAccount]: def create_liquidation_opp( - account: LiquidationAccount, prices: list[PriceFeed] -) -> LiquidationOpportunity: + account: ProtocolAccount, + prices: list[PriceFeed]) -> LiquidationOpportunity: + """ + Constructs a LiquidationOpportunity object from a ProtocolAccount object and a set of relevant Pyth PriceFeeds. + + Args: + account: A ProtocolAccount object, representing an account/vault in the protocol. + prices: A list of PriceFeed objects, representing the relevant Pyth price feeds for the tokens in the ProtocolAccount object. + Returns: + A LiquidationOpportunity object corresponding to the specified account. + """ + # [bytes.fromhex(update['vaa']) for update in prices] ## TODO: uncomment this, to add back price updates price_updates = [] function_signature = web3.Web3.solidity_keccak( @@ -104,17 +124,16 @@ def create_liquidation_opp( "contract": TOKEN_VAULT_ADDRESS, "calldata": calldata, "permission_key": permission, - "account": str(account["account_number"]), "repay_tokens": [ { "contract": account["token_address_debt"], - "amount": str(account["amount_debt"]), + "amount": hex(account["amount_debt"]).replace("0x", ""), } ], "receipt_tokens": [ { "contract": account["token_address_collateral"], - "amount": str(account["amount_collateral"]), + "amount": hex(account["amount_collateral"]).replace("0x", ""), } ] } @@ -128,22 +147,31 @@ def create_liquidation_opp( return opp -""" -get_liquidatable(accounts, prices) is the second method that the protocol should implement. It should take two arguments: account--a list of Account (defined above) objects--and prices--a dictionary of Pyth prices. -accounts should be the list of all open accounts in the protocol (i.e. the output of get_accounts()). -prices should be a dictionary of Pyth prices, where the keys are Pyth feed IDs and the values are PriceFeed objects. prices can be retrieved from the provided price retrieval functions. -This function should return a lists of liquidation opportunities. Each opportunity should be of the form LiquidationOpportunity defined above. -""" +def get_liquidatable(accounts: list[ProtocolAccount], + prices: dict[str, + PriceFeed]) -> (list[LiquidationOpportunity]): + """ + Filters list of ProtocolAccount types to return a list of LiquidationOpportunity types. + Args: + accounts: A list of ProtocolAccount objects, representing all the open accounts in the protocol. + prices: A dictionary of Pyth price feeds, where the keys are Pyth feed IDs and the values are PriceFeed objects. + Returns: + A list of LiquidationOpportunity objects, one per account that is eligible for liquidation. + """ -def get_liquidatable( - accounts: list[LiquidationAccount], prices: dict[str, PriceFeed] -) -> list[LiquidationOpportunity]: liquidatable = [] for account in accounts: - price_collateral = prices[account["token_id_collateral"]] - price_debt = prices[account["token_id_debt"]] + price_collateral = prices.get(account["token_id_collateral"]) + if price_collateral is None: + raise Exception( + f"Price for collateral token {account['token_id_collateral']} not found") + + price_debt = prices.get(account["token_id_debt"]) + if price_debt is None: + raise Exception( + f"Price for debt token {account['token_id_debt']} not found") value_collateral = ( int(price_collateral["price"]["price"]) * @@ -164,23 +192,66 @@ def get_liquidatable( async def main(): - # get all accounts - accounts = await get_accounts() - - # get prices - pyth_price_feed_ids = await get_price_feed_ids() - pyth_prices_latest = [] - i = 0 - cntr = 100 - while len(pyth_price_feed_ids[i:i + cntr]) > 0: - pyth_prices_latest += await get_pyth_prices_latest(pyth_price_feed_ids[i:i + cntr]) - i += cntr - pyth_prices_latest = dict(pyth_prices_latest) - - # get liquidatable accounts - liquidatable = get_liquidatable(accounts, pyth_prices_latest) - - print(liquidatable) + parser = argparse.ArgumentParser() + parser.add_argument("--operator-api-key", type=str, required=True, + help="Operator API key, used to authenticate the surface post request") + parser.add_argument("--rpc-url", type=str, required=True, + help="Chain RPC endpoint, used to fetch on-chain data via get_accounts") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--dry-run", action="store_false", dest="send_beacon", + help="If provided, will not send liquidation opportunities to the beacon server") + group.add_argument("--beacon-server-url", type=str, + help="Beacon server endpoint; if provided, will send liquidation opportunities to the beacon server") + + parser.add_argument("--log-file", type=str, + help="Path of log file where to save log statements; if not provided, will print to stdout") + args = parser.parse_args() + + if args.log_file: + logging.basicConfig(filename=args.log_file, level=logging.INFO) + else: + logging.basicConfig(level=logging.INFO) + + feed_ids = ["ff61491a931112ddf1bd8147cd1b641375f79f5825126d665480874634fd0ace", + "e62df6c8b4a85fe1a67db44dc12de5db330f7ac66b72dc658afedf0f4a415b43"] # TODO: should this be automated rather than hardcoded? + price_feed_client = PriceFeedClient(feed_ids) + + # TODO: sometimes the ws doesn't pull prices, understand why + ws_call = price_feed_client.ws_pyth_prices() + asyncio.create_task(ws_call) + + client = httpx.AsyncClient() + + await asyncio.sleep(2) + + while True: + accounts = await get_accounts(args.rpc_url) + + accounts_liquidatable = get_liquidatable( + accounts, price_feed_client.prices_dict) + + if args.send_beacon: + for account_liquidatable in accounts_liquidatable: + resp = await client.post( + args.beacon_server_url, + json=account_liquidatable + ) + if resp.status_code == 422: + logging.error( + "Invalid request body format, should provide a list of LiquidationOpportunity") + elif resp.status_code == 404: + logging.error( + "Provided beacon server endpoint url not found") + elif resp.status_code == 405: + logging.error( + "Provided beacon server endpoint url does not support POST requests") + else: + logging.info(f"Response, post to beacon: {resp.text}") + else: + logging.info( + f"List of liquidatable accounts:\n{accounts_liquidatable}") + + await asyncio.sleep(2) if __name__ == "__main__": asyncio.run(main()) diff --git a/beacon/protocols/beacon_template.py b/beacon/protocols/beacon_template.py deleted file mode 100644 index 8ceacc25..00000000 --- a/beacon/protocols/beacon_template.py +++ /dev/null @@ -1,92 +0,0 @@ -import web3 -from eth_abi import encode -import json -from typing import TypedDict - -from beacon.utils.pyth_prices import * -from beacon.utils.types_liquidation_adapter import * - -PROTOCOL_ADDRESS = "{INSERT}" -CHAIN_RPC_ENDPOINT = "http://localhost:8545" - -""" -The protocol should implement a class called LiquidationAccount. This will be the type of the objects in the list returned by get_accounts() and fed into get_liquidatable. -This class should contain all the relevant information about a vault/account on this protocol that is necessary for identifying whether it is eligible for liquidation and constructing a LiquidationOpportunity object. -""" - - -class LiquidationAccount(TypedDict): - # Keys of the TypedDict and their types - pass - - -""" -get_accounts() is the first method that the protocol should implement. It should take no arguments and return all the open accounts in the protocol in the form of a list of objects of type LiquidationAccount (defined above). Each LiquidationAccount object represents an account/vault in the protocol. -This function can be implemented in any way, but it should be able to return all the open accounts in the protocol. For some protocols, this may be easily doable by just querying on-chain state; however, most protocols will likely need to maintain or access an off-chain indexer to get the list of all open accounts. -""" - - -async def get_accounts() -> list[LiquidationAccount]: - # Fetch all vaults from on-chain state/indexer - # Filter to just active vaults - # Return list of LiquidationAccount - # TODO: complete - pass - - -""" -create_liquidation_opp is an optional helper function to construct a LiquidationOpportunity from a LiquidationAccount and a set of relevant Pyth PriceFeeds. -If you choose to implement this function, you can call it within get_liquidatable whenever you find a LiquidationAccount eligible for liquidation. -""" - - -def create_liquidation_opp( - account: LiquidationAccount, - prices: list[PriceFeed]) -> LiquidationOpportunity: - pass - - -""" -get_liquidatable(accounts, prices) is the second method that the protocol should implement. It should take two arguments: account--a list of LiquidationAccount (defined above) objects--and prices--a dictionary of Pyth prices. -accounts should be the list of all open accounts in the protocol (i.e. the output of get_accounts()). -prices should be a dictionary of Pyth prices, where the keys are Pyth feed IDs and the values are PriceFeed objects. prices can be retrieved from the provided price retrieval functions. -This function should return a list of type LiquidationOpportunity. -""" - - -def get_liquidatable(accounts: list[LiquidationAccount], - prices: dict[str, - PriceFeed]) -> (list[LiquidationOpportunity]): - # Iterate through accounts - # Determine if account is eligible for liquidation; if so, construct an object of type LiquidationOpportunity and add it to the list - # Return the list of type LiquidationOpportunity containing all the valid liquidation opportunities - pass - - -""" -The main loop below is a good mechanism to check if your implementations of the functions above are working properly. -""" - - -async def main(): - # get all accounts - accounts = await get_accounts() - - # get prices - pyth_price_feed_ids = await get_price_feed_ids() - pyth_prices_latest = [] - i = 0 - cntr = 100 - while len(pyth_price_feed_ids[i:i + cntr]) > 0: - pyth_prices_latest += await get_pyth_prices_latest(pyth_price_feed_ids[i:i + cntr]) - i += cntr - pyth_prices_latest = dict(pyth_prices_latest) - - # get liquidatable accounts - liquidatable = get_liquidatable( - accounts, pyth_prices_latest) - - print(liquidatable) - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/beacon/searcher/README.md b/beacon/searcher/README.md new file mode 100644 index 00000000..124ca118 --- /dev/null +++ b/beacon/searcher/README.md @@ -0,0 +1,10 @@ +# Searcher + +Searchers can integrate with Express Relay by one of two means: + +1. LiquidationAdapter Contract interaction from an EOA +2. Bespoke integration via a deployed contract + +Option 2 requires bespoke work to handle individual protocol interfaces and smart contract risk, and it is similar in nature to how many searchers currently do liquidations via their own deployed contracts--searchers can now call into their smart contracts via the Express Relay workflow. This option allows for greater customization by the searcher, but requires bespoke work per protocol that the searcher wants to integrate with. + +Meanwhile, option 1 requires much less bespoke work and does not require contract deployment by the searcher. For option 1, the searcher submits liquidation transactions to the LiquidationAdapter contract, which handles routing the liquidation logic to the protocol and also performs some basic safety checks to ensure that the searcher is paying and receiving the appropriate amounts. The searcher can submit transactions signed by their EOA that has custody of the tokens they wish to repay with. Searchers can listen to liquidation opportunities at the Beacon server, and if they wish to submit a liquidation transaction through Express Relay, they can submit it to the auction server endpoint. `searcher_template.py` contains a template for the actions that a searcher may wish to perform, namely getting and assessing opportunities at the Beacon server and constructing and sending a liquidation. Helper functions related to constructing the signature for the LiquidationAdapter contract are in `searcher_utils.py`. A sample workflow is in `searcherA.py` (note: this example lacks any serious evaluation of opportunities, and it simply carries out a liquidation if the opportunity is available). diff --git a/beacon/searcher/searcherA.py b/beacon/searcher/searcherA.py index 8070739d..0d006b96 100644 --- a/beacon/searcher/searcherA.py +++ b/beacon/searcher/searcherA.py @@ -1,5 +1,3 @@ -import web3 -from web3.auto import w3 from eth_account import Account from eth_account.signers.local import LocalAccount from eth_abi import encode @@ -7,25 +5,33 @@ import asyncio from beacon.utils.types_liquidation_adapter import * -from beacon.utils.endpoints import * -from beacon.searcher.searcher_utils import * - -TOKEN_VAULT_ADDRESS = "0x72A22FfcAfa6684d4EE449620270ac05afE963d0" +from beacon.utils.endpoints import BEACON_SERVER_ENDPOINT_GETOPPS, AUCTION_SERVER_ENDPOINT +from beacon.searcher.searcher_utils import UserLiquidationParams, construct_signature_liquidator BID = 10 VALID_UNTIL = 1_000_000_000_000 -def create_liquidation_intent( - opp: LiquidationOpportunity, - sk_liquidator: str, - valid_until: int, - bid: int -) -> LiquidationAdapterIntent: +def assess_liquidation_opportunity( + opp: LiquidationOpportunity +) -> UserLiquidationParams | None: + user_liquidation_params: UserLiquidationParams = { + "bid": BID, + "valid_until": VALID_UNTIL + } + return user_liquidation_params + + +def create_liquidation_transaction( + opp: LiquidationOpportunity, + sk_liquidator: str, + valid_until: int, + bid: int +) -> LiquidationAdapterTransaction: repay_tokens = [(opp['repay_tokens'][0]['contract'], - int(opp['repay_tokens'][0]['amount']))] + int(opp['repay_tokens'][0]['amount'], 16))] receipt_tokens = [(opp['receipt_tokens'][0]['contract'], - int(opp['receipt_tokens'][0]['amount']))] + int(opp['receipt_tokens'][0]['amount'], 16))] account: LocalAccount = Account.from_key(sk_liquidator) liquidator = account.address @@ -48,7 +54,7 @@ def create_liquidation_intent( encode([LIQUIDATION_ADAPTER_CALLDATA_TYPES], [ tuple(liquidation_adapter_calldata.values())]).hex() - intent: LiquidationAdapterIntent = { + tx: LiquidationAdapterTransaction = { "bid": str(bid), "calldata": calldata, "chain_id": opp["chain_id"], @@ -56,30 +62,39 @@ def create_liquidation_intent( "permission_key": opp['permission_key'] } - return intent + return tx async def main(): - CLIENT = httpx.AsyncClient() - - params = {"chain_id": "development", "contract": TOKEN_VAULT_ADDRESS} + client = httpx.AsyncClient() - liquidatable = (await CLIENT.get(BEACON_SERVER_ENDPOINT_GETOPPS, params=params)).json() + params = {"chain_id": "development"} # this is hardcoded to the searcher A SK sk_liquidator = "0x5b1efe5da513271c0d30cde7a2ad1d29456d68abd592efdaa7d2302e913b783f" - intent = create_liquidation_intent( - liquidatable[0], sk_liquidator, VALID_UNTIL, BID) - resp = await CLIENT.post( - AUCTION_SERVER_ENDPOINT, - json=intent - ) + while True: + accounts_liquidatable = (await client.get(BEACON_SERVER_ENDPOINT_GETOPPS, params=params)).json() + + for liquidation_opp in accounts_liquidatable: + user_liquidation_params = assess_liquidation_opportunity( + liquidation_opp) + + if user_liquidation_params is not None: + bid, valid_until = user_liquidation_params["bid"], user_liquidation_params["valid_until"] + + tx = create_liquidation_transaction( + liquidation_opp, sk_liquidator, valid_until, bid) + + resp = await client.post( + AUCTION_SERVER_ENDPOINT, + json=tx + ) - print(resp.text) + print(resp.text) - import pdb - pdb.set_trace() + import pdb + pdb.set_trace() if __name__ == "__main__": asyncio.run(main()) diff --git a/beacon/searcher/searcher_template.py b/beacon/searcher/searcher_template.py index e69de29b..b6d19e07 100644 --- a/beacon/searcher/searcher_template.py +++ b/beacon/searcher/searcher_template.py @@ -0,0 +1,44 @@ +from beacon.utils.types_liquidation_adapter import LiquidationOpportunity, LiquidationAdapterTransaction +from beacon.searcher.searcher_utils import UserLiquidationParams + + +def assess_liquidation_opportunity( + opp: LiquidationOpportunity +) -> UserLiquidationParams | None: + """ + Assesses whether a LiquidationOpportunity is worth liquidating; if so, returns a tuple of (bid, valid_until) + + This function can handle assessing the LiquidationOpportunity to determine whether it deals with repay and receipt tokens that the searcher wishes to transact in and whether it is profitable to conduct the liquidation. + There are many ways to evaluate this, but the most common way is to check that the value of the amount the searcher will receive from the liquidation exceeds the value of the amount repaid. + Individual searchers will have their own methods to determine market impact and the profitability of conducting a liquidation. This function can be expanded to include external prices to perform this evaluation. + If the LiquidationOpportunity is deemed worthwhile, this function can return a bid amount representing the amount of native token to bid on this opportunity, and a timestamp representing the time at which the transaction will expire. + Otherwise, this function can return None. + + Args: + opp: A LiquidationOpportunity object, representing a single liquidation opportunity. + Returns: + If the LiquidationOpportunity is deemed worthwhile, this function can return a UserLiquidationParams object, representing the user's bid and the timestamp at which the user's transaction should expire. If the LiquidationOpportunity is not deemed worthwhile, this function can return None. + """ + raise NotImplementedError + + +def create_liquidation_tx( + opp: LiquidationOpportunity, + sk_liquidator: str, + valid_until: int, + bid: int +) -> LiquidationAdapterTransaction: + """ + Processes a LiquidationOpportunity into a LiquidationAdapterTransaction + + This function can handle constructing the LiquidationAdapterTransaction to submit. The calldata for the LiquidationAdapter contract should be constructed according to the LiquidationAdapterCalldata type; you can leverage the construct_signature_liquidator function to construct the signature field. + + Args: + opp: A LiquidationOpportunity object, representing a single liquidation opportunity. + sk_liquidator: A 0x-prefixed hex string representing the liquidator's private key. + valid_until: An integer representing the timestamp at which the transaction will expire. + bid: An integer representing the amount of native token to bid on this opportunity. + Returns: + A LiquidationAdapterTransaction object, representing the transaction to submit. + """ + raise NotImplementedError diff --git a/beacon/searcher/searcher_utils.py b/beacon/searcher/searcher_utils.py index b25caafc..365511a7 100644 --- a/beacon/searcher/searcher_utils.py +++ b/beacon/searcher/searcher_utils.py @@ -1,6 +1,13 @@ import web3 from web3.auto import w3 from eth_abi import encode +from typing import TypedDict +from eth_account.datastructures import SignedMessage + + +class UserLiquidationParams(TypedDict): + bid: int + valid_until: int def construct_signature_liquidator( @@ -11,7 +18,22 @@ def construct_signature_liquidator( bid: int, valid_until: int, secret_key: str -): +) -> SignedMessage: + """ + Constructs a signature for a liquidator's transaction to submit to the LiquidationAdapter contract. + + Args: + repay_tokens: A list of tuples (token address, amount) representing the tokens to repay. + receipt_tokens: A list of tuples (token address, amount) representing the tokens to receive. + address: The address of the LiquidationAdapter contract. + liq_calldata: The calldata for the liquidation method call. + bid: The amount of native token to bid on this opportunity. + valid_until: The timestamp at which the transaction will expire. + secret_key: A 0x-prefixed hex string representing the liquidator's private key. + Returns: + An web3 ECDSASignature object, representing the liquidator's signature. + """ + digest = encode( ['(address,uint256)[]', '(address,uint256)[]', 'address', 'bytes', 'uint256'], diff --git a/beacon/surface_opportunities.py b/beacon/surface_opportunities.py deleted file mode 100644 index 0aad9b0d..00000000 --- a/beacon/surface_opportunities.py +++ /dev/null @@ -1,45 +0,0 @@ -import httpx -import asyncio - -from beacon.protocols import beacon_TokenVault -from beacon.utils.pyth_prices import * -from beacon.utils.endpoints import * - -# TODO: turn on authorization in the surface post requests -OPERATOR_API_KEY = "password" -PROTOCOLS = [beacon_TokenVault] - - -async def main(): - # get prices - pyth_price_feed_ids = await get_price_feed_ids() - pyth_prices_latest = [] - i = 0 - cntr = 100 - while len(pyth_price_feed_ids[i:i + cntr]) > 0: - pyth_prices_latest += await get_pyth_prices_latest(pyth_price_feed_ids[i:i + cntr]) - i += cntr - pyth_prices_latest = dict(pyth_prices_latest) - - liquidatable = [] - - for protocol in PROTOCOLS: - accounts = await protocol.get_accounts() - - liquidatable_protocol = protocol.get_liquidatable( - accounts, pyth_prices_latest) - - liquidatable += liquidatable_protocol - - CLIENT = httpx.AsyncClient() - - for item in liquidatable: - resp = await CLIENT.post( - f"{BEACON_SERVER_ENDPOINT_SURFACE}", - json=item - ) - print(f"Response PER post: {resp.text}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/beacon/utils/endpoints.py b/beacon/utils/endpoints.py index bd969fd6..2dd37960 100644 --- a/beacon/utils/endpoints.py +++ b/beacon/utils/endpoints.py @@ -1,7 +1,5 @@ BEACON_SERVER_ENDPOINT = "http://localhost:9000" AUCTION_SERVER_ENDPOINT = f"http://localhost:9000/bid" -BEACON_SERVER_ENDPOINT_SURFACE = f"{ - BEACON_SERVER_ENDPOINT}/liquidation/submit_opportunity" -BEACON_SERVER_ENDPOINT_GETOPPS = f"{ - BEACON_SERVER_ENDPOINT}/liquidation/fetch_opportunities" +BEACON_SERVER_ENDPOINT_SURFACE = f"{BEACON_SERVER_ENDPOINT}/liquidation/submit_opportunity" +BEACON_SERVER_ENDPOINT_GETOPPS = f"{BEACON_SERVER_ENDPOINT}/liquidation/fetch_opportunities" diff --git a/beacon/utils/pyth_prices.py b/beacon/utils/pyth_prices.py index f8a7f3c1..fe4efcd3 100644 --- a/beacon/utils/pyth_prices.py +++ b/beacon/utils/pyth_prices.py @@ -2,99 +2,163 @@ import asyncio from typing import TypedDict -HERMES_ENDPOINT = "https://hermes.pyth.network/api/" +HERMES_ENDPOINT_HTTPS = "https://hermes.pyth.network/api/" +HERMES_ENDPOINT_WSS = "wss://hermes.pyth.network/ws" class Price(TypedDict): - price: int - conf: int + price: str + conf: str expo: int publish_time: int class PriceFeed(TypedDict): - feed_id: str + id: str price: Price - price_ema: Price + ema_price: Price vaa: str -CLIENT = httpx.AsyncClient() - - -def extract_price_feed(data: dict) -> PriceFeed: - price: Price = data['price'] - price_ema: Price = data['ema_price'] - vaa = data['vaa'] - price_feed: PriceFeed = { - "feed_id": data['id'], - "price": price, - "price_ema": price_ema, - "vaa": vaa - } - return price_feed - - async def get_price_feed_ids() -> list[str]: - url = HERMES_ENDPOINT + "price_feed_ids" - - data = (await CLIENT.get(url)).json() - - return data - - -async def get_pyth_prices_latest( - feedIds: list[str] -) -> list[tuple[str, PriceFeed]]: - url = HERMES_ENDPOINT + "latest_price_feeds?" - params = {"ids[]": feedIds, "binary": "true"} + """ + Queries the Hermes https endpoint for a list of the IDs of all Pyth price feeds. + """ - data = (await CLIENT.get(url, params=params)).json() + url = HERMES_ENDPOINT_HTTPS + "price_feed_ids" + client = httpx.AsyncClient() - results = [] - for res in data: - price_feed = extract_price_feed(res) - results.append((res['id'], price_feed)) + data = (await client.get(url)).json() - return results - - -async def get_pyth_price_at_time( - feed_id: str, - timestamp: int -) -> tuple[str, PriceFeed]: - url = HERMES_ENDPOINT + f"get_price_feed" - params = {"id": feed_id, "publish_time": timestamp, "binary": "true"} - - data = (await CLIENT.get(url, params=params)).json() - - price_feed = extract_price_feed(data) - - return (feed_id, price_feed) - - -async def get_all_prices() -> dict[str, PriceFeed]: - pyth_price_feed_ids = await get_price_feed_ids() + return data - pyth_prices_latest = [] - i = 0 - cntr = 100 - while len(pyth_price_feed_ids[i:i + cntr]) > 0: - pyth_prices_latest += await get_pyth_prices_latest(pyth_price_feed_ids[i:i + cntr]) - i += cntr - return dict(pyth_prices_latest) +class PriceFeedClient: + def __init__(self, feed_ids: list[str]): + self.feed_ids = feed_ids + self.pending_feed_ids = feed_ids + self.prices_dict: dict[str, PriceFeed] = {} + self.client = httpx.AsyncClient() + + def add_feed_ids(self, feed_ids: list[str]): + self.feed_ids += feed_ids + self.feed_ids = list(set(self.feed_ids)) + self.pending_feed_ids += feed_ids + + def extract_price_feed(self, data: dict) -> PriceFeed: + """ + Extracts a PriceFeed object from the JSON response from Hermes. + """ + price = data['price'] + price_ema = data['ema_price'] + vaa = data['vaa'] + price_feed = { + "feed_id": data['id'], + "price": price, + "price_ema": price_ema, + "vaa": vaa + } + return price_feed + + async def get_pyth_prices_latest( + self, + feedIds: list[str] + ) -> list[tuple[str, PriceFeed]]: + """ + Queries the Hermes https endpoint for the latest price feeds for a list of Pyth feed IDs. + """ + url = HERMES_ENDPOINT_HTTPS + "latest_price_feeds?" + params = {"ids[]": feedIds, "binary": "true"} + + data = (await self.client.get(url, params=params)).json() + + results = [] + for res in data: + price_feed = self.extract_price_feed(res) + results.append((res['id'], price_feed)) + + return results + + async def get_pyth_price_at_time( + self, + feed_id: str, + timestamp: int + ) -> tuple[str, PriceFeed]: + """ + Queries the Hermes https endpoint for the price feed for a Pyth feed ID at a given timestamp. + """ + url = HERMES_ENDPOINT_HTTPS + f"get_price_feed" + params = {"id": feed_id, "publish_time": timestamp, "binary": "true"} + + data = (await self.client.get(url, params=params)).json() + + price_feed = self.extract_price_feed(data) + + return (feed_id, price_feed) + + async def get_all_prices(self) -> dict[str, PriceFeed]: + """ + Queries the Hermes http endpoint for the latest price feeds for all feed IDs in the class object. + + There are limitations on the number of feed IDs that can be queried at once, so this function queries the feed IDs in batches. + """ + pyth_prices_latest = [] + i = 0 + batch_size = 100 + while len(self.feed_ids[i:i + batch_size]) > 0: + pyth_prices_latest += await self.get_pyth_prices_latest(self.feed_ids[i:i + batch_size]) + i += batch_size + + return dict(pyth_prices_latest) + + async def ws_pyth_prices(self): + """ + Opens a websocket connection to Hermes for latest prices for all feed IDs in the class object. + """ + import json + import websockets + + async with websockets.connect(HERMES_ENDPOINT_WSS) as ws: + while True: + if len(self.pending_feed_ids) > 0: + json_subscribe = { + "ids": self.pending_feed_ids, + "type": "subscribe", + "verbose": True, + "binary": True + } + await ws.send(json.dumps(json_subscribe)) + self.pending_feed_ids = [] + + msg = json.loads(await ws.recv()) + if msg["type"] == "response": + if msg["status"] != "success": + raise Exception("Error in subscribing to websocket") + try: + if msg["type"] != "price_update": + continue + + feed_id = msg["price_feed"]["id"] + new_feed = msg["price_feed"] + + self.prices_dict[feed_id] = new_feed + + except: + raise Exception("Error in price_update message", msg) async def main(): - pyth_price = await get_pyth_price_at_time("0xff61491a931112ddf1bd8147cd1b641375f79f5825126d665480874634fd0ace", 1703016621) + feed_ids = await get_price_feed_ids() + # TODO: remove this line, once rate limits are figured out + feed_ids = feed_ids[:1] + price_feed_client = PriceFeedClient(feed_ids) - data = await get_all_prices() + print("Starting web socket...") + ws_call = price_feed_client.ws_pyth_prices() + asyncio.create_task(ws_call) - return pyth_price, data + while True: + await asyncio.sleep(1) if __name__ == "__main__": - pyth_price, data = asyncio.run(main()) - - import pdb - pdb.set_trace() + asyncio.run(main()) diff --git a/beacon/utils/types_liquidation_adapter.py b/beacon/utils/types_liquidation_adapter.py index 27cbd3d9..ed947fb1 100644 --- a/beacon/utils/types_liquidation_adapter.py +++ b/beacon/utils/types_liquidation_adapter.py @@ -18,7 +18,6 @@ class LiquidationOpportunity(TypedDict): # The calldata that needs to be passed in with the liquidation method call calldata: str permission_key: str - account: str # A list of tokens that can be used to repay this account's debt. Each entry in the list is a tuple (token address, hex string of repay amount) repay_tokens: list[TokenQty] # A list of tokens that ought to be received by the liquidator in exchange for the repay tokens. Each entry in the list is a tuple (token address, hex string of receipt amount) @@ -41,7 +40,7 @@ class LiquidationAdapterCalldata(TypedDict): signature_liquidator: bytes -class LiquidationAdapterIntent(TypedDict): +class LiquidationAdapterTransaction(TypedDict): bid: str calldata: str chain_id: str