diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..603cfe5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: EtWnn + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1- Give your python version and the version of this library +2- Show the code you tried to execute +3- Show the full error / bug obtained + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/ETHWatch/__init__.py b/ETHWatch/__init__.py deleted file mode 100644 index 9cf9eef..0000000 --- a/ETHWatch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__author__ = 'EtWnn' -__version__ = '0.1.0' diff --git a/README.rst b/README.rst index 74ddffb..3fd0956 100644 --- a/README.rst +++ b/README.rst @@ -1,40 +1,101 @@ -======== -ETHWatch -======== +============================== +Welcome to ScanWatch 0.1.1 +============================== -A local tracker of transactions for ETH address -It is a simple interface with the `etherscan `_ API. +Note +---- + +This library is under development by EtWnn, feel free to drop your suggestions or remarks in +the discussion tab of the git repo. You are also welcome to contribute by submitting PRs. + +This library is made to retrieve price or candle history of crypto assets using multiple sources. + +**Source Code:** + https://github.com/EtWnn/ScanWatch +**Documentation:** + https://scanwatch.readthedocs.io + + +This library is a local tracker of transactions for the Ethereum chain and the Binance Smart chain. +It is a simple interface with the `etherscan `_ and the +`bscscan `_ APIs and will save locally the results to gain time and avoid over-calling the APIs. Quick Tour ---------- +You will need to generate an API token to use this library. +Go on `etherscan `__ for the Ethereum chain and on +`bscscan `__ for the BSC chain. +(If you want to use both chains, you will need an API token for each). + +``ScanWatch`` is not yet available on `PYPI `_, install with ``pip``: -`Generate an API token `_ in your etherscan account. +.. code:: bash -``ETHWatch`` is not yet available on ``PYPI``, but it can be installed with ``pip``: + pip install ScanWatch + +You can also install the latest developments (not stable): .. code:: bash - pip install git+https://github.com/EtWnn/ETHWatch.git + pip install git+https://github.com/EtWnn/ScanWatch.git@develop + +You can then use your API token to instantiate the manager. + +Example for Ethereum: + +.. code:: python + + from ScanWatch import ScanManager + from ScanWatch.utils.enums import NETWORK -Use your api token to initiate the manager: + api_token = "" + address = "" + + manager = ScanManager(address, NETWORK.ETHER, api_token) + +Example for BSC: .. code:: python - from ETHWatch.Client import Client + from ScanWatch import ScanManager + from ScanWatch.utils.enums import NETWORK + + api_token = "" + address = "" + + manager = ScanManager(address, NETWORK.BSC, api_token) + +Once the manager is setup, you can update the locally saved transactions: + +.. code:: python + + manager.update_all_transactions() + # all transactions updated for address 0xaAC...748E8: 100%|████████████| 4/4 [00:02<00:00, 1.86it/s] + +This needs to be done only when new transactions have been made since the last time you called the update method. +Otherwise you can just fetch the transactions that have been previously saved, as shown below +(see the documentation for more details). + +.. code:: python + + from ScanWatch.utils.enums import TRANSACTION + + manager.get_transactions(TRANSACTION.NORMAL) # normal transactions - api_token = "" + manager.get_transactions(TRANSACTION.ERC20) # erc20 transactions - client = Client(api_token) + manager.get_transactions(TRANSACTION.ERC721) # erc721 transactions - eth_address = "" + manager.get_transactions(TRANSACTION.INTERNAL) # internal transactions - # get the ETH balance - client.get_balance(eth_address) - # get your ETH transactions: - client.get_normal_transactions(eth_address) +Donation +-------- - # get your ERC20 transactions: - client.get_erc20_transactions(eth_address) +If this library has helped you in any way, feel free to donate: +- **BTC**: 14ou4fMYoMVYbWEKnhADPJUNVytWQWx9HG +- **ETH**: 0xfb0ebcf8224ce561bfb06a56c3b9a43e1a4d1be2 +- **LTC**: LfHgc969RFUjnmyLn41SRDvmT146jUg9tE +- **EGLD**: erd1qk98xm2hgztvmq6s4jwtk06g6laattewp6vh20z393drzy5zzfrq0gaefh diff --git a/ETHWatch/Client.py b/ScanWatch/Client.py similarity index 78% rename from ETHWatch/Client.py rename to ScanWatch/Client.py index 3517d4e..6bd76a6 100644 --- a/ETHWatch/Client.py +++ b/ScanWatch/Client.py @@ -2,16 +2,30 @@ import requests +from ScanWatch.exceptions import APIException +from ScanWatch.utils.enums import NETWORK + class Client: """ - Client for the etherscan.io API + Client for the etherscan.io API or the bscscan.io API https://etherscan.io/apis + https://bscscan.com/apis """ - BASE_URL = "https://api.etherscan.io/api" + BASE_ETH_URL = "https://api.etherscan.io/api" + BASE_BSC_URL = "https://api.bscscan.com/api" + + def __init__(self, api_token: str, nt_type: NETWORK): + """ - def __init__(self, api_token: str): + + :param api_token: token for the api + :type api_token: str + :param nt_type: type of the network + :type nt_type: NETWORK + """ self.api_token = api_token + self.nt_type = nt_type def get_mined_blocks(self, address: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ @@ -26,13 +40,16 @@ def get_mined_blocks(self, address: str, start_block: Optional[int] = None, end_ :return: List of mined blocks :rtype: List[Dict] """ - return self._get_transactions(address, 'getminedblocks', start_block, end_block) + try: + return self._get_transactions(address, 'getminedblocks', start_block, end_block) + except APIException: + return [] def get_erc721_transactions(self, address: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ - fetch erc721 transactions on an eth address + fetch erc721 transactions on an eth / bsc address - :param address: ETH address + :param address: address :type address: str :param start_block: fetch transactions starting with this block :type start_block: Optional[int] @@ -45,9 +62,9 @@ def get_erc721_transactions(self, address: str, start_block: Optional[int] = Non def get_erc20_transactions(self, address: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ - fetch erc20 transactions on an eth address + fetch erc20 transactions on an eth / bsc address - :param address: ETH address + :param address: address :type address: str :param start_block: fetch transactions starting with this block :type start_block: Optional[int] @@ -85,9 +102,9 @@ def get_erc20_transactions(self, address: str, start_block: Optional[int] = None def get_normal_transactions(self, address: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ - fetch normal transactions on an eth address + fetch normal transactions on an eth / bsc address - :param address: ETH address + :param address: address :type address: str :param start_block: fetch transactions starting with this block :type start_block: Optional[int] @@ -125,9 +142,9 @@ def get_normal_transactions(self, address: str, start_block: Optional[int] = Non def get_internal_transactions(self, address: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ - fetch internal transactions on an eth address + fetch internal transactions on an eth / bsc address - :param address: ETH address + :param address: address :type address: str :param start_block: fetch transactions starting with this block :type start_block: Optional[int] @@ -141,9 +158,9 @@ def get_internal_transactions(self, address: str, start_block: Optional[int] = N def _get_transactions(self, address: str, action: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ - fetch transactions on an eth address + fetch transactions on an eth / bsc address - :param address: ETH address + :param address: address :type address: str :param action: name of the request for the api (ex 'txlist' or 'txlistinternal') :type action: @@ -162,8 +179,8 @@ def _get_transactions(self, address: str, action: str, start_block: Optional[int action=action, sort='asc', address=address, - start_block=start_block, - end_block=end_block, + startblock=start_block, + endblock=end_block, page=page_number, offset=offset) batch_txs = self.get_result(url) @@ -176,9 +193,9 @@ def _get_transactions(self, address: str, action: str, start_block: Optional[int def get_balance(self, address: str) -> float: """ - fetch the current eth balance of an address + fetch the current eth / bnb balance of an address - :param address: ETH address + :param address: address :type address: str :return: ETH amount :rtype: float @@ -201,18 +218,26 @@ def get_url_request(self, **kwargs) -> str: """ _keywords = {**kwargs, "apikey": self.api_token} string_kws = "&".join((f"{key}={value}" for key, value in _keywords.items())) - return f"{Client.BASE_URL}?{string_kws}" + if self.nt_type == NETWORK.ETHER: + base_url = Client.BASE_ETH_URL + elif self.nt_type == NETWORK.BSC: + base_url = Client.BASE_BSC_URL + else: + raise ValueError(f"unknown network type: {self.nt_type}") + return f"{base_url}?{string_kws}" @staticmethod def get_result(url: str): """ call the API with an url, raise if the status is not ok and return the API result - :param url: url to request for the etherscan.io + :param url: url to request for the etherscan.io or bscscan.com :type url: str :return: API result :rtype: depend of the endpoint """ response = requests.get(url) - response.raise_for_status() - return response.json()['result'] + r_json = response.json() + if int(r_json['status']) > 0 or r_json['message'] == 'No transactions found': + return r_json['result'] + raise APIException(response) diff --git a/ScanWatch/ScanManager.py b/ScanWatch/ScanManager.py new file mode 100644 index 0000000..f0a6c3d --- /dev/null +++ b/ScanWatch/ScanManager.py @@ -0,0 +1,77 @@ +from tqdm import tqdm + +from ScanWatch.Client import Client +from ScanWatch.storage.ScanDataBase import ScanDataBase +from ScanWatch.utils.enums import NETWORK, TRANSACTION + + +class ScanManager: + """ + This class is the interface between the user, the API and the Database + """ + + def __init__(self, address: str, nt_type: NETWORK, api_token: str): + """ + Initiate the manager + + :param address: address to monitor + :type address: str + :param nt_type: type of the network + :type nt_type: NETWORK + :param api_token: token to communicate with the API + :type api_token: str + """ + self.address = address + self.nt_type = nt_type + self.client = Client(api_token, self.nt_type) + self.db = ScanDataBase() + + def update_transactions(self, tr_type: TRANSACTION): + """ + Update the transactions of a certain type in the database + + :param tr_type: type of transaction to update + :type tr_type: TRANSACTION + :return: None + :rtype: None + """ + last_block = self.db.get_last_block_number(self.address, self.nt_type, tr_type) + if tr_type == TRANSACTION.NORMAL: + new_transactions = self.client.get_normal_transactions(self.address, start_block=last_block + 1) + elif tr_type == TRANSACTION.INTERNAL: + new_transactions = self.client.get_internal_transactions(self.address, start_block=last_block + 1) + elif tr_type == TRANSACTION.ERC20: + new_transactions = self.client.get_erc20_transactions(self.address, start_block=last_block + 1) + elif tr_type == TRANSACTION.ERC721: + new_transactions = self.client.get_erc721_transactions(self.address, start_block=last_block + 1) + else: + raise ValueError(f"unknown transaction type: {tr_type}") + self.db.add_transactions(self.address, self.nt_type, tr_type, new_transactions) + + def update_all_transactions(self): + """ + Update all the transactions for the address + + :return: None + :rtype: None + """ + tr_types_names = [name for name in dir(TRANSACTION) if not name.startswith('__')] + pbar = tqdm(total=len(tr_types_names)) + for name in tr_types_names: + pbar.set_description(f"fetching {name.lower()} transactions for {self.nt_type.name.lower()} " + f"address {self.address[:5]}...{self.address[-5:]}") + self.update_transactions(getattr(TRANSACTION, name)) + pbar.update() + pbar.set_description(f"all transactions updated for address {self.address[:5]}...{self.address[-5:]}") + pbar.close() + + def get_transactions(self, tr_type: TRANSACTION): + """ + Return the transactions of the provided type that are saved locally for the address of the manager + + :param tr_type: type of transaction to fetch + :type tr_type: TRANSACTION + :return: list of transactions + :rtype: List[Dict] + """ + return self.db.get_transactions(self.address, self.nt_type, tr_type) diff --git a/ScanWatch/__init__.py b/ScanWatch/__init__.py new file mode 100644 index 0000000..4a43b94 --- /dev/null +++ b/ScanWatch/__init__.py @@ -0,0 +1,4 @@ +__author__ = 'EtWnn' +__version__ = '0.1.1' + +from ScanWatch.ScanManager import ScanManager diff --git a/ScanWatch/exceptions.py b/ScanWatch/exceptions.py new file mode 100644 index 0000000..99e793c --- /dev/null +++ b/ScanWatch/exceptions.py @@ -0,0 +1,17 @@ +class APIException(Exception): + + def __init__(self, response): + self.code = 0 + try: + json_res = response.json() + except ValueError: + self.message = 'Invalid JSON error message from the API: {}'.format(response.text) + else: + self.code = int(json_res['status']) + self.message = json_res['result'] + self.status_code = response.status_code + self.response = response + self.request = getattr(response, 'request', None) + + def __str__(self): # pragma: no cover + return 'APIError(code=%s): %s' % (self.code, self.message) diff --git a/ScanWatch/storage/DataBase.py b/ScanWatch/storage/DataBase.py new file mode 100644 index 0000000..22a4df6 --- /dev/null +++ b/ScanWatch/storage/DataBase.py @@ -0,0 +1,332 @@ +from enum import Enum +from typing import List, Tuple, Optional, Any, Union +import sqlite3 + +from ScanWatch.storage.tables import Table +from ScanWatch.utils.LoggerGenerator import LoggerGenerator +from ScanWatch.utils.paths import get_data_path + + +class SQLConditionEnum(Enum): + """ + Enumeration for SQL comparison operator + + https://www.techonthenet.com/sqlite/comparison_operators.php + """ + equal = '=' + greater_equal = '>=' + greater = '>' + lower = '<' + lower_equal = '<=' + diff = '!=' + + +class DataBase: + """ + This class will be used to interact with sqlite3 databases without having to generates sqlite commands + """ + + def __init__(self, name: str): + """ + Initialise a DataBase instance + + :param name: name of the database + :type name: str + """ + self.name = name + self.logger = LoggerGenerator.get_logger(self.name) + self.save_path = get_data_path() / f"{name}.db" + self.db_conn = None + self.db_cursor = None + self.connect() + + def connect(self): + """ + Connect to the sqlite3 database + + :return: None + :rtype: None + """ + self.db_conn = sqlite3.connect(self.save_path) + self.db_cursor = self.db_conn.cursor() + + def close(self): + """ + Close the connection with the sqlite3 database + + :return: None + :rtype: None + """ + self.db_conn.close() + + def _fetch_rows(self, execution_cmd: str) -> List[Tuple]: + """ + Execute a command to fetch some rows and return them + + :param execution_cmd: the command to execute + :type execution_cmd: str + :return: list of the table's rows selected by the command + :rtype: List[Tuple] + """ + rows = [] + try: + self.db_cursor.execute(execution_cmd) + except sqlite3.OperationalError: + return rows + while True: + row = self.db_cursor.fetchone() + if row is None: + break + rows.append(row) + return rows + + def get_row_by_key(self, table: Table, key_value) -> Optional[Tuple]: + """ + Get the row identified by a primary key value from a table + + :param table: table to fetch the key from + :type table: Table + :param key_value: value of the primary key + :type key_value: Any + :return: the raw row of of the table + :rtype: Optional[Tuple] + """ + if table.primary_key is None: + raise ValueError(f"table {table.name} has no explicit primary key") + conditions_list = [(table.primary_key, SQLConditionEnum.equal, key_value)] + rows = self.get_conditions_rows(table, conditions_list=conditions_list) + if len(rows): + return rows[0] + + def get_conditions_rows(self, table: Table, + selection: Union[str, List[str]] = '*', + conditions_list: Optional[List[Tuple[str, SQLConditionEnum, Any]]] = None, + order_list: Optional[List[str]] = None) -> List[Tuple]: + """ + Select rows with optional conditions and optional order + + :param table: table to select the rows from + :type table: Table + :param selection: list of column or SQL type selection + :type selection: Union[str, List[str]] + :param conditions_list: list of conditions to select the row + :type conditions_list: Optional[List[Tuple[str, SQLConditionEnum, Any]]] + :param order_list: List of SQL type order by + :type order_list: Optional[List[str]] + :return: the selected rows + :rtype: List[Tuple] + """ + if isinstance(selection, List): + selection = ','.join(selection) + if conditions_list is None: + conditions_list = [] + if order_list is None: + order_list = [] + execution_cmd = f"SELECT {selection} from {table.name}" + execution_cmd = self._add_conditions(execution_cmd, conditions_list=conditions_list) + execution_cmd = self._add_order(execution_cmd, order_list=order_list) + return self._fetch_rows(execution_cmd) + + def get_all_rows(self, table: Table) -> List[Tuple]: + """ + Get all the rows of a table + + :param table: table to get the rows from + :type table: Table + :return: all the rows of the table + :rtype: List[Tuple] + """ + return self.get_conditions_rows(table) + + def add_row(self, table: Table, row: Tuple, auto_commit: bool = True, update_if_exists: bool = False): + """ + Add a row to a table + + :param table: table to add a row to + :type table: Table + :param row: values to add to the database + :type row: Tuple + :param auto_commit: if the database state should be saved after the changes + :type auto_commit: bool + :param update_if_exists: if an integrity error is raised and this parameter is true, + will update the existing row + :type update_if_exists: bool + :return: None + :rtype: None + """ + row_s = ", ".join(f"'{v}'" for v in row) + row_s = f'({row_s})' + execution_order = f"INSERT INTO {table.name} VALUES {row_s}" + try: + self.db_cursor.execute(execution_order) + if auto_commit: + self.commit() + except sqlite3.OperationalError: + self.create_table(table) + self.db_cursor.execute(execution_order) + if auto_commit: + self.commit() + except sqlite3.IntegrityError as err: + if update_if_exists: + self.update_row(table, row, auto_commit) + else: + existing_row = self.get_row_by_key(table, row[0]) + msg = f"tried to insert {row} in the table {table.name} but the row is occupied: {existing_row}" + self.logger.error(msg) + raise err + + def add_rows(self, table: Table, rows: List[Tuple], auto_commit: bool = True, update_if_exists: bool = False): + """ + Add several rows to a table + + :param table: table to add a row to + :type table: Table + :param rows: list of values to add to the database + :type rows: List[Tuple] + :param auto_commit: if the database state should be saved after the changes + :type auto_commit: bool + :param update_if_exists: if an integrity error is raised and this parameter is true, + will update the existing row + :type update_if_exists: bool + :return: None + :rtype: None + """ + for row in rows: + self.add_row(table, row, auto_commit=False, update_if_exists=update_if_exists) + if auto_commit: + self.commit() + + def update_row(self, table: Table, row: Tuple, auto_commit=True): + """ + Update the value of a row in a table + + :param table: table to get updated + :type table: Table + :param row: values to update + :type row: Tuple + :param auto_commit: if the database state should be saved after the changes + :type auto_commit: bool + :return: None + :rtype: None + """ + row_s = ", ".join(f"{n} = {v}" for n, v in zip(table.columns_names, row)) + execution_order = f"UPDATE {table.name} SET {row_s} WHERE {table.primary_key} = {row[0]}" + self.db_cursor.execute(execution_order) + if auto_commit: + self.commit() + + def create_table(self, table: Table): + """ + Create a table in the database + + :param table: Table instance with the config of the table to create + :type table: Table + :return: None + :rtype: None + """ + create_cmd = self.get_create_cmd(table) + self.db_cursor.execute(create_cmd) + self.db_conn.commit() + + def drop_table(self, table: Union[Table, str]): + """ + Delete a table from the database + + :param table: table or table name to drop + :type table: Union[Table, str] + :return: None + :rtype: None + """ + if isinstance(table, Table): + table = table.name + execution_order = f"DROP TABLE IF EXISTS {table}" + self.db_cursor.execute(execution_order) + self.db_conn.commit() + + def drop_all_tables(self): + """ + Drop all the tables existing in the database + + :return: None + :rtype: None + """ + tables_desc = self.get_all_tables() + for table_desc in tables_desc: + self.drop_table(table_desc[1]) + self.commit() + + def get_all_tables(self) -> List[Tuple]: + """ + Return all the tables existing in the database + + :return: tables descriptions + :rtype: List[Tuple] + """ + cmd = "SELECT * FROM sqlite_master WHERE type='table';" + return self._fetch_rows(cmd) + + def commit(self): + """ + Submit and save the database state + + :return: None + :rtype: None + """ + self.db_conn.commit() + + @staticmethod + def _add_conditions(execution_cmd: str, conditions_list: List[Tuple[str, SQLConditionEnum, Any]]): + """ + Add a list of condition to an SQL command + + :param execution_cmd: SQL command without 'WHERE' statement + :type execution_cmd: str + :param conditions_list: List of condition to add to the SQL command + :type conditions_list: List[Tuple[str, SQLConditionEnum, Any]] + :return: the augmented command + :rtype: str + """ + if len(conditions_list): + add_cmd = ' WHERE' + for column_name, condition, value in conditions_list: + add_cmd = add_cmd + f" {column_name} {condition.value} '{value}' AND" + return execution_cmd + add_cmd[:-4] + else: + return execution_cmd + + @staticmethod + def _add_order(execution_cmd: str, order_list: List[str]): + """ + Add an order specification to an SQL command + + :param execution_cmd: SQL command without 'ORDER BY' statement + :type execution_cmd: str + :param order_list: SQL order + :type order_list: List[str] + :return: the augmented command + :rtype: str + """ + if len(order_list): + add_cmd = ' ORDER BY' + for column_name in order_list: + add_cmd = add_cmd + f" {column_name}," + return execution_cmd + add_cmd[:-1] + ' ASC' + else: + return execution_cmd + + @staticmethod + def get_create_cmd(table: Table) -> str: + """ + Return the command in string format to create a table in the database + + :param table: Table instance with the config if the table to create + :type table: Table + :return: execution command for the table creation + :rtype: str + """ + cmd = "" + if table.primary_key is not None: + cmd = f"[{table.primary_key}] {table.primary_key_sql_type} PRIMARY KEY, " + for arg_name, arg_type in zip(table.columns_names, table.columns_sql_types): + cmd = cmd + f"[{arg_name}] {arg_type}, " + return f"CREATE TABLE {table.name}\n({cmd[:-2]})" diff --git a/ScanWatch/storage/ScanDataBase.py b/ScanWatch/storage/ScanDataBase.py new file mode 100644 index 0000000..2e91241 --- /dev/null +++ b/ScanWatch/storage/ScanDataBase.py @@ -0,0 +1,87 @@ +from typing import Dict, List + +from ScanWatch.storage.DataBase import DataBase +from ScanWatch.storage.tables import get_transaction_table +from ScanWatch.utils.enums import TRANSACTION, NETWORK + + +class ScanDataBase(DataBase): + """ + Handles the recording of the address transactions in a local database + """ + + def __init__(self, name: str = 'scan_db'): + """ + Initialise a Scan database instance + + :param name: name of the database + :type name: str + """ + super().__init__(name) + + def add_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION, transactions: List[Dict]): + """ + Add a list of transactions to the database + + :param address: address involved in the transaction + :type address: str + :param nt_type: type of network + :type nt_type: NETWORK + :param tr_type: type of the transaction to record + :type tr_type: TRANSACTION + :param transactions: list of the transaction to record + :type transactions: List[Dict] + :return: None + :rtype: None + """ + table = get_transaction_table(address, nt_type, tr_type) + for transaction in transactions: + row = table.dict_to_tuple(transaction) + self.add_row(table, row, auto_commit=False) + self.commit() + + def get_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION) -> List[Dict]: + """ + Return the List of the transactions recorded in the database + + :param address: address involved in the transactions + :type address: str + :param nt_type: type of network + :type nt_type: NETWORK + :param tr_type: type of the transaction to fetch + :type tr_type: TRANSACTION + :return: list of the transaction recorded + :rtype: List[Dict] + """ + table = get_transaction_table(address, nt_type, tr_type) + rows = self.get_all_rows(table) + return [table.tuple_to_dict(row) for row in rows] + + def get_last_block_number(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION) -> int: + """ + Return the last block number seen in recorded transactions (per address, type of transaction and network) + If None are found, return 0 + + :param address: address involved in the transactions + :type address: str + :param nt_type: type of network + :type nt_type: NETWORK + :param tr_type: type of the transaction to fetch + :type tr_type: TRANSACTION + :return: last block number + :rtype: int + """ + table = get_transaction_table(address, nt_type, tr_type) + selection = f"MAX({table.blockNumber})" + result = self.get_conditions_rows(table, selection=selection) + default = 0 + try: + result = result[0][0] + except IndexError: + return default + if result is None: + return default + return int(result) + + + diff --git a/ScanWatch/storage/__init__.py b/ScanWatch/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ScanWatch/storage/tables.py b/ScanWatch/storage/tables.py new file mode 100644 index 0000000..7ebaca4 --- /dev/null +++ b/ScanWatch/storage/tables.py @@ -0,0 +1,215 @@ +from typing import List, Optional, Tuple, Dict + +from ScanWatch.utils.enums import NETWORK, TRANSACTION + + +class Table: + """ + This class represent a table in a database. All columns names are dynamic attributes + @DynamicAttrs + This class is used to describe the tables that will be used to in the database + """ + + def __init__(self, name: str, columns_names: List[str], columns_sql_types: List[str], + primary_key: Optional[str] = None, primary_key_sql_type: Optional[str] = None): + """ + Initialise a Table instance + + :param name: name of the table + :type name: str + :param columns_names: names of the columns (except primary column) + :type columns_names: List[str] + :param columns_sql_types: sql types of the previous columns + :type columns_sql_types: List[str] + :param primary_key: name of the primary key (None, if no primary key is needed) + :type primary_key: Optional[str] + :param primary_key_sql_type: sql type of the primary key (None, if no primary key is needed) + :type primary_key_sql_type: Optional[str] + """ + self.name = name + self.columns_names = columns_names + self.columns_sql_types = columns_sql_types + self.primary_key = primary_key + self.primary_key_sql_type = primary_key_sql_type + + for column_name in self.columns_names: + try: + value = getattr(self, column_name) + raise ValueError(f"the name {column_name} conflicts with an existing attribute of value {value}") + except AttributeError: + setattr(self, column_name, column_name) + + if self.primary_key is not None: + setattr(self, self.primary_key, self.primary_key) + + def tuple_to_dict(self, row: Tuple) -> Dict: + """ + Transform a row from Tuple to Dict with column names as keys. + + :param row: a row of this table + :type row: Tuple + :return: the dictionary equivalent of this row + :rtype: Dict + """ + keys = [] + if self.primary_key is not None: + keys.append(self.primary_key) + keys += self.columns_names + if len(keys) != len(row): + raise ValueError(f"{len(keys)} values were expected but the row submitted only has {len(row)}") + return {k: v for k, v in zip(keys, row)} + + def dict_to_tuple(self, row: Dict) -> Tuple: + """ + Transform a row from Dict to Tuple, in the order of the table columns + + :param row: a row of this table + :type row: Dict + :return: the tuple equivalent of this row + :rtype: Tuple + """ + keys = [] + if self.primary_key is not None: + keys.append(self.primary_key) + keys += self.columns_names + try: + return tuple([row[k] for k in keys]) + except KeyError as err: + raise ValueError(f"missing keys in the row provided: {keys} are expected") from err + + +def get_normal_transaction_table(address: str, scan_type: NETWORK): + """ + Return the table used to store the normal transactions for an address and a scan type + + :param address: address of the transactions + :type address: str + :param scan_type: scan type: ether or bsc + :type scan_type: str + :return: table to store the transactions + :rtype: Table + """ + rows = [ + 'blockNumber', + 'timeStamp', + 'hash', + 'nonce', + 'blockHash', + 'transactionIndex', + 'from', + 'to', + 'value', + 'gas', + 'gasPrice', + 'isError', + 'txreceipt_status', + 'input', + 'contractAddress', + 'cumulativeGasUsed', + 'gasUsed', + 'confirmations', + ] + row_types = len(rows) * ['TEXT'] + return Table(f"{scan_type}_{address}_normal_transaction", rows, row_types) + + +def get_transaction_table(address: str, nt_type: NETWORK, tr_type: TRANSACTION): + """ + Return the table used to store the transactions depending on the address, network type and transaction type + + :param address: address of the transactions + :type address: str + :param nt_type: type of network + :type nt_type: NETWORK + :param tr_type: type of the transaction to record + :type tr_type: TRANSACTION + :return: corresponding table + :rtype: Table + """ + if tr_type == TRANSACTION.NORMAL: + rows = [ + 'blockNumber', + 'timeStamp', + 'hash', + 'nonce', + 'blockHash', + 'transactionIndex', + 'from', + 'to', + 'value', + 'gas', + 'gasPrice', + 'isError', + 'txreceipt_status', + 'input', + 'contractAddress', + 'cumulativeGasUsed', + 'gasUsed', + 'confirmations', + ] + elif tr_type == TRANSACTION.ERC20: + rows = [ + 'blockNumber', + 'timeStamp', + 'hash', + 'nonce', + 'blockHash', + 'from', + 'contractAddress', + 'to', + 'value', + 'tokenName', + 'tokenSymbol', + 'tokenDecimal', + 'transactionIndex', + 'gas', + 'gasPrice', + 'gasUsed', + 'cumulativeGasUsed', + 'input', + 'confirmations', + ] + elif tr_type == TRANSACTION.ERC721: + rows = [ + 'blockNumber', + 'timeStamp', + 'hash', + 'nonce', + 'blockHash', + 'from', + 'contractAddress', + 'to', + 'tokenID', + 'tokenName', + 'tokenSymbol', + 'tokenDecimal', + 'transactionIndex', + 'gas', + 'gasPrice', + 'gasUsed', + 'cumulativeGasUsed', + 'input', + 'confirmations', + ] + elif tr_type == TRANSACTION.INTERNAL: + rows = [ + 'blockNumber', + 'timeStamp', + 'hash', + 'from', + 'to', + 'value', + 'contractAddress', + 'input', + 'type', + 'gas', + 'gasUsed', + 'traceId', + 'isError', + 'errCode' + ] + else: + raise ValueError(f"unknown transaction type: {tr_type}") + + row_types = len(rows) * ['TEXT'] + return Table(f"{nt_type.name.lower()}_{tr_type.name.lower()}_{address}_transaction", rows, row_types) diff --git a/ScanWatch/utils/LoggerGenerator.py b/ScanWatch/utils/LoggerGenerator.py new file mode 100644 index 0000000..b5272a6 --- /dev/null +++ b/ScanWatch/utils/LoggerGenerator.py @@ -0,0 +1,92 @@ +import logging +import os +from typing import Optional + +from ScanWatch.utils.paths import get_data_path + + +class LoggerGenerator: + """ + This class is a utility to facilitate the creation of loggers for the different classes / files + """ + LOGS_FOLDER_PATH = get_data_path() / "logs" + + _default_log_level = logging.WARNING + _default_write_file = False + _logger_count = 0 + + @staticmethod + def set_global_log_level(log_level: int): + """ + set the default log level for loggers creation + + :param log_level: threshold to display the message + :type log_level: logging enum (ex: logging.WARNING) + :return: None + :rtype: None + """ + LoggerGenerator._default_log_level = log_level + + @staticmethod + def set_default_write_file(write_file: bool): + """ + set the default write level for loggers creation + + :param write_file: if the logger should save the message in a file + :type write_file: bool + :return: None + :rtype: None + """ + LoggerGenerator._default_write_file = write_file + + @staticmethod + def get_logger(logger_name: str, write_file: Optional[bool] = None, + log_level: Optional[int] = None) -> logging.Logger: + """ + create a logger that will display messages according to the log level threshold. If specified, it will + also save the messages in a file inside the logs folder + + :param logger_name: name of the logger (a unique logger id will be added after the name) + :type logger_name: str + :param write_file: if the logger should save the message in a file + :type write_file: bool + :param log_level: threshold to display the message + :type log_level: logging enum (ex: logging.WARNING) + :return: the logger object + :rtype: logging.Logger + """ + if log_level is None: + log_level = LoggerGenerator._default_log_level + if write_file is None: + write_file = LoggerGenerator._default_write_file + + # create logger + logger = logging.getLogger(f"lg_{LoggerGenerator._logger_count}_{logger_name}") + logger.setLevel(level=log_level) + LoggerGenerator._logger_count += 1 + + # create formatter and add it to the handlers + log_format = '[%(asctime)s %(name)s %(levelname)s] %(message)s [%(pathname)s:%(lineno)d in %(funcName)s]' + formatter = logging.Formatter(log_format) + + if write_file: + # create file handler for logger. + log_file_path = LoggerGenerator.LOGS_FOLDER_PATH / f"{logger_name}.log" + fh = logging.FileHandler(log_file_path) + fh.setLevel(level=log_level) + fh.setFormatter(formatter) + logger.addHandler(fh) + + # create console handler for logger. + ch = logging.StreamHandler() + ch.setLevel(level=log_level) + ch.setFormatter(formatter) + logger.addHandler(ch) + + return logger + + +try: # create logs folder + os.makedirs(LoggerGenerator.LOGS_FOLDER_PATH) +except FileExistsError: + pass diff --git a/ScanWatch/utils/__init__.py b/ScanWatch/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ScanWatch/utils/enums.py b/ScanWatch/utils/enums.py new file mode 100644 index 0000000..2f0b0eb --- /dev/null +++ b/ScanWatch/utils/enums.py @@ -0,0 +1,13 @@ +from enum import Enum + + +class NETWORK(Enum): + ETHER = 1 + BSC = 2 + + +class TRANSACTION(Enum): + NORMAL = 1 + INTERNAL = 2 + ERC20 = 3 + ERC721 = 4 diff --git a/ScanWatch/utils/paths.py b/ScanWatch/utils/paths.py new file mode 100644 index 0000000..c05cb62 --- /dev/null +++ b/ScanWatch/utils/paths.py @@ -0,0 +1,26 @@ +import os +from pathlib import Path +from appdirs import AppDirs + +_app_dirs = AppDirs("ScanWatch", "EtWnn") + + +def get_data_path(): + """ + Return the folder path where to store the data created by this project + It uses the library appdirs to follow the conventions across multi OS(MAc, Linux, Windows) + + https://pypi.org/project/appdirs/ + + :return: path of the folder to use for data saving + :rtype: pathlib.Path + """ + return Path(_app_dirs.user_data_dir) + + +try: # create the data folder path + os.makedirs(get_data_path()) +except FileExistsError: + pass + + diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..9534b01 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/client.rst b/docs/source/client.rst new file mode 100644 index 0000000..6162621 --- /dev/null +++ b/docs/source/client.rst @@ -0,0 +1,7 @@ +Client +=========== + +.. automodule:: ScanWatch.Client + :special-members: __init__ + :members: + :undoc-members: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..9ce82a3 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,54 @@ +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + + +# -- Project information ----------------------------------------------------- + +project = 'ScanWatch' +copyright = '2021, EtWnn' +author = 'EtWnn' + +# The full version, including alpha/beta/rc tags +this_directory = os.path.abspath(os.path.dirname(__file__)) +about = {} +with open(os.path.join(this_directory, f'../../{project}/__init__.py'), encoding='utf-8') as f: + exec(f.read(), about) +release = about['__version__'] + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.viewcode', 'sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +source_suffix = ['.rst', '.md'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/source/database.rst b/docs/source/database.rst new file mode 100644 index 0000000..558a534 --- /dev/null +++ b/docs/source/database.rst @@ -0,0 +1,12 @@ +DataBase +=========== + +.. automodule:: ScanWatch.storage.ScanDataBase + :special-members: __init__ + :members: + :undoc-members: + +.. automodule:: ScanWatch.storage.DataBase + :special-members: __init__ + :members: + :undoc-members: diff --git a/docs/source/enums.rst b/docs/source/enums.rst new file mode 100644 index 0000000..d3aabcb --- /dev/null +++ b/docs/source/enums.rst @@ -0,0 +1,6 @@ +Enums +===== + +.. automodule:: ScanWatch.utils.enums + :members: + :undoc-members: diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..30ee4f6 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,20 @@ +.. include:: ../../README.rst + +Contents +========= + +.. toctree:: + :maxdepth: 2 + + manager + client + database + enums + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/manager.rst b/docs/source/manager.rst new file mode 100644 index 0000000..e6d8ab6 --- /dev/null +++ b/docs/source/manager.rst @@ -0,0 +1,7 @@ +ScanManager +=========== + +.. automodule:: ScanWatch.ScanManager + :special-members: __init__ + :members: + :undoc-members: diff --git a/requirements.txt b/requirements.txt index f229360..2802d1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,6 @@ requests +appdirs +tqdm +sphinx +sphinx_rtd_theme +twine \ No newline at end of file diff --git a/setup.py b/setup.py index 5c69680..260ba2b 100644 --- a/setup.py +++ b/setup.py @@ -7,22 +7,22 @@ long_description = f.read() about = {} -with open(os.path.join(this_directory, 'ETHWatch/__init__.py'), encoding='utf-8') as f: +with open(os.path.join(this_directory, 'ScanWatch/__init__.py'), encoding='utf-8') as f: exec(f.read(), about) setup( - name='ETHWatch', + name='ScanWatch', version=about['__version__'], - packages=['ETHWatch'], - url='https://github.com/EtWnn/ETHWatch', + packages=['ScanWatch', 'ScanWatch.storage', 'ScanWatch.utils'], + url='https://github.com/EtWnn/ScanWatch', author='EtWnn', author_email='', license='MIT', - description='Local tracker of an address', + description='Local tracker of an eth address for ETH and BSC scan', long_description=long_description, long_description_content_type='text/x-rst', - install_requires=['requests'], - keywords='eth wallet save tracking history ethereum tracker', + install_requires=['requests', 'appdirs', 'tqdm'], + keywords='eth bsc wallet save tracking history ethereum tracker binance smartchain smart chain', classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License',