Skip to content

Commit

Permalink
added cost reports for SPL contracts (certain actions)
Browse files Browse the repository at this point in the history
  • Loading branch information
IvanAdamovNeon committed Sep 23, 2024
1 parent a074abb commit f4cac40
Show file tree
Hide file tree
Showing 12 changed files with 316 additions and 110 deletions.
36 changes: 32 additions & 4 deletions .github/workflows/basic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ on:
- 8
- 12
- auto
generate_cost_report:
type: boolean
default: false
required: false
description: "Flag defining whether cost report should be generated"
env:
NETWORK: ${{ github.event.inputs.network || 'terraform' }}
NUMPROCESSES: ${{ github.event.inputs.numprocesses || 8 }}
Expand All @@ -40,6 +45,7 @@ env:
FAUCET_URL: "${{ secrets.DEVNET_FAUCET_URL }}"
IMAGE: ${{ github.repository_owner }}/neon_tests
DOCKER_HUB_ORG_NAME: ${{ github.repository_owner }}
GENERATE_COST_REPORT: ${{ github.event.inputs.generate_cost_report || 'false' }}
jobs:
dockerize:
if: ${{ github.ref_name != 'develop'}}
Expand Down Expand Up @@ -137,12 +143,35 @@ jobs:
timeout-minutes: 60
id: basic
run: |
docker exec -i ${{ env.CONTAINER }} \
/bin/bash -c "export DEVNET_FAUCET_URL=${{ env.FAUCET_URL }} && \
python3 ./clickfile.py run basic --network ${{ env.NETWORK }} --numprocesses ${{ env.NUMPROCESSES }}"
CMD="python3 ./clickfile.py run basic --network ${{ env.NETWORK }} --numprocesses ${{ env.NUMPROCESSES }}"
if [[ "${{ env.GENERATE_COST_REPORT }}" == "true" ]]; then
CMD="$CMD --cost_reports_dir reports/cost_reports"
fi

docker exec -i ${{ env.CONTAINER }} /bin/bash -c "export DEVNET_FAUCET_URL=${{ env.FAUCET_URL }} && $CMD"

- name: Set failed test group to basic
if: failure()
run: echo "FAILED_TEST_GROUP=basic" >> $GITHUB_ENV
- name: Copy cost reports from container
if: ${{ env.GENERATE_COST_REPORT == 'true' }}
run: |
mkdir -p ./reports/cost_reports/ && \
docker cp ${{ env.CONTAINER }}:/opt/neon-tests/reports/cost_reports/. ./reports/cost_reports/
- name: Upload cost reports as artifacts
if: ${{ env.GENERATE_COST_REPORT == 'true' }}
uses: actions/upload-artifact@v4
with:
name: cost-reports
path: reports/cost_reports/**.json
- name: Save Cost Reports to cost_reports.md and echo to Summary
if: ${{ env.GENERATE_COST_REPORT == 'true' }}
run: |
docker exec -i -e NETWORK=${{ env.NETWORK }} ${{ env.CONTAINER }} \
python3 ./clickfile.py dapps save_dapps_cost_report_to_md \
--directory reports/cost_reports && \
docker exec -i ${{ env.CONTAINER }} cat cost_reports.md >> $GITHUB_STEP_SUMMARY
- name: "Generate allure report"
if: always()
uses: ./.github/actions/generate-allure-report
Expand Down Expand Up @@ -181,4 +210,3 @@ jobs:
uses: ./.github/actions/destroy-tf-stand
with:
ci_stands_key_hcloud: ${{ secrets.CI_STANDS_KEY_HCLOUD }}

27 changes: 19 additions & 8 deletions clickfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import pytest

from deploy.cli.cost_report import prepare_report_data, report_data_to_markdown
from deploy.test_results_db.db_handler import PostgresTestResultsHandler
from deploy.test_results_db.test_results_handler import TestResultsHandler
from utils.error_log import error_log
Expand Down Expand Up @@ -58,9 +59,7 @@
}

SRC_ALLURE_CATEGORIES = Path("./allure/categories.json")

DST_ALLURE_CATEGORIES = Path("./allure-results/categories.json")

DST_ALLURE_ENVIRONMENT = Path("./allure-results/environment.properties")

BASE_EXTENSIONS_TPL_DATA = "ui/extensions/data"
Expand Down Expand Up @@ -535,6 +534,7 @@ def update_contracts(branch):
@click.option("-u", "--users", default=8, help="Accounts numbers used in OZ tests")
@click.option("-c", "--case", default="", type=str, help="Specific test case name pattern to run")
@click.option("--marker", help="Run tests by mark")
@click.option("--cost_reports_dir", default="", help="Directory where CostReports will be created")
@click.option(
"--ui-item",
default="all",
Expand Down Expand Up @@ -564,6 +564,7 @@ def run(
case,
keep_error_log: bool,
marker: str,
cost_reports_dir: str,
):
if not network and name == "ui":
network = "devnet"
Expand Down Expand Up @@ -613,14 +614,24 @@ def run(
if network != "geth":
assert wait_for_tracer_service(network)

if case != "":
command += " -vk {}".format(case)
if case:
if " " in case:
command += f' -k "{case}"'
else:
command += f" -k {case}"

if marker:
command += f" -m {marker}"
if " " in marker:
command += f' -m "{marker}"'
else:
command += f" -m {marker}"

command += f" -s --network={network} --make-report --test-group {name}"
if keep_error_log:
command += " --keep-error-log"
if cost_reports_dir:
command += f" --cost_reports_dir {cost_reports_dir}"

args = command.split()[1:]
exit_code = int(pytest.main(args=args))
if name != "ui":
Expand Down Expand Up @@ -1079,7 +1090,7 @@ def save_dapps_cost_report_to_db(
):
tag = evm_tag if repo == "evm" else proxy_tag

report_data = dapps_cli.prepare_report_data(directory)
report_data = prepare_report_data(directory)
db = PostgresTestResultsHandler()

# define if previous similar reports should be deleted
Expand Down Expand Up @@ -1116,7 +1127,7 @@ def save_dapps_cost_report_to_db(
@dapps.command("save_dapps_cost_report_to_md", help="Save dApps Cost Report to cost_reports.md")
@click.option("-d", "--directory", default="reports", help="Directory with reports")
def save_dapps_cost_report_to_md(directory: str):
report_data = dapps_cli.prepare_report_data(directory)
report_data = prepare_report_data(directory)

# Add 'gas_used_%' column after 'gas_used'
report_data.insert(
Expand All @@ -1127,7 +1138,7 @@ def save_dapps_cost_report_to_md(directory: str):
report_data["gas_used_%"] = report_data["gas_used_%"].round(2)

# Dump report_data DataFrame to markdown, grouped by the dApp
report_as_markdown_table = dapps_cli.report_data_to_markdown(df=report_data)
report_as_markdown_table = report_data_to_markdown(df=report_data)
Path("cost_reports.md").write_text(report_as_markdown_table)


Expand Down
19 changes: 19 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import builtins
import os
import json
import shutil
Expand All @@ -24,6 +25,7 @@


pytest_plugins = ["ui.plugins.browser"]
COST_REPORT_DIR: pathlib.Path = pathlib.Path()


@dataclass
Expand Down Expand Up @@ -58,6 +60,12 @@ def pytest_addoption(parser: Parser):
default=False,
help="Store tests result to file",
)
parser.addoption(
"--cost_reports_dir",
default="",
type=pathlib.Path,
help=f"Saves cost reports .json files in {COST_REPORT_DIR}",
)
known_args = parser.parse_known_args(args=sys.argv[1:])
test_group_required = known_args.make_report
parser.addoption(
Expand All @@ -82,6 +90,9 @@ def pytest_sessionstart(session: pytest.Session):
if not keep_error_log:
error_log.clear()

if COST_REPORT_DIR != pathlib.Path() and COST_REPORT_DIR.exists() and COST_REPORT_DIR.is_dir():
shutil.rmtree(COST_REPORT_DIR)


def pytest_runtest_protocol(item: Item, nextitem):
ihook = item.ihook
Expand All @@ -101,6 +112,14 @@ def pytest_runtest_protocol(item: Item, nextitem):


def pytest_configure(config: Config):
# redirect print to stderr for xdist-spawned processes because otherwise print statements get lost
if "PYTEST_XDIST_WORKER" in os.environ:
original_print = builtins.print
builtins.print = lambda *args, **kwargs: original_print(*args, file=sys.stderr, **kwargs)

global COST_REPORT_DIR
COST_REPORT_DIR = config.getoption("--cost_reports_dir")

solana_url_env_vars = ["SOLANA_URL", "DEVNET_INTERNAL_RPC", "MAINNET_INTERNAL_RPC"]
network_name = config.getoption("--network")
envs_file = config.getoption("--envs")
Expand Down
97 changes: 97 additions & 0 deletions deploy/cli/cost_report.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import glob
import json
import os
import pathlib
import re
from collections import Counter

import pandas as pd

from deploy.cli.dapps import NETWORK_MANAGER
from deploy.cli.infrastructure import get_solana_accounts_transactions_compute_units
from utils.web3client import NeonChainWeb3Client


def prepare_report_data(directory: str) -> pd.DataFrame:
proxy_url = NETWORK_MANAGER.get_network_param(os.environ.get("NETWORK"), "proxy_url")
web3_client = NeonChainWeb3Client(proxy_url)

reports = {}
for path in glob.glob(str(pathlib.Path(directory) / "*-report.json")):
with open(path, "r") as f:
rep = json.load(f)
if isinstance(rep, list):
for r in rep:
if "actions" in r:
reports[r["name"]] = r["actions"]
else:
if "actions" in rep:
reports[rep["name"]] = rep["actions"]

data = []

for app, actions in reports.items():
counts = Counter([action["name"].lower().strip() for action in actions])
duplicate_actions = [action for action, count in counts.items() if count > 1]
added_numbers = {dup_action: 1 for dup_action in duplicate_actions}

for action in actions:
# Ensure action name is unique by appending a counter if necessary
base_action_name = action["name"].lower().strip()
if base_action_name in duplicate_actions:
added_number = added_numbers[base_action_name]
unique_action_name = f"{base_action_name} {added_number}"
added_numbers[base_action_name] += 1
else:
unique_action_name = base_action_name

accounts, trx, compute_units = get_solana_accounts_transactions_compute_units(action["tx"])
# accounts, trx, compute_units = (2, 12, 8946)
tx = web3_client.get_transaction_by_hash(action["tx"])
estimated_gas = int(tx.gas) if tx and tx.gas else None
# estimated_gas = 122879
used_gas = int(action["usedGas"])

data.append(
{
"dapp_name": app.lower().strip(),
"action": unique_action_name,
"acc_count": accounts,
"trx_count": trx,
"gas_estimated": estimated_gas,
"gas_used": used_gas,
"compute_units": compute_units,
}
)

df = pd.DataFrame(data)
if df.empty:
raise Exception(f"no reports found in {directory}")
return df


def report_data_to_markdown(df: pd.DataFrame) -> str:
report_content = ""
dapp_names = df['dapp_name'].unique()
df.columns = [col.upper() for col in df.columns]
df['GAS_USED_%'] = df['GAS_USED_%'].apply(lambda x: f"{x:.2f}")

for dapp_name in dapp_names:
dapp_df = df[df['DAPP_NAME'] == dapp_name].drop(columns='DAPP_NAME')

# sort by ACTION (to mitigate [action 1, action 10, action 2, ...])
dapp_df[['ACTION_TEXT', 'ACTION_NUM']] = dapp_df['ACTION'].apply(split_action).apply(pd.Series)
dapp_df = dapp_df.sort_values(by=['ACTION_TEXT', 'ACTION_NUM'])
dapp_df = dapp_df.drop(columns=['ACTION_TEXT', 'ACTION_NUM'])

report_content += f'\n## Cost Report for "{dapp_name.title()}" dApp\n\n'
report_content += dapp_df.to_markdown(index=False) + "\n"

return report_content


def split_action(action) -> tuple[str, int]:
match = re.match(r"(.+?)\s*(\d*)$", action)
text_ = match.group(1)
number = int(match.group(2)) if match.group(2).isdigit() else 0
return text_, number
78 changes: 0 additions & 78 deletions deploy/cli/dapps.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,7 @@
import os
import glob
import json
import typing as tp
import pathlib
from collections import Counter

import pandas as pd

from deploy.cli.infrastructure import get_solana_accounts_transactions_compute_units
from deploy.cli.network_manager import NetworkManager
from utils.web3client import NeonChainWeb3Client


NETWORK_MANAGER = NetworkManager()

Expand All @@ -24,72 +15,3 @@ def set_github_env(envs: tp.Dict, upper=True) -> None:
env_file.write(f"\n{key.upper() if upper else key}={str(value)}")


def prepare_report_data(directory: str) -> pd.DataFrame:
proxy_url = NETWORK_MANAGER.get_network_param(os.environ.get("NETWORK"), "proxy_url")
web3_client = NeonChainWeb3Client(proxy_url)

reports = {}
for path in glob.glob(str(pathlib.Path(directory) / "*-report.json")):
with open(path, "r") as f:
rep = json.load(f)
if isinstance(rep, list):
for r in rep:
if "actions" in r:
reports[r["name"]] = r["actions"]
else:
if "actions" in rep:
reports[rep["name"]] = rep["actions"]

data = []

for app, actions in reports.items():
added_number = 1
counts = Counter([action["name"].lower().strip() for action in actions])
duplicate_actions = [action for action, count in counts.items() if count > 1]

for action in actions:
# Ensure action name is unique by appending a counter if necessary
base_action_name = action["name"].lower().strip()
if base_action_name in duplicate_actions:
unique_action_name = f"{base_action_name} {added_number}"
added_number += 1
else:
unique_action_name = base_action_name

accounts, trx, compute_units = get_solana_accounts_transactions_compute_units(action["tx"])
# accounts, trx, compute_units = (2, 12, 8946)
tx = web3_client.get_transaction_by_hash(action["tx"])
estimated_gas = int(tx.gas) if tx and tx.gas else None
# estimated_gas = 122879
used_gas = int(action["usedGas"])

data.append(
{
"dapp_name": app.lower().strip(),
"action": unique_action_name,
"acc_count": accounts,
"trx_count": trx,
"gas_estimated": estimated_gas,
"gas_used": used_gas,
"compute_units": compute_units,
}
)

df = pd.DataFrame(data)
if df.empty:
raise Exception(f"no reports found in {directory}")
return df


def report_data_to_markdown(df: pd.DataFrame) -> str:
report_content = ""
dapp_names = df['dapp_name'].unique()
df.columns = [col.upper() for col in df.columns]
df['GAS_USED_%'] = df['GAS_USED_%'].apply(lambda x: f"{x:.2f}")

for dapp_name in dapp_names:
dapp_df = df[df['DAPP_NAME'] == dapp_name].drop(columns='DAPP_NAME')
report_content += f'\n## Cost Report for "{dapp_name.title()}" dApp\n\n'
report_content += dapp_df.to_markdown(index=False) + "\n"

return report_content
Loading

0 comments on commit f4cac40

Please sign in to comment.