From d868c4c8154e88cdac6d673906bf34b2376783c5 Mon Sep 17 00:00:00 2001 From: "Charles Chen (Engrg-Hardware 1)" Date: Fri, 11 Feb 2022 12:13:04 -0800 Subject: [PATCH] Better Decoupling of HuggingFace Demo Tests Signed-off-by: Rajeev Rao --- demo/HuggingFace/CHANGELOG.md | 29 +++++++++++++++ demo/HuggingFace/NNDF/README.md | 1 - demo/HuggingFace/README.md | 11 +++++- demo/HuggingFace/run.py | 25 ++++++++++--- demo/HuggingFace/tests/test_interface.py | 45 ++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 demo/HuggingFace/CHANGELOG.md create mode 100644 demo/HuggingFace/tests/test_interface.py diff --git a/demo/HuggingFace/CHANGELOG.md b/demo/HuggingFace/CHANGELOG.md new file mode 100644 index 00000000..10423df0 --- /dev/null +++ b/demo/HuggingFace/CHANGELOG.md @@ -0,0 +1,29 @@ +# HF-OSS Demo changelog + +Uses [changelog conventions](https://keepachangelog.com/en/1.0.0/). +Uses [semantic versioning](https://semver.org/). + +## Guiding Principles +- Changelogs are for humans, not machines. +- There should be an entry for every single version. +- The same types of changes should be grouped. +- Versions and sections should be linkable. +- The latest version comes first. +- The release date of each version is displayed. +- Mention whether you follow Semantic Versioning. + +## Types of changes +- `Added` for new features. +- `Changed` for changes in existing functionality. +- `Deprecated` for soon-to-be removed features. +- `Removed` for now removed features. +- `Fixed` for any bug fixes. +- `Security` in case of vulnerabilities. + +# [1.1.0] - 2022-02-09 + +- Added `-o` or `--save-output-fpath` which saves a pickled version of the `NetworkResult` object. Useful for testing. + +# [1.0.0] - 2022 + +- Added initial working example of HF samples and notebooks. diff --git a/demo/HuggingFace/NNDF/README.md b/demo/HuggingFace/NNDF/README.md index b872c70f..8a0cc98c 100644 --- a/demo/HuggingFace/NNDF/README.md +++ b/demo/HuggingFace/NNDF/README.md @@ -16,4 +16,3 @@ In other words: * Re-use high level measurement tools supplied by well known frameworks * Ensure fair platform for timing TRT performance alongside other frameworks by using the same post-processing code. -* NNDT (Neural Network Driven Testing) can be used to interface with NNDF developed project to test networks directly. diff --git a/demo/HuggingFace/README.md b/demo/HuggingFace/README.md index e768fa16..42a9d346 100644 --- a/demo/HuggingFace/README.md +++ b/demo/HuggingFace/README.md @@ -13,7 +13,7 @@ Currently, this repository supports the following models: t5-small (60M), t5-base (220M), t5-large (770M) -## Installation +## Setup ```python pip3 install -r requirements.txt @@ -33,6 +33,15 @@ The above script reports : | trt | 1 | 0.00494083 | 0.0068982 | 0.0239782 | +## Testing + +```python +pytest +``` + +It is recommended to use Pytest `4.6.x`. Your Python environment must have already had the setup completed. + + ## How to run functional and performance benchmark ```python diff --git a/demo/HuggingFace/run.py b/demo/HuggingFace/run.py index ca6778c2..7f7bd44a 100644 --- a/demo/HuggingFace/run.py +++ b/demo/HuggingFace/run.py @@ -21,6 +21,7 @@ import os import sys +import pickle import argparse import importlib @@ -92,14 +93,31 @@ class RunAction(NetworkScriptAction): def execute(self, args: argparse.Namespace): module = self.load_script(args.script, args) module.RUN_CMD._parser = self.parser - os.chdir(args.network) - print(module.RUN_CMD()) + + old_path = os.getcwd() + # Execute script in each relevant folder + try: + os.chdir(args.network) + results = module.RUN_CMD() + finally: + os.chdir(old_path) + + # Output to terminal + print(results) + + # Dump results as a pickle file if applicable. + # Useful for testing or post-processing. + if args.save_output_fpath: + with open(args.save_output_fpath, "wb") as f: + pickle.dump(results, f) + return 0 def add_args(self, parser: argparse.ArgumentParser): super().add_args(parser) run_group = parser.add_argument_group("run args") run_group.add_argument("script", choices=self.PER_NETWORK_SCRIPTS) + run_group.add_argument("--save-output-fpath", "-o", default=None, help="Outputs a pickled NetworkResult object. See networks.py for definition.") class CompareAction(NetworkScriptAction): @@ -231,9 +249,6 @@ def get_default_parser( def main() -> None: """ Parses network folders and responsible for passing --help flags to subcommands if --network is provided. - - Returns: - None """ # Get all available network scripts networks = register_network_folders(os.getcwd()) diff --git a/demo/HuggingFace/tests/test_interface.py b/demo/HuggingFace/tests/test_interface.py new file mode 100644 index 00000000..6090c7c0 --- /dev/null +++ b/demo/HuggingFace/tests/test_interface.py @@ -0,0 +1,45 @@ +""" +Tests and verifies our interface objects +""" + +# std +import os +import sys + +# pytest +import pytest + +# Add library path +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(TEST_DIR, os.pardir)) + + +@pytest.fixture(scope="session") +def inetwork(): + import NNDF.networks as mod + return mod + + +def test_network_result(inetwork): + # Test the API by explicit flags + inetwork.NetworkResult( + input="example", + output_tensor=[], + semantic_output="hello", + median_runtime=9001, + models=[], + ) + + +def test_network_checkpoint_result(inetwork): + inetwork.NetworkCheckpointResult(network_results=[], accuracy=9001.0) + + +def test_precision(inetwork): + inetwork.Precision(fp16=True) + + +def test_network_metadata(inetwork): + inetwork.NetworkMetadata( + variant="gpt2", precision=inetwork.Precision(fp16=True), other=None + )