Merge pull request #36 from TranslatorSRI/fix_package

Move config folder into package folder for pypi

README.md

@@ -1,17 +1,17 @@
-# Translator Benchmarks
+# Translator Benchmarks Runner
 
 This repository provides a set of benchmarks as well as the code to send the queries and evaluate the returned results of a benchmark.
 
-`benchmarks` contains the code to query targets and evaluate results.
+`benchmarks-runner` contains the code to query targets and evaluate results.
 
-`config` contains the data sets, query templates, targets, and benchmark definitions necessary to run a benchmark. See `config/README.md` for details about targets and benchmarks.
+`benchmarks-runner.config` contains the data sets, query templates, targets, and benchmark definitions necessary to run a benchmark. See `config/README.md` for details about targets and benchmarks.
 
 ## Usage
 Running a benchmark is a two-step process:
 1. Execute the queries of a benchmark and store the scored results.
 2. Evaluate the scored results against the set of ground-truth relevant results.
 
-[Installation](#installation) of the `benchmarks` package provides access to the functions and command-line interface necessary to run benchmarks.
+[Installation](#installation) of the `benchmarks-runner` package provides access to the functions and command-line interface necessary to run benchmarks.
 ### CLI
 The command-line interface is the easiest way to run a benchmark.
 

benchmarks_runner/eval.py

@@ -90,8 +90,8 @@ def evaluate_results(
                 message = results_dir.get(uid, {}).get("message", {})
 
         results_k = sorted(
-            message.get('results', []),
-            key=lambda r: r['analyses'][0].get('score', 0),
+            [] if message.get('results') is None else message['results'],
+            key=lambda r: r.get('analyses', [{ 'score': 0 }])[0].get('score', 0),
             reverse=True
         )[:k]
 

benchmarks_runner/main.py

@@ -28,7 +28,7 @@ async def run_benchmarks(
             output_imgs[target] = imgs
             output_results.extend(output_result)
 
-    return output_results, imgs
+    return output_results, output_imgs
 
 
 if __name__ == "__main__":

benchmarks_runner/utils/constants.py

@@ -1,7 +1,7 @@
 import json
 import pathlib
 
-PROJECT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent
+PROJECT_DIR = pathlib.Path(__file__).resolve().parent.parent
 CONFIG_DIR = PROJECT_DIR / 'config'
 
 with open(CONFIG_DIR / 'benchmarks.json', 'r') as file:

setup.py

@@ -1,3 +1,4 @@
+import os
 from setuptools import find_packages, setup
 
 with open("README.md", encoding="utf-8") as readme_file:
@@ -12,7 +13,11 @@
     description="Translator Benchmarks Runner",
     long_description_content_type="text/markdown",
     long_description=readme,
-    package_data={'benchmarks_runnner': ['config/*']},
+    # include all config files
+    package_data={'benchmarks-runner': [
+        'config/**/*.json',
+        'config/**/data.tsv',
+    ]},
     include_package_data=True,
     packages=find_packages(),
     install_requires=[