Merge PR #77: Fix experiment filters and reporting on codespeed submission errors

Author: smarr

rebench/model/benchmark_suite.py

@@ -98,3 +98,6 @@ def max_runtime(self):
 
     def has_max_runtime(self):
         return self._max_runtime != -1
+
+    def __str__(self):
+        return "Suite(%s, %s)" % (self._name, self._command)

rebench/rebench.py

@@ -42,7 +42,7 @@
 class ReBench:
 
     def __init__(self):
-        self.version = "0.10.0"
+        self.version = "0.10.1"
         self.options = None
         self._config = None
 
@@ -151,6 +151,15 @@ def shell_options(self):
 
         return parser
 
+    @staticmethod
+    def determine_exp_name_and_filters(filters):
+        exp_name = filters[0] if len(filters) > 0 and (
+                not filters[0].startswith("vm:") and
+                not filters[0].startswith("s:")) else "all"
+        exp_filter = [f for f in filters if (f.startswith("vm:") or
+                                             f.startswith("s:"))]
+        return exp_name, exp_filter
+
     def run(self, argv = None):
         if argv is None:
             argv = sys.argv
@@ -161,13 +170,7 @@ def run(self, argv = None):
 
         cli_reporter = CliReporter(args.verbose)
 
-        # interpret remaining args
-        exp_filter = args.exp_filter
-        exp_name = exp_filter[1] if len(exp_filter) > 1 and (
-            not exp_filter[1].startswith("vm:") and
-            not exp_filter[1].startswith("s:")) else "all"
-        exp_filter = [f for f in exp_filter if (f.startswith("vm:") or
-                                                f.startswith("s:"))]
+        exp_name, exp_filter = self.determine_exp_name_and_filters(args.exp_filter)
 
         try:
             config_filename = args.config[0]

rebench/reporter.py

@@ -415,10 +415,10 @@ def _send_to_codespeed(self, results):
                               "either a wrong URL in the config file, or an "
                               "environment not configured in Codespeed. URL: "
                               + self._cfg.url)
-                envs = set([i['environment'] for i in payload])
-                projects = set([i['project'] for i in payload])
-                benchmarks = set([i['benchmark'] for i in payload])
-                executables = set([i['executable'] for i in payload])
+                envs        = list(set([i['environment'] for i in results]))
+                projects    = list(set([i['project']     for i in results]))
+                benchmarks  = list(set([i['benchmark']   for i in results]))
+                executables = list(set([i['executable']  for i in results]))
                 logging.error("Sent data included environments: %s "
                               "projects: %s benchmarks: %s executables: %s"
                               % (envs, projects, benchmarks, executables))

rebench/tests/executor_test.py

@@ -121,7 +121,52 @@ def test_basic_execution_with_magic_all(self):
         cnf = Configurator(self._path + '/small.conf', DataStore(), None, None,
                            'all', standard_data_file = self._tmp_file)
         self._basic_execution(cnf)
-        
+
+    def test_shell_options_without_filters(self):
+        option_parser = ReBench().shell_options()
+        args = option_parser.parse_args(['-d', '-v', 'some.conf'])
+        self.assertEqual(args.exp_filter, [])
+
+    def test_shell_options_with_filters(self):
+        option_parser = ReBench().shell_options()
+        args = option_parser.parse_args(['-d', '-v', 'some.conf', 'exp_name'])
+        self.assertEqual(args.exp_filter, ['exp_name'])
+
+    def test_shell_options_with_vm_filter(self):
+        option_parser = ReBench().shell_options()
+        args = option_parser.parse_args(['-d', '-v', 'some.conf', 'vm:foo'])
+        self.assertEqual(args.exp_filter, ['vm:foo'])
+
+    def test_determine_exp_name_and_filters_empty(self):
+        empty = []
+        exp_name, exp_filter = ReBench.determine_exp_name_and_filters(empty)
+        self.assertEqual(exp_name, "all")
+        self.assertEqual(exp_filter, [])
+
+    def test_determine_exp_name_and_filters_all(self):
+        filters = ['all']
+        exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters)
+        self.assertEqual(exp_name, "all")
+        self.assertEqual(exp_filter, [])
+
+    def test_determine_exp_name_and_filters_some_name(self):
+        filters = ['foo']
+        exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters)
+        self.assertEqual(exp_name, "foo")
+        self.assertEqual(exp_filter, [])
+
+    def test_determine_exp_name_and_filters_all_and_other(self):
+        filters = ['all', 'vm:bar', 's:b']
+        exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters)
+        self.assertEqual(exp_name, "all")
+        self.assertEqual(exp_filter, ['vm:bar', 's:b'])
+
+    def test_determine_exp_name_and_filters_only_others(self):
+        filters = ['vm:bar', 's:b']
+        exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters)
+        self.assertEqual(exp_name, "all")
+        self.assertEqual(exp_filter, ['vm:bar', 's:b'])
+
 
 def Popen_override(cmdline, stdout, stderr=None, shell=None):
     class Popen:
@@ -154,5 +199,6 @@ def __exit__(self, _type, _value, _traceback):
 def test_suite():
     return unittest.makeSuite(ExecutorTest)
 
+
 if __name__ == "__main__":
     unittest.main(defaultTest='test_suite')