Merge pull request flux-framework#563 from cmoussa1/customize.view-jo…

…b-records

`formatter`: add new `JobsFormatter` class, restructure `view-job-records` to use new class

src/bindings/python/fluxacct/accounting/formatter.py

@@ -10,6 +10,9 @@
 # SPDX-License-Identifier: LGPL-3.0
 ###############################################################
 import json
+import string
+
+import flux.util
 
 
 class AccountingFormatter:
@@ -321,3 +324,74 @@ def list_banks(self):
             banks += f"{str(bank[0])}\n"
 
         return banks
+
+
+class JobsFormatter(flux.util.OutputFormat):
+    """
+    Store a parsed version of an output format string for JobRecord objects,
+    allowing the fields to iterated without modifiers, building
+    a new format suitable for headers display, etc...
+    """
+
+    # mapping of legal format fields and their header names
+    headings = {
+        "userid": "userid",
+        "username": "username",
+        "jobid": "jobid",
+        "t_submit": "t_submit",
+        "t_run": "t_run",
+        "t_inactive": "t_inactive",
+        "nnodes": "nnodes",
+        "resources": "resources",
+        "project": "project",
+        "bank": "bank",
+    }
+
+    def __init__(self, fmt, headings=None):
+        """
+        Parse the input format fmt with string.Formatter.
+        Save off the fields and list of format tokens for later use,
+        (converting None to "" in the process)
+
+        Throws an exception if any format fields do not match the allowed
+        list of headings above.
+
+        Special case for annotations, which may be arbitrary
+        creations of scheduler or user.
+        """
+        format_list = string.Formatter().parse(fmt)
+        for _, field, _, _ in format_list:
+            if field and field in self.headings:
+                self.headings[field] = field
+        super().__init__(fmt)
+
+    def build_table(self, items):
+        """
+        Handle constructing a table of job records with the current format.
+
+        Sort items via any sort keys as set by set_sort_keys() or
+        via a ``sort:`` prefix in the supplied format.
+
+        Args:
+            items (iterable): list of items to format
+        """
+        # preprocess original format by processing with filter():
+        newfmt = self.filter(items)
+        # create new instance of the current class from filtered format:
+        formatter = type(self)(newfmt, headings=self.headings)
+
+        items = self.sort_items(items)
+
+        output_str = f"{formatter.header()}\n"
+        for item in items:
+            line = formatter.format(item)
+            if not line or line.isspace():
+                continue
+            try:
+                output_str += f"{line}\n"
+            except UnicodeEncodeError:
+                output_str += (
+                    f"{line.encode('utf-8', errors='surrogateescape').decode()}"
+                )
+
+        return output_str

src/bindings/python/fluxacct/accounting/jobs_table_subcommands.py

@@ -16,6 +16,7 @@
 from flux.resource import ResourceSet
 from flux.util import parse_datetime
 from flux.job.JobID import JobID
+from fluxacct.accounting import formatter as fmt
 
 
 def get_username(userid):
@@ -123,41 +124,22 @@ def write_records_to_file(job_records, output_file):
             )
 
 
-def convert_to_str(job_records):
+def convert_to_str(job_records, fmt_string=None):
     """
     Convert the results of a query to the jobs table to a readable string
     that can either be output to stdout or written to a file.
     """
-    job_record_str = []
-    job_record_str.append(
-        "{:<10} {:<10} {:<20} {:<20} {:<20} {:<20} {:<10} {:<20} {:<20}".format(
-            "UserID",
-            "Username",
-            "JobID",
-            "T_Submit",
-            "T_Run",
-            "T_Inactive",
-            "Nodes",
-            "Project",
-            "Bank",
-        )
-    )
-    for record in job_records:
-        job_record_str.append(
-            "{:<10} {:<10} {:<20} {:<20} {:<20} {:<20} {:<10} {:<20} {:<20}".format(
-                record.userid,
-                record.username,
-                record.jobid,
-                record.t_submit,
-                record.t_run,
-                record.t_inactive,
-                record.nnodes,
-                record.project,
-                record.bank,
-            )
+    # default format string
+    if not fmt_string:
+        fmt_string = (
+            "{jobid:<15} | {username:<8} | {userid:<8} | {t_submit:<15.2f} | "
+            + "{t_run:<15.2f} | {t_inactive:<15.2f} | {nnodes:<8} | {project:<8} | "
+            + "{bank:<8}"
         )
+    output = fmt.JobsFormatter(fmt_string)
+    job_record_str = output.build_table(job_records)
 
-    return "\n".join(job_record_str)
+    return job_record_str
 
 
 def convert_to_obj(rows):
@@ -307,11 +289,11 @@ def get_jobs(conn, **kwargs):
     return job_records
 
 
-def view_jobs(conn, output_file, **kwargs):
+def view_jobs(conn, output_file, fields, **kwargs):
     # look up jobs in jobs table
     job_records = convert_to_obj(get_jobs(conn, **kwargs))
     # convert query result to a readable string
-    job_records_str = convert_to_str(job_records)
+    job_records_str = convert_to_str(job_records, fields)
 
     if output_file is None:
         return job_records_str

src/cmd/flux-account-service.py

@@ -365,6 +365,7 @@ def view_job_records(self, handle, watcher, msg, arg):
             val = j.view_jobs(
                 self.conn,
                 msg.payload["output_file"],
+                msg.payload["format"],
                 jobid=msg.payload["jobid"],
                 user=msg.payload["user"],
                 before_end_time=msg.payload["before_end_time"],

src/cmd/flux-account.py

@@ -293,6 +293,17 @@ def add_view_job_records_arg(subparsers):
         help="bank",
         metavar="BANK",
     )
+    subparser_view_job_records.add_argument(
+        "-o",
+        "--format",
+        type=str,
+        help=(
+            "Specify output format using Python's string format syntax. "
+            "Available fields: jobid,username,userid,t_submit,t_run,t_inactive,nnodes,"
+            "project,bank"
+        ),
+        metavar="FORMAT",
+    )
 
 
 def add_create_db_arg(subparsers):

t/Makefile.am

@@ -120,7 +120,6 @@ EXTRA_DIST= \
 	expected/flux_account/F_bank_users.expected \
 	expected/pop_db/db_hierarchy_base.expected \
 	expected/pop_db/db_hierarchy_new_users.expected \
-	expected/job_usage/no_jobs.expected \
 	expected/sample_payloads/same_fairshare.json \
 	expected/sample_payloads/small_no_tie.json \
 	expected/sample_payloads/small_tie_all.json \

t/t1011-job-archive-interface.t

@@ -67,7 +67,10 @@ test_expect_success 'check that usage does not get affected by canceled jobs' '
 
 test_expect_success 'check that no jobs show up under user' '
 	flux account -p ${DB_PATH} view-job-records --user $username > no_jobs.test &&
-	test_cmp ${NO_JOBS} no_jobs.test
+	cat <<-EOF >no_jobs.expected &&
+	jobid           | username | userid   | t_submit        | t_run           | t_inactive      | nnodes   | project  | bank
+	EOF
+	grep -f no_jobs.expected no_jobs.test
 '
 
 test_expect_success 'submit some jobs and wait for them to finish running' '
@@ -149,6 +152,15 @@ test_expect_success 'call update-usage in the same half-life period where no job
 	jq -e ".[1].job_usage >= 4" <query2.json
 '
 
+test_expect_success 'call view-job-records with custom format string' '
+	flux account view-job-records -o "{userid:<8} || {t_inactive:<12.3f}"
+'
+
+test_expect_success 'call view-job-records -o with an invalid field' '
+	test_must_fail flux account view-job-records -o "{foo}" > invalid_field.out 2>&1 &&
+	grep "Unknown format field: foo" invalid_field.out
+'
+
 test_expect_success 'remove flux-accounting DB' '
 	rm $(pwd)/FluxAccountingTest.db
 '

t/t1041-view-jobs-by-project.t

@@ -75,7 +75,8 @@ test_expect_success 'run fetch-job-records script' '
 
 test_expect_success 'look at all jobs (will show 4 records)' '
 	flux account view-job-records > all_jobs.out &&
-	test $(grep -c "project" all_jobs.out) -eq 4
+	test $(grep -c "projectA" all_jobs.out) -eq 2 &&
+	test $(grep -c "projectB" all_jobs.out) -eq 2
 '
 
 test_expect_success 'filter jobs by projectA (will show 2 records)' '