Fix --only-incomplete-operations, expand basic info in status overview (

#501)

* Fix --only-incomplete-operations.

* Fix incorrect numbers of job labels when used with --only-incomplete-operations.

* Display current/total job counts in label progress bars.

* Fix total number of jobs.

* Show number of jobs with any eligible operations.

* Update changelog.

changelog.txt

@@ -17,6 +17,7 @@ Added
 - Documentation for all directives (#480).
 - Defined validators for the ``fork`` directive (#480).
 - Submission summary now appears in ``FlowProject`` status output, showing the number of queued, running, unknown statuses. (#472, #488).
+- Status overview now shows the number of jobs with incomplete operations and totals for the label overviews (#501).
 
 Changed
 +++++++
@@ -31,6 +32,7 @@ Fixed
 - Strictly enforce that operation functions cannot be used as condition functions (and vice-versa) and prevent the registration of two operations with the same name (#496).
 - Changed default value of ``status_parallelization`` to none, to avoid bugs in user code caused by thread parallelism and overhead in process parallelism (#486).
 - Memory directives are converted to an integer number of gigabytes or megabytes in submission scripts (#482, #484).
+- Fixed behavior of ``--only-incomplete-operations`` (#501).
 
 Removed
 +++++++

flow/project.py

@@ -2624,31 +2624,38 @@ def print_status(
             for i, error in enumerate(errors):
                 logger.debug("Status update error #%i: '%s'", i + 1, error)
 
-        if only_incomplete:
-            # Remove jobs with no eligible groups from the status info
+        # Get the total number of statuses before removing those with no
+        # eligible groups.
+        total_num_jobs = len(status_results)
 
-            def _incomplete(status_entry):
-                return any(
-                    group["eligible"] for group in status_entry["groups"].values()
-                )
+        def _has_any_eligible_group(status_entry):
+            return any(group["eligible"] for group in status_entry["groups"].values())
 
-            status_results = list(filter(_incomplete, status_results))
+        if only_incomplete:
+            # Remove jobs with no eligible groups from the status info.
+            status_results = list(filter(_has_any_eligible_group, status_results))
+            total_num_eligible_jobs = len(status_results)
+        else:
+            total_num_eligible_jobs = sum(
+                1 for _ in filter(_has_any_eligible_group, status_results)
+            )
 
         statuses = {
             status_entry["aggregate_id"]: status_entry
             for status_entry in status_results
         }
 
-        # Add labels to the status information
+        # Add labels to the status information.
         for job_label_data in job_labels:
             job_id = job_label_data["job_id"]
             # There is no status information if the project has no operations.
             # If no status information exists for this job, we need to set
             # default values.
-            statuses.setdefault(job_id, {})
-            statuses[job_id].setdefault("aggregate_id", job_id)
-            statuses[job_id].setdefault("groups", {})
-            statuses[job_id]["labels"] = job_label_data["labels"]
+            if job_id in statuses:
+                # Don't create label entries for job ids that were removed by
+                # --only-incomplete-operations.
+                statuses[job_id].setdefault("groups", {})
+                statuses[job_id]["labels"] = job_label_data["labels"]
 
         # If the dump_json variable is set, just dump all status info
         # formatted in JSON to screen.
@@ -2663,7 +2670,7 @@ def _incomplete(status_entry):
                 for label in status["labels"]:
                     progress[label] += 1
             # Sort the label progress by amount complete (descending), then
-            # alphabetically
+            # alphabetically.
             progress_sorted = list(
                 islice(
                     sorted(progress.items(), key=lambda x: (-x[1], x[0])),
@@ -2752,6 +2759,9 @@ def dotted_get(mapping, key):
 
         status_legend = " ".join(f"[{v}]:{k}" for k, v in self.ALIASES.items())
         context["jobs"] = list(statuses.values())
+        context["total_num_jobs"] = total_num_jobs
+        context["total_num_eligible_jobs"] = total_num_eligible_jobs
+        context["total_num_job_labels"] = len(job_labels)
         context["overview"] = overview
         context["detailed"] = detailed
         context["all_ops"] = all_ops

flow/render_status.py

@@ -79,7 +79,12 @@ def draw_progress_bar(value, total, escape="", width=40):
 
         """
         assert value >= 0 and total > 0
-        bar_format = escape + f"|{{bar:{width}}}" + escape + "| {percentage:<0.2f}%"
+        bar_format = (
+            escape
+            + f"|{{bar:{width}}}"
+            + escape
+            + "| {n_fmt}/{total_fmt} ({percentage:<0.2f}%)"
+        )
         return tqdm.format_meter(n=value, total=total, elapsed=0, bar_format=bar_format)
 
     def job_filter(job_op, scheduler_status_code, all_ops):

flow/templates/base_status.jinja

@@ -15,14 +15,13 @@
 
 {% block overview %}
 {% if overview %}
-{{ 'Overview: \n' }}
-{{ 'Total # of jobs: %s ' | format(jobs | length) }}
+Overview: {{ total_num_jobs }} jobs, {{ total_num_eligible_jobs }} jobs with eligible operations.
 
 {% block progress %}
 | label | ratio |
 | ----- | ----- |
 {% for label in progress_sorted %}
-| {{ label[0] }} | {{ label[1]|draw_progress_bar(jobs | length, '\\') }} |
+| {{ label[0] }} | {{ label[1]|draw_progress_bar(total_num_job_labels, '\\') }} |
 {% endfor %}
 {% endblock%}