Percent complete

src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob.cs

@@ -76,7 +76,7 @@ CancellationToken cancellationToken
             {
                 clearMLTaskId = clearMLTask.Id;
             }
-
+            ProgressStatus previousStatus = new();
             int lastIteration = 0;
             while (true)
             {
@@ -109,10 +109,31 @@ or ClearMLTaskStatus.Completed
                 switch (clearMLTask.Status)
                 {
                     case ClearMLTaskStatus.InProgress:
+                        float inferencePercentComplete = await _clearMLService.GetInferencePercentCompleteAsync(
+                            clearMLTaskId
+                        );
+                        ProgressStatus currentStatus =
+                            new(
+                                clearMLTask.LastIteration,
+                                (
+                                    inferencePercentComplete > 0.0f
+                                        ? 90.0f
+                                        : 90.0f * (clearMLTask.LastIteration / (float)_options.CurrentValue.MaxSteps)
+                                )
+                                    + 10.0f * (inferencePercentComplete / 100.0f)
+                            );
+                        if (!previousStatus.Equals(currentStatus))
+                        {
+                            await _platformService.UpdateBuildStatusAsync(buildId, currentStatus);
+                            previousStatus = currentStatus;
+                        }
+                        lastIteration = clearMLTask.LastIteration;
+                        break;
                     case ClearMLTaskStatus.Completed:
+                        currentStatus = new(clearMLTask.LastIteration, 100.0);
                         if (lastIteration != clearMLTask.LastIteration)
                         {
-                            await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration);
+                            await _platformService.UpdateBuildStatusAsync(buildId, currentStatus);
                             lastIteration = clearMLTask.LastIteration;
                         }
                         break;

src/SIL.Machine.AspNetCore/Services/ClearMLService.cs

@@ -195,6 +195,39 @@ public async Task<IReadOnlyDictionary<string, double>> GetTaskMetricsAsync(
         return results;
     }
 
+    private async Task<string?> GetTaskMetricAsync(
+        string taskId,
+        string metricName,
+        string variantName,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var body = new JsonObject { ["id"] = taskId };
+        JsonObject? result = await CallAsync("tasks", "get_by_id_ex", body, cancellationToken);
+        var tasks = (JsonArray?)result?["data"]?["tasks"];
+        if (tasks is null || tasks.Count == 0)
+            return null;
+        JsonObject task = (JsonObject)tasks[0]!;
+        //ClearML represents metrics and values of a hash (using MD5) map of an array of
+        //hash(metric_name) each with a subarray of hash(variant_name) each with a metric
+        //object containing the value.
+        string metricNameHash,
+            variantNameHash;
+        using (var md5 = MD5.Create())
+        {
+            metricNameHash = Convert.ToHexString(md5.ComputeHash(Encoding.ASCII.GetBytes(metricName))).ToLower();
+            variantNameHash = Convert.ToHexString(md5.ComputeHash(Encoding.ASCII.GetBytes(variantName))).ToLower();
+        }
+        return (string?)task?["last_metrics"]?[metricNameHash]?[variantNameHash]?["value"];
+    }
+
+    public async Task<float> GetInferencePercentCompleteAsync(string id, CancellationToken cancellationToken = default)
+    {
+        return float.Parse(
+            await GetTaskMetricAsync(id, "inference_percent_complete", "inference_percent_complete") ?? "0.0"
+        );
+    }
+
     private async Task<ClearMLTask?> GetTaskAsync(JsonObject body, CancellationToken cancellationToken = default)
     {
         body["only_fields"] = new JsonArray(

src/SIL.Machine.AspNetCore/Services/IClearMLService.cs

@@ -28,4 +28,5 @@ Task<IReadOnlyDictionary<string, double>> GetTaskMetricsAsync(
         string id,
         CancellationToken cancellationToken = default
     );
+    Task<float> GetInferencePercentCompleteAsync(string id, CancellationToken cancellationToken = default);
 }