diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob.cs index 10839bde..428c6bea 100644 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob.cs @@ -76,7 +76,7 @@ CancellationToken cancellationToken { clearMLTaskId = clearMLTask.Id; } - + ProgressStatus previousStatus = new(); int lastIteration = 0; while (true) { @@ -109,10 +109,31 @@ or ClearMLTaskStatus.Completed switch (clearMLTask.Status) { case ClearMLTaskStatus.InProgress: + float inferencePercentComplete = await _clearMLService.GetInferencePercentCompleteAsync( + clearMLTaskId + ); + ProgressStatus currentStatus = + new( + clearMLTask.LastIteration, + ( + inferencePercentComplete > 0.0f + ? 90.0f + : 90.0f * (clearMLTask.LastIteration / (float)_options.CurrentValue.MaxSteps) + ) + + 10.0f * (inferencePercentComplete / 100.0f) + ); + if (!previousStatus.Equals(currentStatus)) + { + await _platformService.UpdateBuildStatusAsync(buildId, currentStatus); + previousStatus = currentStatus; + } + lastIteration = clearMLTask.LastIteration; + break; case ClearMLTaskStatus.Completed: + currentStatus = new(clearMLTask.LastIteration, 100.0); if (lastIteration != clearMLTask.LastIteration) { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + await _platformService.UpdateBuildStatusAsync(buildId, currentStatus); lastIteration = clearMLTask.LastIteration; } break; diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs index 0d8eb18a..e2ed8e20 100644 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs +++ b/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs @@ -195,6 +195,39 @@ public async Task> GetTaskMetricsAsync( return results; } + private async Task GetTaskMetricAsync( + string taskId, + string metricName, + string variantName, + CancellationToken cancellationToken = default + ) + { + var body = new JsonObject { ["id"] = taskId }; + JsonObject? result = await CallAsync("tasks", "get_by_id_ex", body, cancellationToken); + var tasks = (JsonArray?)result?["data"]?["tasks"]; + if (tasks is null || tasks.Count == 0) + return null; + JsonObject task = (JsonObject)tasks[0]!; + //ClearML represents metrics and values of a hash (using MD5) map of an array of + //hash(metric_name) each with a subarray of hash(variant_name) each with a metric + //object containing the value. + string metricNameHash, + variantNameHash; + using (var md5 = MD5.Create()) + { + metricNameHash = Convert.ToHexString(md5.ComputeHash(Encoding.ASCII.GetBytes(metricName))).ToLower(); + variantNameHash = Convert.ToHexString(md5.ComputeHash(Encoding.ASCII.GetBytes(variantName))).ToLower(); + } + return (string?)task?["last_metrics"]?[metricNameHash]?[variantNameHash]?["value"]; + } + + public async Task GetInferencePercentCompleteAsync(string id, CancellationToken cancellationToken = default) + { + return float.Parse( + await GetTaskMetricAsync(id, "inference_percent_complete", "inference_percent_complete") ?? "0.0" + ); + } + private async Task GetTaskAsync(JsonObject body, CancellationToken cancellationToken = default) { body["only_fields"] = new JsonArray( diff --git a/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs b/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs index b76152cd..1785fd7a 100644 --- a/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs +++ b/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs @@ -28,4 +28,5 @@ Task> GetTaskMetricsAsync( string id, CancellationToken cancellationToken = default ); + Task GetInferencePercentCompleteAsync(string id, CancellationToken cancellationToken = default); } diff --git a/src/sentencepiece4cbuild/_deps/sentencepiece-src b/src/sentencepiece4cbuild/_deps/sentencepiece-src new file mode 160000 index 00000000..d8711f55 --- /dev/null +++ b/src/sentencepiece4cbuild/_deps/sentencepiece-src @@ -0,0 +1 @@ +Subproject commit d8711f55d9b2cb9c77a00adcc18108482b29b675