Merge pull request #222 from pangeachat/testing-speech-to-text

ensure speech_to_text endpoint is only called once, round words for p…
pangeachat · May 20, 2024 · 9a11834 · 9a11834
2 parents 4466c64 + f797cdb
commit 9a11834
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 11 deletions.
diff --git a/lib/pangea/enum/message_mode_enum.dart b/lib/pangea/enum/message_mode_enum.dart
@@ -1,6 +1,7 @@
 import 'package:flutter/material.dart';
 import 'package:flutter_gen/gen_l10n/l10n.dart';
 import 'package:material_symbols_icons/symbols.dart';
+import 'package:matrix/matrix.dart';
 
 enum MessageMode { translation, definition, speechToText, textToSpeech }
 
@@ -52,4 +53,17 @@ extension MessageModeExtension on MessageMode {
             .oopsSomethingWentWrong; // Title to indicate an error or unsupported mode
     }
   }
+
+  bool isValidMode(Event event) {
+    switch (this) {
+      case MessageMode.translation:
+      case MessageMode.textToSpeech:
+      case MessageMode.definition:
+        return event.messageType == MessageTypes.Text;
+      case MessageMode.speechToText:
+        return event.messageType == MessageTypes.Audio;
+      default:
+        return true;
+    }
+  }
 }
diff --git a/lib/pangea/matrix_event_wrappers/pangea_message_event.dart b/lib/pangea/matrix_event_wrappers/pangea_message_event.dart
@@ -346,6 +346,19 @@ class PangeaMessageEvent {
       ),
     );
 
+    _representations?.add(
+      RepresentationEvent(
+        timeline: timeline,
+        content: PangeaRepresentation(
+          langCode: response.langCode,
+          text: response.transcript.text,
+          originalSent: false,
+          originalWritten: false,
+          speechToText: response,
+        ),
+      ),
+    );
+
     return response;
   }
 

diff --git a/lib/pangea/matrix_event_wrappers/pangea_representation_event.dart b/lib/pangea/matrix_event_wrappers/pangea_representation_event.dart
@@ -3,7 +3,6 @@ import 'dart:developer';
 import 'package:fluffychat/pangea/extensions/pangea_event_extension.dart';
 import 'package:fluffychat/pangea/matrix_event_wrappers/pangea_choreo_event.dart';
 import 'package:fluffychat/pangea/models/pangea_token_model.dart';
-import 'package:fluffychat/pangea/models/speech_to_text_models.dart';
 import 'package:fluffychat/pangea/models/tokens_event_content_model.dart';
 import 'package:fluffychat/pangea/repo/tokens_repo.dart';
 import 'package:flutter/foundation.dart';
@@ -27,15 +26,12 @@ class RepresentationEvent {
   ChoreoRecord? _choreo;
   Timeline timeline;
 
-  SpeechToTextModel? _speechToTextResponse;
-
   RepresentationEvent({
     required this.timeline,
     Event? event,
     PangeaRepresentation? content,
     PangeaMessageTokens? tokens,
     ChoreoRecord? choreo,
-    SpeechToTextModel? speechToTextResponse,
   }) {
     if (event != null && event.type != PangeaEventTypes.representation) {
       throw Exception(
@@ -46,7 +42,6 @@ class RepresentationEvent {
     _content = content;
     _tokens = tokens;
     _choreo = choreo;
-    _speechToTextResponse = speechToTextResponse;
   }
 
   Event? get event => _event;

diff --git a/lib/pangea/widgets/chat/message_speech_to_text_card.dart b/lib/pangea/widgets/chat/message_speech_to_text_card.dart
@@ -138,7 +138,7 @@ class MessageSpeechToTextCardState extends State<MessageSpeechToTextCard> {
   }
 
   String? get wordsPerMinuteString =>
-      speechToTextResponse?.transcript.wordsPerMinute?.toString();
+      speechToTextResponse?.transcript.wordsPerMinute?.toStringAsFixed(2);
 
   @override
   Widget build(BuildContext context) {

diff --git a/lib/pangea/widgets/chat/message_toolbar.dart b/lib/pangea/widgets/chat/message_toolbar.dart
@@ -172,6 +172,19 @@ class MessageToolbarState extends State<MessageToolbar> {
     debugPrint("updating toolbar mode");
     final bool subscribed =
         MatrixState.pangeaController.subscriptionController.isSubscribed;
+
+    if (!newMode.isValidMode(widget.pangeaMessageEvent.event)) {
+      ErrorHandler.logError(
+        e: "Invalid mode for event",
+        s: StackTrace.current,
+        data: {
+          "newMode": newMode,
+          "event": widget.pangeaMessageEvent.event,
+        },
+      );
+      return;
+    }
+
     setState(() {
       currentMode = newMode;
       updatingMode = true;
@@ -274,12 +287,14 @@ class MessageToolbarState extends State<MessageToolbar> {
             PLocalKey.autoPlayMessages,
           ) ??
           true;
+
+      if (widget.pangeaMessageEvent.isAudioMessage) {
+        updateMode(MessageMode.speechToText);
+        return;
+      }
+
       autoplay
-          ? updateMode(
-              widget.pangeaMessageEvent.isAudioMessage
-                  ? MessageMode.speechToText
-                  : MessageMode.textToSpeech,
-            )
+          ? updateMode(MessageMode.textToSpeech)
           : updateMode(MessageMode.translation);
     });