Merge pull request #436 from pangeachat/pos-morph

Pos-morph

lib/pangea/controllers/language_list_controller.dart

@@ -93,7 +93,6 @@ class PangeaLanguage {
   }
 
   static LanguageModel byLangCode(String langCode) {
-    final list = _langList;
     for (final element in _langList) {
       if (element.langCode == langCode) return element;
     }

lib/pangea/enum/activity_display_instructions_enum.dart

@@ -0,0 +1,13 @@
+enum ActivityDisplayInstructionsEnum { highlight, hide }
+
+extension ActivityDisplayInstructionsEnumExt
+    on ActivityDisplayInstructionsEnum {
+  String get string {
+    switch (this) {
+      case ActivityDisplayInstructionsEnum.highlight:
+        return 'highlight';
+      case ActivityDisplayInstructionsEnum.hide:
+        return 'hide';
+    }
+  }
+}

lib/pangea/matrix_event_wrappers/pangea_message_event.dart

@@ -687,29 +687,34 @@ class PangeaMessageEvent {
 
     for (final itStep in originalSent!.choreo!.itSteps) {
       for (final continuance in itStep.continuances) {
-        // this seems to always be false for continuances right now
+        final List<PangeaToken> tokensToSave =
+            continuance.tokens.where((t) => t.lemma.saveVocab).toList();
 
         if (originalSent!.choreo!.finalMessage.contains(continuance.text)) {
           continue;
         }
         if (continuance.wasClicked) {
           //PTODO - account for end of flow score
           if (continuance.level != ChoreoConstants.levelThresholdForGreen) {
-            uses.addAll(
-              _lemmasToVocabUses(
-                continuance.lemmas,
-                ConstructUseTypeEnum.incIt,
-              ),
-            );
+            for (final token in tokensToSave) {
+              uses.add(
+                _lemmaToVocabUse(
+                  token.lemma,
+                  ConstructUseTypeEnum.incIt,
+                ),
+              );
+            }
           }
         } else {
           if (continuance.level != ChoreoConstants.levelThresholdForGreen) {
-            uses.addAll(
-              _lemmasToVocabUses(
-                continuance.lemmas,
-                ConstructUseTypeEnum.ignIt,
-              ),
-            );
+            for (final token in tokensToSave) {
+              uses.add(
+                _lemmaToVocabUse(
+                  token.lemma,
+                  ConstructUseTypeEnum.ignIt,
+                ),
+              );
+            }
           }
         }
       }
@@ -728,14 +733,16 @@ class PangeaMessageEvent {
     }
 
     // for each token, record whether selected in ga, ta, or wa
-    for (final token in originalSent!.tokens!) {
-      uses.addAll(_getVocabUseForToken(token));
+    for (final token in originalSent!.tokens!
+        .where((token) => token.lemma.saveVocab)
+        .toList()) {
+      uses.add(_getVocabUseForToken(token));
     }
 
     return uses;
   }
 
-  /// Returns a list of [OneConstructUse] objects for the given [token]
+  /// Returns a [OneConstructUse] for the given [token]
   /// If there is no [originalSent] or [originalSent.choreo], the [token] is
   /// considered to be a [ConstructUseTypeEnum.wa] as long as it matches the target language.
   /// Later on, we may want to consider putting it in some category of like 'pending'
@@ -744,11 +751,11 @@ class PangeaMessageEvent {
   /// If the [token] is in the [originalSent.choreo.acceptedOrIgnoredMatch.choices],
   /// it is considered to be a [ConstructUseTypeEnum.corIt].
   /// If the [token] is not included in any choreoStep, it is considered to be a [ConstructUseTypeEnum.wa].
-  List<OneConstructUse> _getVocabUseForToken(PangeaToken token) {
+  OneConstructUse _getVocabUseForToken(PangeaToken token) {
     if (originalSent?.choreo == null) {
       final bool inUserL2 = originalSent?.langCode == l2Code;
-      return _lemmasToVocabUses(
-        token.lemmas,
+      return _lemmaToVocabUse(
+        token.lemma,
         inUserL2 ? ConstructUseTypeEnum.wa : ConstructUseTypeEnum.unk,
       );
     }
@@ -763,45 +770,34 @@ class PangeaMessageEvent {
                     step.text.contains(r.value),
               ) ??
               false)) {
-        return _lemmasToVocabUses(token.lemmas, ConstructUseTypeEnum.ga);
+        return _lemmaToVocabUse(token.lemma, ConstructUseTypeEnum.ga);
       }
       if (step.itStep != null) {
         final bool pickedThroughIT =
             step.itStep!.chosenContinuance?.text.contains(token.text.content) ??
                 false;
         if (pickedThroughIT) {
-          return _lemmasToVocabUses(token.lemmas, ConstructUseTypeEnum.corIt);
+          return _lemmaToVocabUse(token.lemma, ConstructUseTypeEnum.corIt);
           //PTODO - check if added via custom input in IT flow
         }
       }
     }
-    return _lemmasToVocabUses(token.lemmas, ConstructUseTypeEnum.wa);
+    return _lemmaToVocabUse(token.lemma, ConstructUseTypeEnum.wa);
   }
 
-  /// Convert a list of [lemmas] into a list of vocab uses
-  /// with the given [type]
-  List<OneConstructUse> _lemmasToVocabUses(
-    List<Lemma> lemmas,
+  OneConstructUse _lemmaToVocabUse(
+    Lemma lemma,
     ConstructUseTypeEnum type,
-  ) {
-    final List<OneConstructUse> uses = [];
-    for (final lemma in lemmas) {
-      if (lemma.saveVocab) {
-        uses.add(
-          OneConstructUse(
-            useType: type,
-            chatId: event.roomId!,
-            timeStamp: event.originServerTs,
-            lemma: lemma.text,
-            form: lemma.form,
-            msgId: event.eventId,
-            constructType: ConstructTypeEnum.vocab,
-          ),
-        );
-      }
-    }
-    return uses;
-  }
+  ) =>
+      OneConstructUse(
+        useType: type,
+        chatId: event.roomId!,
+        timeStamp: event.originServerTs,
+        lemma: lemma.text,
+        form: lemma.form,
+        msgId: event.eventId,
+        constructType: ConstructTypeEnum.vocab,
+      );
 
   /// get construct uses of type grammar for the message
   List<OneConstructUse> get _grammarConstructUses {

lib/pangea/models/it_response_model.dart

@@ -2,11 +2,10 @@ import 'package:collection/collection.dart';
 import 'package:fluffychat/pangea/constants/choreo_constants.dart';
 import 'package:fluffychat/pangea/constants/model_keys.dart';
 import 'package:fluffychat/pangea/extensions/my_list_extension.dart';
+import 'package:fluffychat/pangea/models/pangea_token_model.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_gen/gen_l10n/l10n.dart';
 
-import 'lemma.dart';
-
 class ITResponseModel {
   String fullTextTranslation;
   List<Continuance> continuances;
@@ -79,7 +78,7 @@ class Continuance {
   double probability;
   int level;
   String text;
-  List<Lemma> lemmas;
+  List<PangeaToken> tokens;
 
   /// saving this in a full json form
   String description;
@@ -99,19 +98,18 @@ class Continuance {
     required this.inDictionary,
     required this.hasInfo,
     required this.gold,
-    required this.lemmas,
+    required this.tokens,
   });
 
   factory Continuance.fromJson(Map<String, dynamic> json) {
-    final List<Lemma> lemmaInternal =
-        (json[ModelKey.lemma] != null && json[ModelKey.lemma] is Iterable)
-            ? (json[ModelKey.lemma] as Iterable)
-                .map<Lemma>(
-                  (e) => Lemma.fromJson(e as Map<String, dynamic>),
-                )
-                .toList()
-                .cast<Lemma>()
-            : [];
+    final List<PangeaToken> tokensInternal = (json[ModelKey.tokens] != null)
+        ? (json[ModelKey.tokens] as Iterable)
+            .map<PangeaToken>(
+              (e) => PangeaToken.fromJson(e as Map<String, dynamic>),
+            )
+            .toList()
+            .cast<PangeaToken>()
+        : [];
     return Continuance(
       probability: json['probability'].toDouble(),
       level: json['level'],
@@ -122,7 +120,7 @@ class Continuance {
       wasClicked: json['clkd'] ?? false,
       hasInfo: json['has_info'] ?? false,
       gold: json['gold'] ?? false,
-      lemmas: lemmaInternal,
+      tokens: tokensInternal,
     );
   }
 
@@ -132,7 +130,7 @@ class Continuance {
     data['level'] = level;
     data['text'] = text;
     data['clkd'] = wasClicked;
-    data[ModelKey.lemma] = lemmas.map((e) => e.toJson()).toList();
+    data[ModelKey.tokens] = tokens.map((e) => e.toJson()).toList();
 
     if (!condensed) {
       data['description'] = description;

lib/pangea/models/lemma.dart

@@ -8,31 +8,20 @@ class Lemma {
 
   /// [saveVocab] true - whether to save the lemma to the user's vocabulary
   /// vocab that are not saved: emails, urls, numbers, punctuation, etc.
+  /// server handles this determination
   final bool saveVocab;
 
-  /// [pos] ex "v" - part of speech of the lemma
-  /// https://universaldependencies.org/u/pos/
-  final String pos;
-
-  /// [morph] ex {} - morphological features of the lemma
-  /// https://universaldependencies.org/u/feat/
-  final Map<String, dynamic> morph;
-
   Lemma({
     required this.text,
     required this.saveVocab,
     required this.form,
-    this.pos = '',
-    this.morph = const {},
   });
 
   factory Lemma.fromJson(Map<String, dynamic> json) {
     return Lemma(
       text: json['text'],
       saveVocab: json['save_vocab'] ?? json['saveVocab'] ?? false,
       form: json["form"] ?? json['text'],
-      pos: json['pos'] ?? '',
-      morph: json['morph'] ?? {},
     );
   }
 
@@ -41,8 +30,6 @@ class Lemma {
       'text': text,
       'save_vocab': saveVocab,
       'form': form,
-      'pos': pos,
-      'morph': morph,
     };
   }
 

lib/pangea/models/pangea_token_model.dart

@@ -1,63 +1,70 @@
 import 'dart:developer';
 
 import 'package:flutter/foundation.dart';
-import 'package:sentry_flutter/sentry_flutter.dart';
 
 import '../constants/model_keys.dart';
-import '../utils/error_handler.dart';
 import 'lemma.dart';
 
 class PangeaToken {
   PangeaTokenText text;
-  List<Lemma> lemmas;
+  Lemma lemma;
+
+  /// [pos] ex "VERB" - part of speech of the token
+  /// https://universaldependencies.org/u/pos/
+  final String pos;
+
+  /// [morph] ex {} - morphological features of the token
+  /// https://universaldependencies.org/u/feat/
+  final Map<String, dynamic> morph;
 
   PangeaToken({
     required this.text,
-    required this.lemmas,
+    required this.lemma,
+    required this.pos,
+    required this.morph,
   });
 
-  static getLemmas(String text, Iterable? json) {
+  static Lemma _getLemmas(String text, dynamic json) {
     if (json != null) {
-      return json
-          .map<Lemma>(
-            (e) => Lemma.fromJson(e as Map<String, dynamic>),
-          )
-          .toList()
-          .cast<Lemma>();
+      // July 24, 2024 - we're changing from a list to a single lemma and this is for backwards compatibility
+      // previously sent tokens have lists of lemmas
+      if (json is Iterable) {
+        return json
+                .map<Lemma>(
+                  (e) => Lemma.fromJson(e as Map<String, dynamic>),
+                )
+                .toList()
+                .cast<Lemma>()
+                .firstOrNull ??
+            Lemma(text: text, saveVocab: false, form: text);
+      } else {
+        return Lemma.fromJson(json);
+      }
     } else {
-      return [Lemma(text: text, saveVocab: false, form: text)];
+      // earlier still, we didn't have lemmas so this is for really old tokens
+      return Lemma(text: text, saveVocab: false, form: text);
     }
   }
 
   factory PangeaToken.fromJson(Map<String, dynamic> json) {
-    try {
-      final PangeaTokenText text =
-          PangeaTokenText.fromJson(json[_textKey] as Map<String, dynamic>);
-      return PangeaToken(
-        text: text,
-        lemmas: getLemmas(text.content, json[_lemmaKey]),
-      );
-    } catch (err, s) {
-      debugger(when: kDebugMode);
-      Sentry.addBreadcrumb(
-        Breadcrumb(
-          message: "PangeaToken.fromJson error",
-          data: {
-            "json": json,
-          },
-        ),
-      );
-      ErrorHandler.logError(e: err, s: s);
-      rethrow;
-    }
+    final PangeaTokenText text =
+        PangeaTokenText.fromJson(json[_textKey] as Map<String, dynamic>);
+    return PangeaToken(
+      text: text,
+      lemma: _getLemmas(text.content, json[_lemmaKey]),
+      pos: json['pos'] ?? '',
+      morph: json['morph'] ?? {},
+    );
   }
 
   static const String _textKey = "text";
   static const String _lemmaKey = ModelKey.lemma;
 
   Map<String, dynamic> toJson() => {
         _textKey: text.toJson(),
-        _lemmaKey: lemmas.map((e) => e.toJson()).toList(),
+        _lemmaKey: [lemma.toJson()],
+        'pos': pos,
+        'morph': morph,
       };
 
   int get end => text.offset + text.length;