Localize TTS system announcements to the reading language

integration_test/app_test.dart

@@ -1,9 +1,11 @@
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:cached_network_image/cached_network_image.dart';
 import 'package:firebase_auth/firebase_auth.dart';
 import 'package:firebase_core/firebase_core.dart';
 import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
 import 'package:provider/provider.dart';
@@ -99,6 +101,19 @@ void main() {
 
   setUpAll(() async {
     await initializeFirebaseEmulators();
+    // These fake users have no production avatars. Seed the image cache so
+    // account sign-out never races an external request or a delayed 404.
+    final avatar = await rootBundle.load('assets/providers/twitch.png');
+    for (final channel in [
+      Channel(Providers.twitch, channelId, 'e2estreamer'),
+      Channel(Providers.kick, kickChannelId, 'E2EKick'),
+    ]) {
+      await CachedNetworkImageProvider.defaultCacheManager.putFile(
+        channel.profilePictureUrl.toString(),
+        avatar.buffer.asUint8List(avatar.offsetInBytes, avatar.lengthInBytes),
+        fileExtension: 'png',
+      );
+    }
     callables = await CallableStub.start(callableHost, callablePort);
   });
 
@@ -500,17 +515,6 @@ void main() {
       final kickTile = find.widgetWithText(ListTile, 'E2EKick');
       await pumpUntil(tester, kickTile);
       expect(find.widgetWithText(ListTile, 'e2estreamer'), findsOneWidget);
-      // let the kick avatar finish loading (the emulated channel has none),
-      // or its error is reported once the tile is gone.
-      final avatar =
-          Channel(Providers.kick, kickChannelId, 'E2EKick').profilePicture;
-      final end = DateTime.now().add(const Duration(seconds: 30));
-      do {
-        await tester.pump(const Duration(milliseconds: 250));
-      } while (PaintingBinding.instance.imageCache
-              .statusForKey(avatar)
-              .pending &&
-          DateTime.now().isBefore(end));
       await tester.tap(
         find.descendant(of: kickTile, matching: find.byTooltip('Sign out')),
       );

lib/models/tts.dart

@@ -77,9 +77,9 @@ enum NeuralTtsFallback {
 /// 3. and by the platform's voices too when the neural engine fails, for
 ///    that message only (unless it was unsaid in the meantime).
 ///
-/// Chat is in its channel's language ([languageFor]). Alerts and system
-/// messages are made of the app's strings, so they are in its language, and
-/// always read with the chosen voice.
+/// Chat is in its channel's language ([languageFor]). Generated mode/backlog
+/// announcements use the current reading language when translated; other alerts
+/// use the app's language. Announcements and alerts use the chosen voice.
 ///
 /// A loaded neural model takes a few hundred megabytes, so it's released
 /// ([releaseNeuralModels]) when text to speech is turned off, after
@@ -227,6 +227,7 @@ class TtsModel extends ChangeNotifier {
     bool includeAuthorPrelude = false,
   }) {
     if (model is ChatMessage) {
+      l10n = _localizationsForLanguage(l10n, languageFor(model).languageCode);
       final text = model.tokenized
           .where(
             (token) =>
@@ -348,11 +349,30 @@ class TtsModel extends ChangeNotifier {
     }
   }
 
+  AppLocalizations _localizationsForLanguage(
+    AppLocalizations fallback,
+    String language,
+  ) {
+    final available = AppLocalizations.supportedLocales;
+    final match = LanguageTag.bestMatch(
+      language,
+      available.map((locale) => locale.toLanguageTag()),
+    );
+    if (match == null) return fallback;
+    return lookupAppLocalizations(
+      available.firstWhere((locale) => locale.toLanguageTag() == match),
+    );
+  }
+
+  AppLocalizations _announcementLocalizations(AppLocalizations fallback) =>
+      _localizationsForLanguage(fallback, readingLanguage.languageCode);
+
   void setMode(AppLocalizations localizations, TtsMode mode) {
     if (_mode == mode) {
       return;
     }
     _mode = mode;
+    localizations = _announcementLocalizations(localizations);
     switch (mode) {
       case TtsMode.alertsOnly:
         say(
@@ -744,12 +764,14 @@ class TtsModel extends ChangeNotifier {
       // the system voices read chat in its channel's language, when known;
       // otherwise, and for alerts, they keep their default.
       final systemLanguage =
-          model is ChatMessage ? _knownLanguageFor(model) : null;
+          model is ChatMessage
+              ? _knownLanguageFor(model)?.languageCode
+              : localizations.localeName.replaceAll('_', '-');
       await _systemEngine.speak(
         vocalization,
         rate: _speed,
         pitch: _pitch,
-        language: systemLanguage?.languageCode,
+        language: systemLanguage,
       );
     } catch (e, st) {
       _reportError(e, st);
@@ -863,6 +885,7 @@ class TtsModel extends ChangeNotifier {
   });
 
   void _turnOffForBacklog(AppLocalizations localizations) {
+    localizations = _announcementLocalizations(localizations);
     _pending.clear();
     _mode = TtsMode.disabled;
     say(

test/components/image/cached_network_image_test.dart

@@ -165,8 +165,8 @@ void main() {
       );
       await tester.pump();
       await _expectAnimation(tester);
-      // The 36px button's vertical padding constrains its icon to 20px.
-      expect(tester.getSize(find.byType(CrossFadeImage)), const Size(36, 20));
+      // The 36px button's 2px inset on each side leaves a 32px icon.
+      expect(tester.getSize(find.byType(CrossFadeImage)), const Size(36, 32));
       expect(requests, hasLength(1));
       expect(tester.takeException(), isNull);
       await tester.pumpWidget(const SizedBox());

test/models/tts_neural_engine_test.dart

@@ -23,6 +23,7 @@ void main() {
   setUpAll(() => initializeDateFormatting('en'));
 
   final l10n = lookupAppLocalizations(const Locale('en'));
+  final spanish = lookupAppLocalizations(const Locale('es'));
   late SpeechLog log;
   late FakeTtsEngine system;
   late FakeTtsEngine neural;
@@ -66,15 +67,17 @@ void main() {
     await pumpEventQueue();
 
     expect(neural.spoken, [
-      'Text to speech enabled',
-      'user1 said: konnichiwa',
-      'user2 said: hola',
+      spanish.textToSpeechEnabled,
+      lookupAppLocalizations(
+        const Locale('ja'),
+      ).saidMessage('user1', 'konnichiwa'),
+      spanish.saidMessage('user2', 'hola'),
     ]);
     expect(neural.utterances.map((u) => u.language), [
-      'en',
+      'es',
       'ja',
       'es',
-    ], reason: 'alerts are in the app language');
+    ], reason: 'announcements follow the reading language');
     expect(neural.utterances.last.rate, 0.8);
     expect(system.spoken, isEmpty);
   });
@@ -88,8 +91,8 @@ void main() {
       model.say(l10n, chat('second', 2));
       await pumpEventQueue();
 
-      expect(system.spoken, ['user1 said: first']);
-      expect(neural.spoken.last, 'user2 said: second');
+      expect(system.spoken, [spanish.saidMessage('user1', 'first')]);
+      expect(neural.spoken.last, spanish.saidMessage('user2', 'second'));
       expect(errors.single, isA<PlatformException>());
 
       // but not a message unsaid in the meantime.
@@ -124,9 +127,9 @@ void main() {
 
     await drain();
     expect(log.events, [
-      'neural: Text to speech enabled',
+      'neural: ${spanish.textToSpeechEnabled}',
       'neural dispose',
-      'next: user1 said: first',
+      'next: ${spanish.saidMessage('user1', 'first')}',
     ]);
     expect(log.maxSpeaking, 1);
 
@@ -146,7 +149,7 @@ void main() {
 
     expect(model.mode, TtsMode.disabled);
     expect(neural.stopCount, greaterThan(0));
-    expect(neural.spoken.last, l10n.textToSpeechTooManyMessages);
+    expect(neural.spoken.last, spanish.textToSpeechTooManyMessages);
     expect(log.maxSpeaking, 1);
   });
 

test/models/tts_system_localization_test.dart

@@ -0,0 +1,176 @@
+import 'package:flutter/widgets.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:rtchat/l10n/app_localizations.dart';
+import 'package:rtchat/models/channels.dart';
+import 'package:rtchat/models/messages/message.dart';
+import 'package:rtchat/models/messages/twitch/user.dart';
+import 'package:rtchat/models/tts.dart';
+import 'package:rtchat/models/tts/language.dart';
+import 'package:rtchat/models/views.dart';
+
+import '../fakes/fake_tts_engine.dart';
+import 'tts_model_test.dart' show createMessage;
+
+void main() {
+  TestWidgetsFlutterBinding.ensureInitialized();
+  final english = lookupAppLocalizations(const Locale('en'));
+  final german = lookupAppLocalizations(const Locale('de'));
+
+  test(
+    'mode announcements use reading-language text and a matching system voice',
+    () async {
+      final engine = FakeTtsEngine();
+      final model = TtsModel.fromJson({'language': 'de'}, engine: engine);
+      addTearDown(model.dispose);
+      for (final mode in [
+        TtsMode.enabled,
+        TtsMode.alertsOnly,
+        TtsMode.disabled,
+      ]) {
+        model.setMode(english, mode);
+        await pumpEventQueue();
+      }
+      expect(engine.spoken, [
+        german.textToSpeechEnabled,
+        german.alertsEnabled,
+        german.textToSpeechDisabled,
+      ]);
+      expect(engine.utterances.map((u) => u.language), ['de', 'de', 'de']);
+    },
+  );
+
+  test('every app locale can announce in its reading language', () async {
+    for (final locale in AppLocalizations.supportedLocales) {
+      final translated = lookupAppLocalizations(locale);
+      final engine = FakeTtsEngine();
+      final model = TtsModel.fromJson({
+        'language': locale.toLanguageTag(),
+      }, engine: engine);
+      addTearDown(model.dispose);
+      model.setMode(english, TtsMode.enabled);
+      await pumpEventQueue();
+      expect(
+        engine.spoken.single,
+        translated.textToSpeechEnabled,
+        reason: locale.toLanguageTag(),
+      );
+      expect(
+        engine.utterances.single.language,
+        // The speech router canonicalizes Norwegian to Bokmål.
+        locale.languageCode == 'no'
+            ? 'nb'
+            : translated.localeName.replaceAll('_', '-'),
+        reason: locale.toLanguageTag(),
+      );
+    }
+  });
+
+  test(
+    'announcements follow the primary stream language instead of the UI',
+    () async {
+      final engine = FakeTtsEngine();
+      final model = TtsModel.fromJson(
+        {'language': 'en'},
+        engine: engine,
+        lookUpStreamLanguage: (_) async => 'ja',
+      );
+      addTearDown(model.dispose);
+      model.update(
+        ViewsModel()..openChannel(Channel(Providers.twitch, '1', 'stream')),
+      );
+      await pumpEventQueue();
+      model.setMode(english, TtsMode.enabled);
+      await pumpEventQueue();
+      expect(
+        engine.spoken.single,
+        lookupAppLocalizations(const Locale('ja')).textToSpeechEnabled,
+      );
+      expect(engine.utterances.single.language, 'ja');
+    },
+  );
+
+  test(
+    'regional language tags select their translated parent locale',
+    () async {
+      final engine = FakeTtsEngine();
+      final model = TtsModel.fromJson({'language': 'pt-BR'}, engine: engine);
+      addTearDown(model.dispose);
+      model.setMode(english, TtsMode.enabled);
+      await pumpEventQueue();
+      expect(
+        engine.spoken.single,
+        lookupAppLocalizations(const Locale('pt')).textToSpeechEnabled,
+      );
+      expect(engine.utterances.single.language, 'pt');
+    },
+  );
+
+  test(
+    'untranslated languages keep the UI text and its matching voice',
+    () async {
+      final engine = FakeTtsEngine();
+      final model = TtsModel.fromJson({'language': 'eo'}, engine: engine);
+      addTearDown(model.dispose);
+      model.setMode(german, TtsMode.enabled);
+      await pumpEventQueue();
+      expect(engine.spoken.single, german.textToSpeechEnabled);
+      expect(engine.utterances.single.language, 'de');
+    },
+  );
+
+  test('backlog announcement uses the same localized speech policy', () async {
+    final engine = FakeTtsEngine()..holdSpeech = true;
+    final model = TtsModel.fromJson({'language': 'de'}, engine: engine);
+    addTearDown(() {
+      engine.finishSpeaking();
+      model.dispose();
+    });
+    model.setMode(english, TtsMode.enabled);
+    await pumpEventQueue();
+    for (var i = 0; i < 25; i++) {
+      model.say(
+        english,
+        createMessage(
+          'message $i',
+          messageId: 'localized-$i',
+          author: TwitchUserModel(userId: '$i', login: 'user$i'),
+        ),
+      );
+    }
+    await pumpEventQueue();
+    expect(model.mode, TtsMode.disabled);
+    expect(engine.spoken.last, german.textToSpeechTooManyMessages);
+    expect(engine.utterances.last.language, 'de');
+  });
+  test(
+    'said follows each message channel while names and chat text stay intact',
+    () async {
+      final model = TtsModel.fromJson({
+        'language': 'en',
+      }, engine: FakeTtsEngine());
+      addTearDown(model.dispose);
+      for (final language in ['de', 'fr', 'es', 'ja']) {
+        final channel = Channel(Providers.twitch, language, 'stream');
+        model.setChannelLanguage(channel, Language(language));
+        final message = createMessage(
+          'Keep THIS text unchanged!',
+          messageId: 'prelude-$language',
+          author: const TwitchUserModel(
+            userId: '1',
+            login: 'user',
+            displayName: 'Viewer_Name',
+          ),
+        )..channel = channel;
+        final translated = lookupAppLocalizations(Locale(language));
+        expect(
+          model.getVocalization(english, message, includeAuthorPrelude: true),
+          translated.saidMessage('Viewer_Name', 'Keep THIS text unchanged!'),
+        );
+        expect(
+          model.getVocalization(english, message, includeAuthorPrelude: false),
+          'Keep THIS text unchanged!',
+        );
+      }
+    },
+  );
+}

test/models/tts_voice_selection_test.dart

@@ -69,7 +69,12 @@ void main() {
 
     expect(neural.utterances.map((u) => u.language), ['ja', 'pt', 'en']);
     expect(system.utterances.map((u) => (u.text, u.language)), [
-      ('user1 said: ni hao', 'zh'),
+      (
+        lookupAppLocalizations(
+          const Locale('zh'),
+        ).saidMessage('user1', 'ni hao'),
+        'zh',
+      ),
     ]);
   });