Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement BackgroundTaskCompletion and notification #290

Merged
merged 8 commits into from
Jun 5, 2023
Merged

Implement BackgroundTaskCompletion and notification #290

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
947c134
Implement BackgroundTaskCompletion
May 24, 2023
6ee01da
implement background alert notificaiton
May 24, 2023
60b4c34
change to use private partial sheet
May 24, 2023
ab5e70e
implement multilang notification
May 24, 2023
a571593
fix resource
May 29, 2023
c122971
fix notification message
Jun 2, 2023
a4d8889
format
Jun 2, 2023
5cb6706
Merge remote-tracking branch 'origin' into fix/background-task
Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Localization/en.lproj/Localizable.strings
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,3 +228,6 @@

/* No comment provided by engineer. */
"もう一度認識をお願いいたします。" = "Please try again.";

/* No comment provided by engineer. */
"30s以上バックグラウンド状態の場合、アプリの動作が停止します。" = "Background for more than 30 seconds will stop the app.";
44 changes: 21 additions & 23 deletions whisper-ios.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,10 @@
objects = {

/* Begin PBXBuildFile section */
1324459529AFA9370063A4D5 /* ggml-small.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = 1324459429AFA9370063A4D5 /* ggml-small.en.bin */; };
1324459929AFAA2A0063A4D5 /* ggml-small.multi.bin in Resources */ = {isa = PBXBuildFile; fileRef = 1324459829AFAA2A0063A4D5 /* ggml-small.multi.bin */; };
13516D992960369D005013C8 /* RecognizedSpeechMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 13516D982960369D005013C8 /* RecognizedSpeechMock.swift */; };
13516D9D2961A900005013C8 /* SideMenu.swift in Sources */ = {isa = PBXBuildFile; fileRef = 13516D9C2961A900005013C8 /* SideMenu.swift */; };
136E051F2965F57F00DEC112 /* ModelLoadMenuItems.swift in Sources */ = {isa = PBXBuildFile; fileRef = 136E051E2965F57F00DEC112 /* ModelLoadMenuItems.swift */; };
13BB395A29A3548500082D1F /* PartialSheet in Frameworks */ = {isa = PBXBuildFile; productRef = 13DF2F2E29A0848C0030E39A /* PartialSheet */; };
13BB395D29A3694200082D1F /* RecognitionSettingPane.swift in Sources */ = {isa = PBXBuildFile; fileRef = 13BB395C29A3694200082D1F /* RecognitionSettingPane.swift */; };
13BB396629A7AED600082D1F /* SafariServicesUI in Frameworks */ = {isa = PBXBuildFile; productRef = 5A91ACE62999514D004BE94A /* SafariServicesUI */; };
13BB396829A894D900082D1F /* RecognitionPresetPane.swift in Sources */ = {isa = PBXBuildFile; fileRef = 13BB396729A894D900082D1F /* RecognitionPresetPane.swift */; };
Expand Down Expand Up @@ -51,7 +49,6 @@
57EAB6ED296214D300BB59BE /* RecognizedSpeechData+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 57EAB6EB296214D300BB59BE /* RecognizedSpeechData+CoreDataProperties.swift */; };
5A2284CE29A79EA5008E848F /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = 5A2284D029A79EA5008E848F /* Localizable.strings */; };
5A2284D429A79F61008E848F /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = 5A2284D229A79F61008E848F /* InfoPlist.strings */; };
5A733C1929B10C59002B397C /* ggml-small.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = 5A733C1829B10C59002B397C /* ggml-small.en.bin */; };
5A8F54F52948CCDB00E87D25 /* whisper_iosApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5A8F54F42948CCDB00E87D25 /* whisper_iosApp.swift */; };
5A8F54F92948CCDD00E87D25 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 5A8F54F82948CCDD00E87D25 /* Assets.xcassets */; };
5A8F54FC2948CCDD00E87D25 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 5A8F54FB2948CCDD00E87D25 /* Preview Assets.xcassets */; };
Expand All @@ -67,6 +64,8 @@
5AADC50A29A245EE003AD311 /* FileDownloader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5AADC50929A245EE003AD311 /* FileDownloader.swift */; };
5AADC50C29A24623003AD311 /* LicenseView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5AADC50B29A24623003AD311 /* LicenseView.swift */; };
5AADC50E29A24636003AD311 /* AppInfoMenuItems.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5AADC50D29A24636003AD311 /* AppInfoMenuItems.swift */; };
DBCD3B652A1DE42200F081E6 /* PartialSheet in Frameworks */ = {isa = PBXBuildFile; productRef = DBCD3B642A1DE42200F081E6 /* PartialSheet */; };
DBCD3B692A24C98100F081E6 /* ggml-small.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = DBCD3B682A24C98100F081E6 /* ggml-small.en.bin */; };
/* End PBXBuildFile section */

/* Begin PBXContainerItemProxy section */
Expand All @@ -87,7 +86,6 @@
/* End PBXContainerItemProxy section */

/* Begin PBXFileReference section */
1324459429AFA9370063A4D5 /* ggml-small.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "ggml-small.en.bin"; path = "../../../../Downloads/ggml-small.en.bin"; sourceTree = "<group>"; };
1324459829AFAA2A0063A4D5 /* ggml-small.multi.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; path = "ggml-small.multi.bin"; sourceTree = "<group>"; };
13516D982960369D005013C8 /* RecognizedSpeechMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecognizedSpeechMock.swift; sourceTree = "<group>"; };
13516D9C2961A900005013C8 /* SideMenu.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SideMenu.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -136,7 +134,6 @@
5A2284D129A79EA9008E848F /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/Localizable.strings; sourceTree = "<group>"; };
5A2284D329A79F61008E848F /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/InfoPlist.strings; sourceTree = "<group>"; };
5A2284DF29ABB437008E848F /* ja */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ja; path = ja.lproj/InfoPlist.strings; sourceTree = "<group>"; };
5A733C1829B10C59002B397C /* ggml-small.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; path = "ggml-small.en.bin"; sourceTree = "<group>"; };
5A8F54F12948CCDB00E87D25 /* VoiScribe.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VoiScribe.app; sourceTree = BUILT_PRODUCTS_DIR; };
5A8F54F42948CCDB00E87D25 /* whisper_iosApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = whisper_iosApp.swift; sourceTree = "<group>"; };
5A8F54F82948CCDD00E87D25 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
Expand All @@ -155,15 +152,16 @@
5AADC50929A245EE003AD311 /* FileDownloader.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FileDownloader.swift; sourceTree = "<group>"; };
5AADC50B29A24623003AD311 /* LicenseView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LicenseView.swift; sourceTree = "<group>"; };
5AADC50D29A24636003AD311 /* AppInfoMenuItems.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppInfoMenuItems.swift; sourceTree = "<group>"; };
DBCD3B682A24C98100F081E6 /* ggml-small.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; path = "ggml-small.en.bin"; sourceTree = "<group>"; };
/* End PBXFileReference section */

/* Begin PBXFrameworksBuildPhase section */
5A8F54EE2948CCDB00E87D25 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
DBCD3B652A1DE42200F081E6 /* PartialSheet in Frameworks */,
13BB396629A7AED600082D1F /* SafariServicesUI in Frameworks */,
13BB395A29A3548500082D1F /* PartialSheet in Frameworks */,
57B26A2929649A7F007A7B9B /* DequeModule in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down Expand Up @@ -417,7 +415,7 @@
5A8F552C2948CEBD00E87D25 /* Resources */ = {
isa = PBXGroup;
children = (
5A733C1829B10C59002B397C /* ggml-small.en.bin */,
DBCD3B682A24C98100F081E6 /* ggml-small.en.bin */,
1324459829AFAA2A0063A4D5 /* ggml-small.multi.bin */,
572D0CA529A17F4800D62256 /* sample_ja.wav */,
572D0CA229A17BCA00D62256 /* sample_ja.csv */,
Expand Down Expand Up @@ -451,7 +449,7 @@
packageProductDependencies = (
57B26A2829649A7F007A7B9B /* DequeModule */,
5A91ACE62999514D004BE94A /* SafariServicesUI */,
13DF2F2E29A0848C0030E39A /* PartialSheet */,
DBCD3B642A1DE42200F081E6 /* PartialSheet */,
);
productName = "whisper-ios";
productReference = 5A8F54F12948CCDB00E87D25 /* VoiScribe.app */;
Expand Down Expand Up @@ -530,7 +528,7 @@
packageReferences = (
57B26A2729649A7E007A7B9B /* XCRemoteSwiftPackageReference "swift-collections" */,
5A91ACE52999514D004BE94A /* XCRemoteSwiftPackageReference "SafariServicesUI" */,
13DF2F2D29A0848C0030E39A /* XCRemoteSwiftPackageReference "PartialSheet" */,
DBCD3B632A1DE42200F081E6 /* XCRemoteSwiftPackageReference "PartialSheet" */,
);
productRefGroup = 5A8F54F22948CCDB00E87D25 /* Products */;
projectDirPath = "";
Expand All @@ -549,10 +547,10 @@
buildActionMask = 2147483647;
files = (
572D0CA429A17CF300D62256 /* sample_ja.csv in Resources */,
DBCD3B692A24C98100F081E6 /* ggml-small.en.bin in Resources */,
1324459929AFAA2A0063A4D5 /* ggml-small.multi.bin in Resources */,
572D0CA629A17F4800D62256 /* sample_ja.wav in Resources */,
5A8F54FC2948CCDD00E87D25 /* Preview Assets.xcassets in Resources */,
5A733C1929B10C59002B397C /* ggml-small.en.bin in Resources */,
5A8F54F92948CCDD00E87D25 /* Assets.xcassets in Resources */,
5A2284CE29A79EA5008E848F /* Localizable.strings in Resources */,
5A2284D429A79F61008E848F /* InfoPlist.strings in Resources */,
Expand Down Expand Up @@ -986,14 +984,6 @@
/* End XCConfigurationList section */

/* Begin XCRemoteSwiftPackageReference section */
13DF2F2D29A0848C0030E39A /* XCRemoteSwiftPackageReference "PartialSheet" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/AndreaMiotto/PartialSheet.git";
requirement = {
branch = master;
kind = branch;
};
};
57B26A2729649A7E007A7B9B /* XCRemoteSwiftPackageReference "swift-collections" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/apple/swift-collections.git";
Expand All @@ -1010,14 +1000,17 @@
minimumVersion = 0.1.0;
};
};
DBCD3B632A1DE42200F081E6 /* XCRemoteSwiftPackageReference "PartialSheet" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/openly-jp/PartialSheet";
requirement = {
branch = master;
kind = branch;
};
};
/* End XCRemoteSwiftPackageReference section */

/* Begin XCSwiftPackageProductDependency section */
13DF2F2E29A0848C0030E39A /* PartialSheet */ = {
isa = XCSwiftPackageProductDependency;
package = 13DF2F2D29A0848C0030E39A /* XCRemoteSwiftPackageReference "PartialSheet" */;
productName = PartialSheet;
};
57B26A2829649A7F007A7B9B /* DequeModule */ = {
isa = XCSwiftPackageProductDependency;
package = 57B26A2729649A7E007A7B9B /* XCRemoteSwiftPackageReference "swift-collections" */;
Expand All @@ -1028,6 +1021,11 @@
package = 5A91ACE52999514D004BE94A /* XCRemoteSwiftPackageReference "SafariServicesUI" */;
productName = SafariServicesUI;
};
DBCD3B642A1DE42200F081E6 /* PartialSheet */ = {
isa = XCSwiftPackageProductDependency;
package = DBCD3B632A1DE42200F081E6 /* XCRemoteSwiftPackageReference "PartialSheet" */;
productName = PartialSheet;
};
/* End XCSwiftPackageProductDependency section */

/* Begin XCVersionGroup section */
Expand Down
27 changes: 27 additions & 0 deletions whisper-ios/Models/Recognizer/WhisperRecognizer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import AVFoundation
import Dispatch
import Foundation
import SwiftUI

var numRecognitionTasks = 0

class WhisperRecognizer: Recognizer {
@Published var whisperModel: WhisperModel
Expand Down Expand Up @@ -114,8 +117,13 @@ class WhisperRecognizer: Recognizer {
serialDispatchQueue.async {
defer {
self.isRecognizing = false
numRecognitionTasks -= 1
}
let identifier = UIApplication.shared.beginBackgroundTask(expirationHandler: {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://developer.apple.com/documentation/uikit/app_and_environment/scenes/preparing_your_ui_to_run_in_the_background/extending_your_app_s_background_execution_time

一応これを見る感じ、中でも
UIApplication.shared.endBackgroundTask(identifier)を読んだ方がよさそうかも？

Logger.warning("background task expired")
})

numRecognitionTasks += 1
// prohibit user from changing model
self.isRecognizing = true

Expand Down Expand Up @@ -232,6 +240,7 @@ class WhisperRecognizer: Recognizer {
}
callback(recognizingSpeech)
}
UIApplication.shared.endBackgroundTask(identifier)
}
}
}
Expand Down Expand Up @@ -306,3 +315,21 @@ func newSegmentCallback(
)
}
}

func sendBackgroundAlertNotification() {
let BACKGROUND_ALERT_NOTIFICATION_IDENTIFIER = "background-alert-notification"
let BACKGROUND_ALERT_NOTIFICATION_TITLE = "VoiScribe"
let BACKGROUND_ALERT_NOTIFICATION_BODY = NSLocalizedString("30s以上バックグラウンド状態の場合、アプリの動作が停止します。", comment: "")

let backgroundAlertNotificationContent = UNMutableNotificationContent()
backgroundAlertNotificationContent.title = BACKGROUND_ALERT_NOTIFICATION_TITLE
backgroundAlertNotificationContent.body = BACKGROUND_ALERT_NOTIFICATION_BODY

let backgroundAlertNotificationTrigger = UNTimeIntervalNotificationTrigger(timeInterval: 1, repeats: false)
let backgroundAlertNotificationRequest = UNNotificationRequest(
identifier: BACKGROUND_ALERT_NOTIFICATION_IDENTIFIER,
content: backgroundAlertNotificationContent,
trigger: backgroundAlertNotificationTrigger
)
UNUserNotificationCenter.current().add(backgroundAlertNotificationRequest)
}
10 changes: 10 additions & 0 deletions whisper-ios/whisper_iosApp.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,17 @@ import SwiftUI

@main
struct WhisperTestApp: App {
@Environment(\.scenePhase) private var scenePhase
var body: some Scene {
WindowGroup {
StartView()
.onChange(of: scenePhase) { phase in
if phase == .background {
if numRecognitionTasks > 0 {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

厳密にいったらスレッドセーフじゃなさそうなのと、グローバル変数よりはRecognizerのプロパティのほうがいいかな、とか思ったけど、なおすのめんどうだなのでとりあえずコメントだけ！

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

numRecognitionTasksの書き換えはserialDispatchQueueの中でしか起きないので現状は安全ですが、確かにメインスレッドでも参照する以上、スレッドセーフであるべきですね。
Issueに切り出しておきます。

sendBackgroundAlertNotification()
}
}
}
}
}
}
Expand All @@ -27,6 +35,8 @@ struct StartView: View {
UserDefaults.standard.set(false, forKey: isDownloadingKey)
}
}
let notificationCenter = UNUserNotificationCenter.current()
notificationCenter.requestAuthorization(options: .alert, completionHandler: { _, _ in })
}

var body: some View {
Expand Down