diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..9fe10f6 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - "**" + workflow_dispatch: + inputs: + sdk_git_ref: + type: string + description: "Which git ref of the SDK to build" + +concurrency: + group: build-android-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + sdk: + name: "SDK" + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.sdk_git_ref || github.ref }} + + - name: "Install Java" + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Build core SDK + run: ./gradlew :rtvi-client-android:assembleRelease diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..10cfdbf --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.iml +.gradle +/local.properties +/.idea +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/README.md b/README.md new file mode 100644 index 0000000..0ac7369 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# Real-Time Voice Inference Android Client SDK + +[RTVI](https://github.com/rtvi-ai/) is an open standard for Real-Time Voice (and Video) Inference. + +This Android library contains the core components and types needed to set up an RTVI session. + +When building an RTVI application, you should use the transport-specific client library (see +[here](https://rtvi.mintlify.app/api-reference/transports/introduction) for available first-party +packages.) The base `VoiceClient` has no transport included. + +## Usage + +Add the following dependency to your `build.gradle` file: + +``` +ai.rtvi:rtvi-client-android:0.1.0 +``` + +Then instantiate the `VoiceClient` from your code, specifying the backend `baseUrl` and transport. + +```kotlin +val callbacks = object : VoiceEventCallbacks() { + + override fun onBackendError(message: String) { + Log.e(TAG, "Error from backend: $message") + } + + // ... +} + +val client = VoiceClient(baseUrl, transport, callbacks) + +client.start().withCallback { + // ... +} +``` + +`client.start()` (and other APIs) return a `Future`, which can give callbacks, or be awaited +using Kotlin Coroutines (`client.start().await()`). diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..e8f55f6 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,5 @@ +plugins { + alias(libs.plugins.android.library) apply false + alias(libs.plugins.jetbrains.kotlin.android) apply false + alias(libs.plugins.jetbrains.dokka) apply true +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..20e2a01 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,23 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. For more details, visit +# https://developer.android.com/r/tools/gradle-multi-project-decoupled-projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000..835b803 --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,25 @@ +[versions] +agp = "8.5.1" +kotlin = "2.0.0" +coreKtx = "1.13.1" +junit = "4.13.2" +junitVersion = "1.2.1" +appcompat = "1.7.0" +kotlinxSerializationJson = "1.7.1" +kotlinxSerializationPlugin = "2.0.0" +okhttp = "4.12.0" +dokka = "1.9.20" + +[libraries] +androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } +junit = { group = "junit", name = "junit", version.ref = "junit" } +androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } +androidx-appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" } +kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } +okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } + +[plugins] +android-library = { id = "com.android.library", version.ref = "agp" } +jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } +jetbrains-kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlinxSerializationPlugin" } +jetbrains-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..e708b1c Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..35c800a --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Aug 05 13:01:27 BST 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..4f906e0 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..107acd3 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/rtvi-client-android/.gitignore b/rtvi-client-android/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/rtvi-client-android/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/rtvi-client-android/build.gradle.kts b/rtvi-client-android/build.gradle.kts new file mode 100644 index 0000000..95d5173 --- /dev/null +++ b/rtvi-client-android/build.gradle.kts @@ -0,0 +1,48 @@ +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.jetbrains.kotlin.android) + alias(libs.plugins.jetbrains.kotlin.serialization) + alias(libs.plugins.jetbrains.dokka) + `maven-publish` +} + +android { + namespace = "ai.rtvi.client" + compileSdk = 34 + + defaultConfig { + minSdk = 24 + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + lint { + targetSdk = 35 + } + + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.appcompat) + implementation(libs.okhttp) + implementation(libs.kotlinx.serialization.json) + testImplementation(libs.junit) + androidTestImplementation(libs.androidx.junit) +} diff --git a/rtvi-client-android/proguard-rules.pro b/rtvi-client-android/proguard-rules.pro new file mode 100644 index 0000000..481bb43 --- /dev/null +++ b/rtvi-client-android/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/rtvi-client-android/src/main/AndroidManifest.xml b/rtvi-client-android/src/main/AndroidManifest.xml new file mode 100644 index 0000000..568741e --- /dev/null +++ b/rtvi-client-android/src/main/AndroidManifest.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt b/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt new file mode 100644 index 0000000..6d03e47 --- /dev/null +++ b/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt @@ -0,0 +1,493 @@ +package ai.rtvi.client + +import ai.rtvi.client.helper.RegisteredVoiceClient +import ai.rtvi.client.helper.VoiceClientHelper +import ai.rtvi.client.result.Future +import ai.rtvi.client.result.Promise +import ai.rtvi.client.result.Result +import ai.rtvi.client.result.VoiceError +import ai.rtvi.client.result.VoiceException +import ai.rtvi.client.result.resolvedPromiseErr +import ai.rtvi.client.result.withPromise +import ai.rtvi.client.result.withTimeout +import ai.rtvi.client.transport.AuthBundle +import ai.rtvi.client.transport.MsgClientToServer +import ai.rtvi.client.transport.MsgServerToClient +import ai.rtvi.client.transport.Transport +import ai.rtvi.client.transport.TransportContext +import ai.rtvi.client.transport.TransportFactory +import ai.rtvi.client.types.ActionDescription +import ai.rtvi.client.types.Config +import ai.rtvi.client.types.MediaDeviceId +import ai.rtvi.client.types.Option +import ai.rtvi.client.types.RegisteredHelper +import ai.rtvi.client.types.ServiceConfig +import ai.rtvi.client.types.ServiceConfigDescription +import ai.rtvi.client.types.Transcript +import ai.rtvi.client.types.TransportState +import ai.rtvi.client.types.Value +import ai.rtvi.client.utils.ConnectionBundle +import ai.rtvi.client.utils.JSON_INSTANCE +import ai.rtvi.client.utils.ThreadRef +import ai.rtvi.client.utils.post +import android.util.Log +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.json.jsonObject +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody + +/** + * An RTVI client. Connects to an RTVI backend and handles bidirectional audio and video + * streaming. + * + * The client must be cleaned up using the [release] method when it is no longer required. + * + * @param baseUrl URL of the RTVI backend. + * @param transport Transport for media streaming. + * @param callbacks Callbacks invoked when changes occur in the voice session. + * @param options Additional options for configuring the client and backend. + */ +@Suppress("unused") +open class VoiceClient( + private val baseUrl: String, + transport: TransportFactory, + callbacks: VoiceEventCallbacks, + private var options: VoiceClientOptions = VoiceClientOptions() +) { + companion object { + private const val TAG = "VoiceClient" + } + + /** + * The thread used by the VoiceClient for callbacks and other operations. + */ + val thread = ThreadRef.forCurrent() + + private val callbacks = CallbackInterceptor(object : VoiceEventCallbacks() { + override fun onBackendError(message: String) {} + + override fun onDisconnected() { + discardWaitingResponses() + connection?.ready?.resolveErr(VoiceError.OperationCancelled) + connection = null + } + }, callbacks) + + private val helpers = mutableMapOf() + + private val awaitingServerResponse = + mutableMapOf) -> Unit>() + + private inline fun handleResponse( + msg: MsgServerToClient, + action: ((Result) -> Unit) -> Unit + ) { + val id = msg.id ?: throw Exception("${msg.type} missing ID") + + val respondTo = awaitingServerResponse.remove(id) + ?: throw Exception("${msg.type}: no responder for $id") + + action(respondTo) + } + + private val transportCtx = object : TransportContext { + + override val options + get() = this@VoiceClient.options + + override val callbacks + get() = this@VoiceClient.callbacks + + override val thread = this@VoiceClient.thread + + override fun onMessage(msg: MsgServerToClient) = thread.runOnThread { + + Log.i(TAG, "onMessage($msg)") + + try { + when (msg.type) { + MsgServerToClient.Type.BotReady -> { + + val data = + JSON_INSTANCE.decodeFromJsonElement(msg.data) + + this@VoiceClient.transport.setState(TransportState.Ready) + + connection?.ready?.resolveOk(Unit) + + callbacks.onBotReady( + version = data.version, + config = data.config + ) + } + + MsgServerToClient.Type.Error -> { + val data = + JSON_INSTANCE.decodeFromJsonElement(msg.data) + callbacks.onBackendError(data.error) + } + + MsgServerToClient.Type.ErrorResponse -> { + handleResponse(msg) { respondTo -> + val data = + JSON_INSTANCE.decodeFromJsonElement( + msg.data + ) + respondTo(Result.Err(VoiceError.ErrorResponse(data.error))) + } + } + + MsgServerToClient.Type.ActionResponse, + MsgServerToClient.Type.DescribeActionsResponse, + MsgServerToClient.Type.DescribeConfigResponse, + MsgServerToClient.Type.GetOrUpdateConfigResponse -> { + handleResponse(msg) { respondTo -> + respondTo(Result.Ok(msg.data)) + } + } + + MsgServerToClient.Type.UserTranscription -> { + val data = JSON_INSTANCE.decodeFromJsonElement(msg.data) + callbacks.onUserTranscript(data) + } + + MsgServerToClient.Type.BotTranscription -> { + val text = (msg.data.jsonObject.get("text") as JsonPrimitive).content + callbacks.onBotTranscript(text) + } + + MsgServerToClient.Type.UserStartedSpeaking -> { + callbacks.onUserStartedSpeaking() + } + + MsgServerToClient.Type.UserStoppedSpeaking -> { + callbacks.onUserStoppedSpeaking() + } + + else -> { + + var match = false + + helpers.values + .filter { it.supportedMessages.contains(msg.type) } + .forEach { entry -> + match = true + entry.helper.handleMessage(msg) + } + + if (!match) { + Log.w(TAG, "Unexpected message type '${msg.type}'") + + callbacks.onGenericMessage(msg) + } + } + } + } catch (e: Exception) { + Log.e(TAG, "Exception while handling VoiceMessage", e) + } + } + } + + private val transport: Transport = transport.createTransport(transportCtx) + + private inner class Connection { + val ready = Promise(thread) + } + + private var connection: Connection? = null + + /** + * Initialize local media devices such as camera and microphone. + * + * @return A Future, representing the asynchronous result of this operation. + */ + fun initDevices(): Future = transport.initDevices() + + /** + * Initiate an RTVI session, connecting to the backend. + */ + fun start(): Future = thread.runOnThreadReturningFuture { + + if (connection != null) { + return@runOnThreadReturningFuture resolvedPromiseErr( + thread, + VoiceError.PreviousConnectionStillActive + ) + } + + transport.setState(TransportState.Authorizing) + + // Send POST request to the provided base_url to connect and start the bot + + val body = JSON_INSTANCE.encodeToString( + ConnectionBundle.serializer(), + ConnectionBundle( + services = options.services.associate { it.service to it.value }, + config = options.config + ) + ).toRequestBody("application/json".toMediaType()) + + val currentConnection = Connection().apply { connection = this } + + return@runOnThreadReturningFuture post( + thread = thread, + url = baseUrl, + body = body, + customHeaders = options.customHeaders + ) + .mapError { + VoiceError.FailedToFetchAuthBundle(it) + } + .chain { authBundle -> + if (currentConnection == connection) { + transport.connect(AuthBundle(authBundle)) + } else { + resolvedPromiseErr(thread, VoiceError.OperationCancelled) + } + } + .chain { currentConnection.ready } + .withTimeout(30000) + .withErrorCallback { + disconnect() + } + } + + /** + * Disconnect an active RTVI session. + * + * @return A Future, representing the asynchronous result of this operation. + */ + fun disconnect(): Future { + return transport.disconnect() + } + + /** + * Directly send a message to the bot via the transport. + */ + fun sendMessage(msg: MsgClientToServer) = transport.sendMessage(msg) + + private fun discardWaitingResponses() { + + thread.assertCurrent() + + awaitingServerResponse.values.forEach { + it(Result.Err(VoiceError.OperationCancelled)) + } + + awaitingServerResponse.clear() + } + + /** + * Registers a new helper with the client. + * + * @param service Target service for this helper + * @param helper Helper instance + */ + @Throws(VoiceException::class) + fun registerHelper(service: String, helper: E): E { + + thread.assertCurrent() + + if (helpers.containsKey(service)) { + throw VoiceException(VoiceError.OtherError("Helper targeting service '$service' already registered")) + } + + helper.registerVoiceClient(RegisteredVoiceClient(this, service)) + + val entry = RegisteredHelper( + helper = helper, + supportedMessages = HashSet(helper.getMessageTypes()) + ) + + helpers[service] = entry + + return helper + } + + /** + * Unregisters a helper from the client. + */ + @Throws(VoiceException::class) + fun unregisterHelper(service: String) { + + thread.assertCurrent() + + val entry = helpers.remove(service) + ?: throw VoiceException(VoiceError.OtherError("Helper targeting service '$service' not found")) + + entry.helper.unregisterVoiceClient() + } + + private inline fun sendWithResponse( + msg: MsgClientToServer, + crossinline filter: (M) -> R + ): Future = withPromise(thread) { promise -> + thread.runOnThread { + + awaitingServerResponse[msg.id] = { result -> + when (result) { + is Result.Err -> promise.resolveErr(result.error) + is Result.Ok -> { + val data = JSON_INSTANCE.decodeFromJsonElement(result.value) + promise.resolveOk(filter(data)) + } + } + } + + transport.sendMessage(msg).withErrorCallback { + awaitingServerResponse.remove(msg.id) + promise.resolveErr(it) + } + } + }.withTimeout(10000) + + /** + * Instruct a backend service to perform an action. + */ + fun action( + service: String, + action: String, + arguments: List