diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 0000000..9fe10f6
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,37 @@
+name: CI
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - "**"
+ workflow_dispatch:
+ inputs:
+ sdk_git_ref:
+ type: string
+ description: "Which git ref of the SDK to build"
+
+concurrency:
+ group: build-android-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ sdk:
+ name: "SDK"
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.inputs.sdk_git_ref || github.ref }}
+
+ - name: "Install Java"
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: '17'
+
+ - name: Build core SDK
+ run: ./gradlew :rtvi-client-android:assembleRelease
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..10cfdbf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,10 @@
+*.iml
+.gradle
+/local.properties
+/.idea
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0ac7369
--- /dev/null
+++ b/README.md
@@ -0,0 +1,39 @@
+# Real-Time Voice Inference Android Client SDK
+
+[RTVI](https://github.com/rtvi-ai/) is an open standard for Real-Time Voice (and Video) Inference.
+
+This Android library contains the core components and types needed to set up an RTVI session.
+
+When building an RTVI application, you should use the transport-specific client library (see
+[here](https://rtvi.mintlify.app/api-reference/transports/introduction) for available first-party
+packages.) The base `VoiceClient` has no transport included.
+
+## Usage
+
+Add the following dependency to your `build.gradle` file:
+
+```
+ai.rtvi:rtvi-client-android:0.1.0
+```
+
+Then instantiate the `VoiceClient` from your code, specifying the backend `baseUrl` and transport.
+
+```kotlin
+val callbacks = object : VoiceEventCallbacks() {
+
+ override fun onBackendError(message: String) {
+ Log.e(TAG, "Error from backend: $message")
+ }
+
+ // ...
+}
+
+val client = VoiceClient(baseUrl, transport, callbacks)
+
+client.start().withCallback {
+ // ...
+}
+```
+
+`client.start()` (and other APIs) return a `Future`, which can give callbacks, or be awaited
+using Kotlin Coroutines (`client.start().await()`).
diff --git a/build.gradle.kts b/build.gradle.kts
new file mode 100644
index 0000000..e8f55f6
--- /dev/null
+++ b/build.gradle.kts
@@ -0,0 +1,5 @@
+plugins {
+ alias(libs.plugins.android.library) apply false
+ alias(libs.plugins.jetbrains.kotlin.android) apply false
+ alias(libs.plugins.jetbrains.dokka) apply true
+}
diff --git a/gradle.properties b/gradle.properties
new file mode 100644
index 0000000..20e2a01
--- /dev/null
+++ b/gradle.properties
@@ -0,0 +1,23 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. For more details, visit
+# https://developer.android.com/r/tools/gradle-multi-project-decoupled-projects
+# org.gradle.parallel=true
+# AndroidX package structure to make it clearer which packages are bundled with the
+# Android operating system, and which are packaged with your app's APK
+# https://developer.android.com/topic/libraries/support-library/androidx-rn
+android.useAndroidX=true
+# Kotlin code style for this project: "official" or "obsolete":
+kotlin.code.style=official
+# Enables namespacing of each library's R class so that its R class includes only the
+# resources declared in the library itself and none from the library's dependencies,
+# thereby reducing the size of the R class for that library
+android.nonTransitiveRClass=true
\ No newline at end of file
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
new file mode 100644
index 0000000..835b803
--- /dev/null
+++ b/gradle/libs.versions.toml
@@ -0,0 +1,25 @@
+[versions]
+agp = "8.5.1"
+kotlin = "2.0.0"
+coreKtx = "1.13.1"
+junit = "4.13.2"
+junitVersion = "1.2.1"
+appcompat = "1.7.0"
+kotlinxSerializationJson = "1.7.1"
+kotlinxSerializationPlugin = "2.0.0"
+okhttp = "4.12.0"
+dokka = "1.9.20"
+
+[libraries]
+androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
+junit = { group = "junit", name = "junit", version.ref = "junit" }
+androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" }
+androidx-appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" }
+kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
+okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
+
+[plugins]
+android-library = { id = "com.android.library", version.ref = "agp" }
+jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
+jetbrains-kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlinxSerializationPlugin" }
+jetbrains-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" }
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 0000000..e708b1c
Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 0000000..35c800a
--- /dev/null
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,6 @@
+#Mon Aug 05 13:01:27 BST 2024
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
new file mode 100755
index 0000000..4f906e0
--- /dev/null
+++ b/gradlew
@@ -0,0 +1,185 @@
+#!/usr/bin/env sh
+
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+##
+## Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`"/$link"
+ fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+ echo "$*"
+}
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+ CYGWIN* )
+ cygwin=true
+ ;;
+ Darwin* )
+ darwin=true
+ ;;
+ MINGW* )
+ msys=true
+ ;;
+ NONSTOP* )
+ nonstop=true
+ ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD="java"
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+ MAX_FD_LIMIT=`ulimit -H -n`
+ if [ $? -eq 0 ] ; then
+ if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+ MAX_FD="$MAX_FD_LIMIT"
+ fi
+ ulimit -n $MAX_FD
+ if [ $? -ne 0 ] ; then
+ warn "Could not set maximum file descriptor limit: $MAX_FD"
+ fi
+ else
+ warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+ fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+ GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
+ APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+ CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+ JAVACMD=`cygpath --unix "$JAVACMD"`
+
+ # We build the pattern for arguments to be converted via cygpath
+ ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+ SEP=""
+ for dir in $ROOTDIRSRAW ; do
+ ROOTDIRS="$ROOTDIRS$SEP$dir"
+ SEP="|"
+ done
+ OURCYGPATTERN="(^($ROOTDIRS))"
+ # Add a user-defined pattern to the cygpath arguments
+ if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+ OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+ fi
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ i=0
+ for arg in "$@" ; do
+ CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+ CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
+
+ if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
+ eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+ else
+ eval `echo args$i`="\"$arg\""
+ fi
+ i=`expr $i + 1`
+ done
+ case $i in
+ 0) set -- ;;
+ 1) set -- "$args0" ;;
+ 2) set -- "$args0" "$args1" ;;
+ 3) set -- "$args0" "$args1" "$args2" ;;
+ 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+ 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+ 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+ 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+ 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+ 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+ esac
+fi
+
+# Escape application args
+save () {
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+ echo " "
+}
+APP_ARGS=`save "$@"`
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+exec "$JAVACMD" "$@"
diff --git a/gradlew.bat b/gradlew.bat
new file mode 100644
index 0000000..107acd3
--- /dev/null
+++ b/gradlew.bat
@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/rtvi-client-android/.gitignore b/rtvi-client-android/.gitignore
new file mode 100644
index 0000000..42afabf
--- /dev/null
+++ b/rtvi-client-android/.gitignore
@@ -0,0 +1 @@
+/build
\ No newline at end of file
diff --git a/rtvi-client-android/build.gradle.kts b/rtvi-client-android/build.gradle.kts
new file mode 100644
index 0000000..95d5173
--- /dev/null
+++ b/rtvi-client-android/build.gradle.kts
@@ -0,0 +1,48 @@
+plugins {
+ alias(libs.plugins.android.library)
+ alias(libs.plugins.jetbrains.kotlin.android)
+ alias(libs.plugins.jetbrains.kotlin.serialization)
+ alias(libs.plugins.jetbrains.dokka)
+ `maven-publish`
+}
+
+android {
+ namespace = "ai.rtvi.client"
+ compileSdk = 34
+
+ defaultConfig {
+ minSdk = 24
+ }
+
+ buildTypes {
+ release {
+ isMinifyEnabled = false
+ proguardFiles(
+ getDefaultProguardFile("proguard-android-optimize.txt"),
+ "proguard-rules.pro"
+ )
+ }
+ }
+
+ compileOptions {
+ sourceCompatibility = JavaVersion.VERSION_1_8
+ targetCompatibility = JavaVersion.VERSION_1_8
+ }
+
+ lint {
+ targetSdk = 35
+ }
+
+ kotlinOptions {
+ jvmTarget = "1.8"
+ }
+}
+
+dependencies {
+ implementation(libs.androidx.core.ktx)
+ implementation(libs.androidx.appcompat)
+ implementation(libs.okhttp)
+ implementation(libs.kotlinx.serialization.json)
+ testImplementation(libs.junit)
+ androidTestImplementation(libs.androidx.junit)
+}
diff --git a/rtvi-client-android/proguard-rules.pro b/rtvi-client-android/proguard-rules.pro
new file mode 100644
index 0000000..481bb43
--- /dev/null
+++ b/rtvi-client-android/proguard-rules.pro
@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+# http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+# public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile
\ No newline at end of file
diff --git a/rtvi-client-android/src/main/AndroidManifest.xml b/rtvi-client-android/src/main/AndroidManifest.xml
new file mode 100644
index 0000000..568741e
--- /dev/null
+++ b/rtvi-client-android/src/main/AndroidManifest.xml
@@ -0,0 +1,2 @@
+
+
\ No newline at end of file
diff --git a/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt b/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt
new file mode 100644
index 0000000..6d03e47
--- /dev/null
+++ b/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt
@@ -0,0 +1,493 @@
+package ai.rtvi.client
+
+import ai.rtvi.client.helper.RegisteredVoiceClient
+import ai.rtvi.client.helper.VoiceClientHelper
+import ai.rtvi.client.result.Future
+import ai.rtvi.client.result.Promise
+import ai.rtvi.client.result.Result
+import ai.rtvi.client.result.VoiceError
+import ai.rtvi.client.result.VoiceException
+import ai.rtvi.client.result.resolvedPromiseErr
+import ai.rtvi.client.result.withPromise
+import ai.rtvi.client.result.withTimeout
+import ai.rtvi.client.transport.AuthBundle
+import ai.rtvi.client.transport.MsgClientToServer
+import ai.rtvi.client.transport.MsgServerToClient
+import ai.rtvi.client.transport.Transport
+import ai.rtvi.client.transport.TransportContext
+import ai.rtvi.client.transport.TransportFactory
+import ai.rtvi.client.types.ActionDescription
+import ai.rtvi.client.types.Config
+import ai.rtvi.client.types.MediaDeviceId
+import ai.rtvi.client.types.Option
+import ai.rtvi.client.types.RegisteredHelper
+import ai.rtvi.client.types.ServiceConfig
+import ai.rtvi.client.types.ServiceConfigDescription
+import ai.rtvi.client.types.Transcript
+import ai.rtvi.client.types.TransportState
+import ai.rtvi.client.types.Value
+import ai.rtvi.client.utils.ConnectionBundle
+import ai.rtvi.client.utils.JSON_INSTANCE
+import ai.rtvi.client.utils.ThreadRef
+import ai.rtvi.client.utils.post
+import android.util.Log
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.json.jsonObject
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.RequestBody.Companion.toRequestBody
+
+/**
+ * An RTVI client. Connects to an RTVI backend and handles bidirectional audio and video
+ * streaming.
+ *
+ * The client must be cleaned up using the [release] method when it is no longer required.
+ *
+ * @param baseUrl URL of the RTVI backend.
+ * @param transport Transport for media streaming.
+ * @param callbacks Callbacks invoked when changes occur in the voice session.
+ * @param options Additional options for configuring the client and backend.
+ */
+@Suppress("unused")
+open class VoiceClient(
+ private val baseUrl: String,
+ transport: TransportFactory,
+ callbacks: VoiceEventCallbacks,
+ private var options: VoiceClientOptions = VoiceClientOptions()
+) {
+ companion object {
+ private const val TAG = "VoiceClient"
+ }
+
+ /**
+ * The thread used by the VoiceClient for callbacks and other operations.
+ */
+ val thread = ThreadRef.forCurrent()
+
+ private val callbacks = CallbackInterceptor(object : VoiceEventCallbacks() {
+ override fun onBackendError(message: String) {}
+
+ override fun onDisconnected() {
+ discardWaitingResponses()
+ connection?.ready?.resolveErr(VoiceError.OperationCancelled)
+ connection = null
+ }
+ }, callbacks)
+
+ private val helpers = mutableMapOf()
+
+ private val awaitingServerResponse =
+ mutableMapOf) -> Unit>()
+
+ private inline fun handleResponse(
+ msg: MsgServerToClient,
+ action: ((Result) -> Unit) -> Unit
+ ) {
+ val id = msg.id ?: throw Exception("${msg.type} missing ID")
+
+ val respondTo = awaitingServerResponse.remove(id)
+ ?: throw Exception("${msg.type}: no responder for $id")
+
+ action(respondTo)
+ }
+
+ private val transportCtx = object : TransportContext {
+
+ override val options
+ get() = this@VoiceClient.options
+
+ override val callbacks
+ get() = this@VoiceClient.callbacks
+
+ override val thread = this@VoiceClient.thread
+
+ override fun onMessage(msg: MsgServerToClient) = thread.runOnThread {
+
+ Log.i(TAG, "onMessage($msg)")
+
+ try {
+ when (msg.type) {
+ MsgServerToClient.Type.BotReady -> {
+
+ val data =
+ JSON_INSTANCE.decodeFromJsonElement(msg.data)
+
+ this@VoiceClient.transport.setState(TransportState.Ready)
+
+ connection?.ready?.resolveOk(Unit)
+
+ callbacks.onBotReady(
+ version = data.version,
+ config = data.config
+ )
+ }
+
+ MsgServerToClient.Type.Error -> {
+ val data =
+ JSON_INSTANCE.decodeFromJsonElement(msg.data)
+ callbacks.onBackendError(data.error)
+ }
+
+ MsgServerToClient.Type.ErrorResponse -> {
+ handleResponse(msg) { respondTo ->
+ val data =
+ JSON_INSTANCE.decodeFromJsonElement(
+ msg.data
+ )
+ respondTo(Result.Err(VoiceError.ErrorResponse(data.error)))
+ }
+ }
+
+ MsgServerToClient.Type.ActionResponse,
+ MsgServerToClient.Type.DescribeActionsResponse,
+ MsgServerToClient.Type.DescribeConfigResponse,
+ MsgServerToClient.Type.GetOrUpdateConfigResponse -> {
+ handleResponse(msg) { respondTo ->
+ respondTo(Result.Ok(msg.data))
+ }
+ }
+
+ MsgServerToClient.Type.UserTranscription -> {
+ val data = JSON_INSTANCE.decodeFromJsonElement(msg.data)
+ callbacks.onUserTranscript(data)
+ }
+
+ MsgServerToClient.Type.BotTranscription -> {
+ val text = (msg.data.jsonObject.get("text") as JsonPrimitive).content
+ callbacks.onBotTranscript(text)
+ }
+
+ MsgServerToClient.Type.UserStartedSpeaking -> {
+ callbacks.onUserStartedSpeaking()
+ }
+
+ MsgServerToClient.Type.UserStoppedSpeaking -> {
+ callbacks.onUserStoppedSpeaking()
+ }
+
+ else -> {
+
+ var match = false
+
+ helpers.values
+ .filter { it.supportedMessages.contains(msg.type) }
+ .forEach { entry ->
+ match = true
+ entry.helper.handleMessage(msg)
+ }
+
+ if (!match) {
+ Log.w(TAG, "Unexpected message type '${msg.type}'")
+
+ callbacks.onGenericMessage(msg)
+ }
+ }
+ }
+ } catch (e: Exception) {
+ Log.e(TAG, "Exception while handling VoiceMessage", e)
+ }
+ }
+ }
+
+ private val transport: Transport = transport.createTransport(transportCtx)
+
+ private inner class Connection {
+ val ready = Promise(thread)
+ }
+
+ private var connection: Connection? = null
+
+ /**
+ * Initialize local media devices such as camera and microphone.
+ *
+ * @return A Future, representing the asynchronous result of this operation.
+ */
+ fun initDevices(): Future = transport.initDevices()
+
+ /**
+ * Initiate an RTVI session, connecting to the backend.
+ */
+ fun start(): Future = thread.runOnThreadReturningFuture {
+
+ if (connection != null) {
+ return@runOnThreadReturningFuture resolvedPromiseErr(
+ thread,
+ VoiceError.PreviousConnectionStillActive
+ )
+ }
+
+ transport.setState(TransportState.Authorizing)
+
+ // Send POST request to the provided base_url to connect and start the bot
+
+ val body = JSON_INSTANCE.encodeToString(
+ ConnectionBundle.serializer(),
+ ConnectionBundle(
+ services = options.services.associate { it.service to it.value },
+ config = options.config
+ )
+ ).toRequestBody("application/json".toMediaType())
+
+ val currentConnection = Connection().apply { connection = this }
+
+ return@runOnThreadReturningFuture post(
+ thread = thread,
+ url = baseUrl,
+ body = body,
+ customHeaders = options.customHeaders
+ )
+ .mapError {
+ VoiceError.FailedToFetchAuthBundle(it)
+ }
+ .chain { authBundle ->
+ if (currentConnection == connection) {
+ transport.connect(AuthBundle(authBundle))
+ } else {
+ resolvedPromiseErr(thread, VoiceError.OperationCancelled)
+ }
+ }
+ .chain { currentConnection.ready }
+ .withTimeout(30000)
+ .withErrorCallback {
+ disconnect()
+ }
+ }
+
+ /**
+ * Disconnect an active RTVI session.
+ *
+ * @return A Future, representing the asynchronous result of this operation.
+ */
+ fun disconnect(): Future {
+ return transport.disconnect()
+ }
+
+ /**
+ * Directly send a message to the bot via the transport.
+ */
+ fun sendMessage(msg: MsgClientToServer) = transport.sendMessage(msg)
+
+ private fun discardWaitingResponses() {
+
+ thread.assertCurrent()
+
+ awaitingServerResponse.values.forEach {
+ it(Result.Err(VoiceError.OperationCancelled))
+ }
+
+ awaitingServerResponse.clear()
+ }
+
+ /**
+ * Registers a new helper with the client.
+ *
+ * @param service Target service for this helper
+ * @param helper Helper instance
+ */
+ @Throws(VoiceException::class)
+ fun registerHelper(service: String, helper: E): E {
+
+ thread.assertCurrent()
+
+ if (helpers.containsKey(service)) {
+ throw VoiceException(VoiceError.OtherError("Helper targeting service '$service' already registered"))
+ }
+
+ helper.registerVoiceClient(RegisteredVoiceClient(this, service))
+
+ val entry = RegisteredHelper(
+ helper = helper,
+ supportedMessages = HashSet(helper.getMessageTypes())
+ )
+
+ helpers[service] = entry
+
+ return helper
+ }
+
+ /**
+ * Unregisters a helper from the client.
+ */
+ @Throws(VoiceException::class)
+ fun unregisterHelper(service: String) {
+
+ thread.assertCurrent()
+
+ val entry = helpers.remove(service)
+ ?: throw VoiceException(VoiceError.OtherError("Helper targeting service '$service' not found"))
+
+ entry.helper.unregisterVoiceClient()
+ }
+
+ private inline fun sendWithResponse(
+ msg: MsgClientToServer,
+ crossinline filter: (M) -> R
+ ): Future = withPromise(thread) { promise ->
+ thread.runOnThread {
+
+ awaitingServerResponse[msg.id] = { result ->
+ when (result) {
+ is Result.Err -> promise.resolveErr(result.error)
+ is Result.Ok -> {
+ val data = JSON_INSTANCE.decodeFromJsonElement(result.value)
+ promise.resolveOk(filter(data))
+ }
+ }
+ }
+
+ transport.sendMessage(msg).withErrorCallback {
+ awaitingServerResponse.remove(msg.id)
+ promise.resolveErr(it)
+ }
+ }
+ }.withTimeout(10000)
+
+ /**
+ * Instruct a backend service to perform an action.
+ */
+ fun action(
+ service: String,
+ action: String,
+ arguments: List