feat: Audio messages new design #WPB-11723

app/build.gradle.kts

@@ -259,6 +259,7 @@ dependencies {
     implementation(libs.aboutLibraries.core)
     implementation(libs.aboutLibraries.ui)
     implementation(libs.compose.qr.code)
+    implementation(libs.audio.amplituda)
 
     // screenshot testing
     screenshotTestImplementation(libs.compose.ui.tooling)

app/src/main/kotlin/com/wire/android/di/AppModule.kt

@@ -38,6 +38,7 @@ import dagger.Provides
 import dagger.hilt.InstallIn
 import dagger.hilt.android.qualifiers.ApplicationContext
 import dagger.hilt.components.SingletonComponent
+import linc.com.amplituda.Amplituda
 import javax.inject.Qualifier
 import javax.inject.Singleton
 
@@ -84,6 +85,9 @@ object AppModule {
         }
     }
 
+    @Provides
+    fun provideAmplituda(appContext: Context): Amplituda = Amplituda(appContext)
+
     @Singleton
     @Provides
     fun provideCurrentTimestampProvider(): CurrentTimestampProvider = { System.currentTimeMillis() }

app/src/main/kotlin/com/wire/android/media/audiomessage/AudioState.kt

@@ -17,20 +17,24 @@
  */
 package com.wire.android.media.audiomessage
 
+import androidx.annotation.StringRes
+import com.wire.android.R
+
 data class AudioState(
     val audioMediaPlayingState: AudioMediaPlayingState,
     val currentPositionInMs: Int,
-    val totalTimeInMs: TotalTimeInMs
+    val totalTimeInMs: TotalTimeInMs,
+    val wavesMask: List<Int>
 ) {
     companion object {
-        val DEFAULT = AudioState(AudioMediaPlayingState.Stopped, 0, TotalTimeInMs.NotKnown)
+        val DEFAULT = AudioState(AudioMediaPlayingState.Stopped, 0, TotalTimeInMs.NotKnown, listOf())
     }
 
     // if the back-end returned the total time, we use that, in case it didn't we use what we get from
     // the [ConversationAudioMessagePlayer.kt] which will emit the time once the users play the audio.
     fun sanitizeTotalTime(otherClientTotalTime: Int): TotalTimeInMs {
         if (otherClientTotalTime != 0) {
-           return TotalTimeInMs.Known(otherClientTotalTime)
+            return TotalTimeInMs.Known(otherClientTotalTime)
         }
 
         return totalTimeInMs
@@ -43,6 +47,27 @@ data class AudioState(
     }
 }
 
+@Suppress("MagicNumber")
+enum class AudioSpeed(val value: Float, @StringRes val titleRes: Int) {
+    NORMAL(1f, R.string.audio_speed_1),
+    FAST(1.5f, R.string.audio_speed_1_5),
+    MAX(2f, R.string.audio_speed_2);
+
+    fun toggle(): AudioSpeed = when (this) {
+        NORMAL -> FAST
+        FAST -> MAX
+        MAX -> NORMAL
+    }
+
+    companion object {
+        fun fromFloat(speed: Float): AudioSpeed = when {
+            (speed < FAST.value) -> NORMAL
+            (speed < MAX.value) -> FAST
+            else -> MAX
+        }
+    }
+}
+
 sealed class AudioMediaPlayingState {
     object Playing : AudioMediaPlayingState()
     object Stopped : AudioMediaPlayingState()
@@ -75,6 +100,11 @@ sealed class AudioMediaPlayerStateUpdate(
         override val messageId: String,
         val totalTimeInMs: Int
     ) : AudioMediaPlayerStateUpdate(messageId)
+
+    data class WaveMaskUpdate(
+        override val messageId: String,
+        val waveMask: List<Int>
+    ) : AudioMediaPlayerStateUpdate(messageId)
 }
 
 sealed class RecordAudioMediaPlayerStateUpdate {
@@ -89,4 +119,8 @@ sealed class RecordAudioMediaPlayerStateUpdate {
     data class TotalTimeUpdate(
         val totalTimeInMs: Int
     ) : RecordAudioMediaPlayerStateUpdate()
+
+    data class WaveMaskUpdate(
+        val waveMask: List<Int>
+    ) : RecordAudioMediaPlayerStateUpdate()
 }

app/src/main/kotlin/com/wire/android/media/audiomessage/AudioWavesMaskHelper.kt

@@ -0,0 +1,81 @@
+/*
+ * Wire
+ * Copyright (C) 2024 Wire Swiss GmbH
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/.
+ */
+package com.wire.android.media.audiomessage
+
+import linc.com.amplituda.Amplituda
+import linc.com.amplituda.Cache
+import okio.Path
+import java.io.File
+import javax.inject.Inject
+import kotlin.math.roundToInt
+
+class AudioWavesMaskHelper @Inject constructor(
+    private val amplituda: Amplituda
+) {
+
+    companion object {
+        private const val WAVES_AMOUNT = 75
+        private const val WAVE_MAX = 32
+    }
+
+    fun getWaveMask(decodedAssetPath: Path): List<Int> = getWaveMask(File(decodedAssetPath.toString()))
+
+    fun getWaveMask(file: File): List<Int> = amplituda
+        .processAudio(file, Cache.withParams(Cache.REUSE))
+        .get()
+        .amplitudesAsList()
+        .averageWavesMask()
+        .equalizeWavesMask()
+
+    private fun List<Double>.equalizeWavesMask(): List<Int> {
+        if (this.isEmpty()) return listOf()
+
+        val divider = max() / (WAVE_MAX - 1)
+        return map { (it / divider).roundToInt() + 1 }
+    }
+
+    private fun List<Int>.averageWavesMask(): List<Double> {
+        val wavesSize = size
+        val sectionSize = (wavesSize.toFloat() / WAVES_AMOUNT).roundToInt()
+
+        if (wavesSize < WAVES_AMOUNT || sectionSize == 1) return map { it.toDouble() }
+
+        val averagedWaves = mutableListOf<Double>()
+        for (i in 0..<(wavesSize / sectionSize)) {
+            val startIndex = (i * sectionSize)
+            if (startIndex >= wavesSize) continue
+            val endIndex = (startIndex + sectionSize).coerceAtMost(wavesSize - 1)
+            averagedWaves.add(subList(startIndex, endIndex).averageInt())
+        }
+        return averagedWaves
+    }
+
+    private fun List<Int>.averageInt(): Double {
+        var sum = 0.0
+        var count = 0
+        for (element in this) {
+            sum += element
+            ++count
+        }
+        return if (count == 0) 0.0 else sum / count
+    }
+
+    fun clear() {
+        amplituda.clearCache()
+    }
+}

app/src/main/kotlin/com/wire/android/media/audiomessage/ConversationAudioMessagePlayer.kt

@@ -46,14 +46,17 @@ class ConversationAudioMessagePlayerProvider
 @Inject constructor(
     private val context: Context,
     private val audioMediaPlayer: MediaPlayer,
+    private val wavesMaskHelper: AudioWavesMaskHelper,
     @KaliumCoreLogic private val coreLogic: CoreLogic,
 ) {
     private var player: ConversationAudioMessagePlayer? = null
     private var usageCount: Int = 0
 
     @Synchronized
     fun provide(): ConversationAudioMessagePlayer {
-        val player = player ?: ConversationAudioMessagePlayer(context, audioMediaPlayer, coreLogic).also { player = it }
+        val player = player ?: ConversationAudioMessagePlayer(context, audioMediaPlayer, wavesMaskHelper, coreLogic).also {
+            player = it
+        }
         usageCount++
 
         return player
@@ -69,14 +72,16 @@ class ConversationAudioMessagePlayerProvider
     }
 }
 
+@Suppress("TooManyFunctions")
 class ConversationAudioMessagePlayer
 internal constructor(
     private val context: Context,
     private val audioMediaPlayer: MediaPlayer,
+    private val wavesMaskHelper: AudioWavesMaskHelper,
     @KaliumCoreLogic private val coreLogic: CoreLogic,
 ) {
     private companion object {
-        const val UPDATE_POSITION_INTERVAL_IN_MS = 1000L
+        const val UPDATE_POSITION_INTERVAL_IN_MS = 100L
     }
 
     init {
@@ -99,6 +104,12 @@ internal constructor(
             extraBufferCapacity = 1
         )
 
+    private val audioSpeedFlow = MutableSharedFlow<AudioSpeed>(
+        onBufferOverflow = BufferOverflow.DROP_OLDEST,
+        extraBufferCapacity = 1,
+        replay = 1
+    )
+
     // MediaPlayer API does not have any mechanism that would inform as about the currentPosition,
     // in a callback manner, therefore we need to create a timer manually that ticks every 1 second
     // and emits the current position
@@ -166,11 +177,22 @@ internal constructor(
                         )
                     }
                 }
+
+                is AudioMediaPlayerStateUpdate.WaveMaskUpdate -> {
+                    audioMessageStateHistory = audioMessageStateHistory.toMutableMap().apply {
+                        put(
+                            audioMessageStateUpdate.messageId,
+                            currentState.copy(wavesMask = audioMessageStateUpdate.waveMask)
+                        )
+                    }
+                }
             }
 
             audioMessageStateHistory
         }.onStart { emit(audioMessageStateHistory) }
 
+    val audioSpeed: Flow<AudioSpeed> = audioSpeedFlow.onStart { emit(AudioSpeed.NORMAL) }
+
     private var currentAudioMessageId: String? = null
 
     suspend fun playAudio(
@@ -190,6 +212,12 @@ internal constructor(
         }
     }
 
+    suspend fun setSpeed(speed: AudioSpeed) {
+        val currentParams = audioMediaPlayer.playbackParams
+        audioMediaPlayer.playbackParams = currentParams.setSpeed(speed.value)
+        updateSpeedFlow()
+    }
+
     private fun previouslyResumedPosition(requestedAudioMessageId: String): Int? {
         return audioMessageStateHistory[requestedAudioMessageId]?.run {
             if (audioMediaPlayingState == AudioMediaPlayingState.Completed) {
@@ -230,16 +258,10 @@ internal constructor(
                 if (currentAccountResult is CurrentSessionResult.Failure) return@launch
 
                 audioMessageStateUpdate.emit(
-                    AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(
-                        messageId,
-                        AudioMediaPlayingState.Fetching
-                    )
+                    AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(messageId, AudioMediaPlayingState.Fetching)
                 )
 
-                val assetMessage = coreLogic
-                    .getSessionScope((currentAccountResult as CurrentSessionResult.Success).accountInfo.userId)
-                    .messages
-                    .getAssetMessage(conversationId, messageId)
+                val assetMessage = getAssetMessage(currentAccountResult, conversationId, messageId)
 
                 when (val result = assetMessage.await()) {
                     is MessageAssetResult.Success -> {
@@ -253,38 +275,35 @@ internal constructor(
                         val isFetchedAudioCurrentlyQueuedToPlay = messageId == currentAudioMessageId
 
                         if (isFetchedAudioCurrentlyQueuedToPlay) {
-                            audioMediaPlayer.setDataSource(
-                                context,
-                                Uri.parse(result.decodedAssetPath.toString())
-                            )
+                            audioMediaPlayer.setDataSource(context, Uri.parse(result.decodedAssetPath.toString()))
                             audioMediaPlayer.prepare()
 
+                            audioMessageStateUpdate.emit(
+                                AudioMediaPlayerStateUpdate.WaveMaskUpdate(
+                                    messageId,
+                                    wavesMaskHelper.getWaveMask(result.decodedAssetPath)
+                                )
+                            )
+
                             if (position != null) audioMediaPlayer.seekTo(position)
 
                             audioMediaPlayer.start()
 
+                            updateSpeedFlow()
+
                             audioMessageStateUpdate.emit(
-                                AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(
-                                    messageId,
-                                    AudioMediaPlayingState.Playing
-                                )
+                                AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(messageId, AudioMediaPlayingState.Playing)
                             )
 
                             audioMessageStateUpdate.emit(
-                                AudioMediaPlayerStateUpdate.TotalTimeUpdate(
-                                    messageId,
-                                    audioMediaPlayer.duration
-                                )
+                                AudioMediaPlayerStateUpdate.TotalTimeUpdate(messageId, audioMediaPlayer.duration)
                             )
                         }
                     }
 
                     is MessageAssetResult.Failure -> {
                         audioMessageStateUpdate.emit(
-                            AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(
-                                messageId,
-                                AudioMediaPlayingState.Failed
-                            )
+                            AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(messageId, AudioMediaPlayingState.Failed)
                         )
                     }
                 }
@@ -306,8 +325,38 @@ internal constructor(
         seekToAudioPosition.emit(messageId to position)
     }
 
+    suspend fun fetchWavesMask(conversationId: ConversationId, messageId: String) {
+        val currentAccountResult = coreLogic.getGlobalScope().session.currentSession()
+        if (currentAccountResult is CurrentSessionResult.Failure) return
+
+        val result = coreLogic
+            .getSessionScope((currentAccountResult as CurrentSessionResult.Success).accountInfo.userId)
+            .messages
+            .getAssetMessage(conversationId, messageId)
+            .await()
+
+        if (result is MessageAssetResult.Success) {
+            audioMessageStateUpdate.emit(
+                AudioMediaPlayerStateUpdate.WaveMaskUpdate(
+                    messageId,
+                    wavesMaskHelper.getWaveMask(result.decodedAssetPath)
+                )
+            )
+        }
+    }
+
+    private suspend fun getAssetMessage(
+        currentAccountResult: CurrentSessionResult,
+        conversationId: ConversationId,
+        messageId: String
+    ) = coreLogic
+        .getSessionScope((currentAccountResult as CurrentSessionResult.Success).accountInfo.userId)
+        .messages
+        .getAssetMessage(conversationId, messageId)
+
     private suspend fun resumeAudio(messageId: String) {
         audioMediaPlayer.start()
+        updateSpeedFlow()
 
         audioMessageStateUpdate.emit(
             AudioMediaPlayerStateUpdate.AudioMediaPlayingStateUpdate(messageId, AudioMediaPlayingState.Playing)
@@ -330,7 +379,13 @@ internal constructor(
         )
     }
 
+    private suspend fun updateSpeedFlow() {
+        val currentSpeed = AudioSpeed.fromFloat(audioMediaPlayer.playbackParams.speed)
+        audioSpeedFlow.emit(currentSpeed)
+    }
+
     internal fun close() {
         audioMediaPlayer.reset()
+        wavesMaskHelper.clear()
     }
 }