Merge branch 'ai-video' into ai-orchestrator-metrics

cmd/livepeer/starter/starter.go

@@ -613,6 +613,18 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 					constraints[core.Capability_Upscale].Models[config.ModelID] = modelConstraint
 
 					n.SetBasePriceForCap("default", core.Capability_Upscale, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit))
+				case "audio-to-text":
+					_, ok := constraints[core.Capability_AudioToText]
+					if !ok {
+						aiCaps = append(aiCaps, core.Capability_AudioToText)
+						constraints[core.Capability_AudioToText] = &core.Constraints{
+							Models: make(map[string]*core.ModelConstraint),
+						}
+					}
+
+					constraints[core.Capability_AudioToText].Models[config.ModelID] = modelConstraint
+
+					n.SetBasePriceForCap("default", core.Capability_AudioToText, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit))
 				}
 
 				if len(aiCaps) > 0 {

common/util.go

@@ -25,6 +25,7 @@ import (
 	"github.com/jaypipes/ghw/pkg/pci"
 	"github.com/livepeer/go-livepeer/net"
 	ffmpeg "github.com/livepeer/lpms/ffmpeg"
+	"github.com/oapi-codegen/runtime/types"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc/peer"
 )
@@ -74,6 +75,8 @@ var (
 	ErrProfEncoder  = fmt.Errorf("unknown VideoProfile encoder for protobufs")
 	ErrProfName     = fmt.Errorf("unknown VideoProfile profile name")
 
+	ErrAudioDurationCalculation = fmt.Errorf("audio duration calculation failed")
+
 	ext2mime = map[string]string{
 		".ts":  "video/mp2t",
 		".mp4": "video/mp4",
@@ -530,3 +533,25 @@ func ParseEthAddr(strJsonKey string) (string, error) {
 	}
 	return "", errors.New("Error parsing address from keyfile")
 }
+
+// CalculateAudioDuration calculates audio file duration using the lpms/ffmpeg package.
+func CalculateAudioDuration(audio types.File) (int64, error) {
+	read, err := audio.Reader()
+	if err != nil {
+		return 0, err
+	}
+	defer read.Close()
+
+	bytearr, _ := audio.Bytes()
+	_, mediaFormat, err := ffmpeg.GetCodecInfoBytes(bytearr)
+	if err != nil {
+		return 0, errors.New("Error getting codec info")
+	}
+
+	duration := int64(mediaFormat.DurSecs)
+	if duration <= 0 {
+		return 0, ErrAudioDurationCalculation
+	}
+
+	return duration, nil
+}

core/ai.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -16,6 +17,7 @@ type AI interface {
 	ImageToImage(context.Context, worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error)
 	ImageToVideo(context.Context, worker.ImageToVideoMultipartRequestBody) (*worker.VideoResponse, error)
 	Upscale(context.Context, worker.UpscaleMultipartRequestBody) (*worker.ImageResponse, error)
+	AudioToText(context.Context, worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error)
 	Warm(context.Context, string, string, worker.RunnerEndpoint, worker.OptimizationFlags) error
 	Stop(context.Context) error
 	HasCapacity(pipeline, modelID string) bool
@@ -85,3 +87,19 @@ func ParseAIModelConfigs(config string) ([]AIModelConfig, error) {
 
 	return configs, nil
 }
+
+// parseStepsFromModelID parses the number of inference steps from the model ID suffix.
+func ParseStepsFromModelID(modelID *string, defaultSteps float64) float64 {
+	numInferenceSteps := defaultSteps
+
+	// Regular expression to find "_<number>step" pattern anywhere in the model ID.
+	stepPattern := regexp.MustCompile(`_(\d+)step`)
+	matches := stepPattern.FindStringSubmatch(*modelID)
+	if len(matches) == 2 {
+		if parsedSteps, err := strconv.Atoi(matches[1]); err == nil {
+			numInferenceSteps = float64(parsedSteps)
+		}
+	}
+
+	return numInferenceSteps
+}

core/capabilities.go

@@ -71,6 +71,7 @@ const (
 	Capability_ImageToImage
 	Capability_ImageToVideo
 	Capability_Upscale
+	Capability_AudioToText
 )
 
 var CapabilityNameLookup = map[Capability]string{
@@ -106,6 +107,7 @@ var CapabilityNameLookup = map[Capability]string{
 	Capability_ImageToImage:               "Image to image",
 	Capability_ImageToVideo:               "Image to video",
 	Capability_Upscale:                    "Upscale",
+	Capability_AudioToText:                "Audio to text",
 }
 
 var CapabilityTestLookup = map[Capability]CapabilityTest{
@@ -195,6 +197,7 @@ func OptionalCapabilities() []Capability {
 		Capability_ImageToImage,
 		Capability_ImageToVideo,
 		Capability_Upscale,
+		Capability_AudioToText,
 	}
 }
 

core/orchestrator.go

@@ -126,6 +126,10 @@ func (orch *orchestrator) Upscale(ctx context.Context, req worker.UpscaleMultipa
 	return orch.node.upscale(ctx, req)
 }
 
+func (orch *orchestrator) AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) {
+	return orch.node.AudioToText(ctx, req)
+}
+
 func (orch *orchestrator) ProcessPayment(ctx context.Context, payment net.Payment, manifestID ManifestID) error {
 	if orch.node == nil || orch.node.Recipient == nil {
 		return nil
@@ -945,6 +949,10 @@ func (n *LivepeerNode) upscale(ctx context.Context, req worker.UpscaleMultipartR
 	return n.AIWorker.Upscale(ctx, req)
 }
 
+func (n *LivepeerNode) AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) {
+	return n.AIWorker.AudioToText(ctx, req)
+}
+
 func (n *LivepeerNode) imageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) {
 	// We might support generating more than one video in the future (i.e. multiple input images/prompts)
 	numVideos := 1

go.mod

@@ -9,13 +9,13 @@ require (
 	github.com/getkin/kin-openapi v0.124.0
 	github.com/golang/glog v1.1.1
 	github.com/golang/mock v1.6.0
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.4
 	github.com/jaypipes/ghw v0.10.0
 	github.com/jaypipes/pcidb v1.0.0
-	github.com/livepeer/ai-worker v0.0.8
+	github.com/livepeer/ai-worker v0.1.0
 	github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b
 	github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18
-	github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69
+	github.com/livepeer/lpms v0.0.0-20240711175220-227325841434
 	github.com/livepeer/m3u8 v0.11.1
 	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/oapi-codegen/nethttp-middleware v1.0.1
@@ -32,7 +32,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	golang.org/x/net v0.25.0
 	google.golang.org/grpc v1.57.1
-	google.golang.org/protobuf v1.31.0
+	google.golang.org/protobuf v1.33.0
 	pgregory.net/rapid v1.1.0
 )
 
@@ -85,6 +85,7 @@ require (
 	github.com/go-openapi/swag v0.22.8 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/go-test/deep v1.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect

go.sum

@@ -251,8 +251,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
 github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
-github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
-github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -297,8 +297,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -531,16 +531,16 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n
 github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI=
 github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo=
 github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc=
-github.com/livepeer/ai-worker v0.0.8 h1:FAjYJgSOaZslA06Wb6MolYohI30IMIujDTB26nfw8YE=
-github.com/livepeer/ai-worker v0.0.8/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE=
+github.com/livepeer/ai-worker v0.1.0 h1:SJBZuxeK0vEzJPBzf5osdgVCxHYZt7ZKR2CvZ7Q7iog=
+github.com/livepeer/ai-worker v0.1.0/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE=
 github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE=
 github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw=
 github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA=
 github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded/go.mod h1:xkDdm+akniYxVT9KW1Y2Y7Hso6aW+rZObz3nrA9yTHw=
 github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 h1:4oH3NqV0NvcdS44Ld3zK2tO8IUiNozIggm74yobQeZg=
 github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18/go.mod h1:Jpf4jHK+fbWioBHRDRM1WadNT1qmY27g2YicTdO0Rtc=
-github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 h1:4A6geMb+HfxBBfaS24t8R3ddpEDfWbpx7NTQZMt5Fp4=
-github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM=
+github.com/livepeer/lpms v0.0.0-20240711175220-227325841434 h1:E7PKN6q/jMLapEV+eEwlwv87Xe5zacaVhvZ8T6AJR3c=
+github.com/livepeer/lpms v0.0.0-20240711175220-227325841434/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM=
 github.com/livepeer/m3u8 v0.11.1 h1:VkUJzfNTyjy9mqsgp5JPvouwna8wGZMvd/gAfT5FinU=
 github.com/livepeer/m3u8 v0.11.1/go.mod h1:IUqAtwWPAG2CblfQa4SVzTQoDcEMPyfNOaBSxqHMS04=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -1217,8 +1217,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

install_ffmpeg.sh

@@ -208,13 +208,13 @@ if [[ ! -e "$ROOT/ffmpeg/libavcodec/libavcodec.a" ]]; then
   ./configure ${TARGET_OS:-} $DISABLE_FFMPEG_COMPONENTS --fatal-warnings \
     --enable-libx264 --enable-gpl \
     --enable-protocol=rtmp,file,pipe \
-    --enable-muxer=mpegts,hls,segment,mp4,hevc,matroska,webm,null --enable-demuxer=flv,mpegts,mp4,mov,webm,matroska,image2 \
+    --enable-muxer=mp3,wav,flac,mpegts,hls,segment,mp4,hevc,matroska,webm,null --enable-demuxer=mp3,wav,flac,flv,mpegts,mp4,mov,webm,matroska,image2 \
     --enable-bsf=h264_mp4toannexb,aac_adtstoasc,h264_metadata,h264_redundant_pps,hevc_mp4toannexb,extract_extradata \
-    --enable-parser=aac,aac_latm,h264,hevc,vp8,vp9,png \
+    --enable-parser=mpegaudio,vorbis,opus,flac,aac,aac_latm,h264,hevc,vp8,vp9,png \
     --enable-filter=abuffer,buffer,abuffersink,buffersink,afifo,fifo,aformat,format \
     --enable-filter=aresample,asetnsamples,fps,scale,hwdownload,select,livepeer_dnn,signature \
-    --enable-encoder=aac,opus,libx264 \
-    --enable-decoder=aac,opus,h264,png \
+    --enable-encoder=mp3,vorbis,flac,aac,opus,libx264 \
+    --enable-decoder=mp3,vorbis,flac,aac,opus,h264,png \
     --extra-cflags="${EXTRA_CFLAGS} -I${ROOT}/compiled/include -I/usr/local/cuda/include" \
     --extra-ldflags="${EXTRA_FFMPEG_LDFLAGS} -L${ROOT}/compiled/lib -L/usr/local/cuda/lib64" \
     --prefix="$ROOT/compiled" \

server/ai_http.go

@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"image"
 	"net/http"
@@ -44,6 +43,7 @@ func startAIServer(lp lphttp) error {
 	lp.transRPC.Handle("/image-to-image", oapiReqValidator(lp.ImageToImage()))
 	lp.transRPC.Handle("/image-to-video", oapiReqValidator(lp.ImageToVideo()))
 	lp.transRPC.Handle("/upscale", oapiReqValidator(lp.Upscale()))
+	lp.transRPC.Handle("/audio-to-text", oapiReqValidator(lp.AudioToText()))
 
 	return nil
 }
@@ -134,6 +134,29 @@ func (h *lphttp) Upscale() http.Handler {
 	})
 }
 
+func (h *lphttp) AudioToText() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		orch := h.orchestrator
+
+		remoteAddr := getRemoteAddr(r)
+		ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr)
+
+		multiRdr, err := r.MultipartReader()
+		if err != nil {
+			respondWithError(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		var req worker.AudioToTextMultipartRequestBody
+		if err := runtime.BindMultipart(&req, *multiRdr); err != nil {
+			respondWithError(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		handleAIRequest(ctx, w, r, orch, req)
+	})
+}
+
 func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, orch Orchestrator, req interface{}) {
 	payment, err := getPayment(r.Header.Get(paymentHeader))
 	if err != nil {
@@ -151,15 +174,15 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 	var cap core.Capability
 	var pipeline string
 	var modelID string
-	var submitFn func(context.Context) (*worker.ImageResponse, error)
+	var submitFn func(context.Context) (interface{}, error)
 	var outPixels int64
 
 	switch v := req.(type) {
 	case worker.TextToImageJSONRequestBody:
 		pipeline = "text-to-image"
 		cap = core.Capability_TextToImage
 		modelID = *v.ModelId
-		submitFn = func(ctx context.Context) (*worker.ImageResponse, error) {
+		submitFn = func(ctx context.Context) (interface{}, error) {
 			return orch.TextToImage(ctx, v)
 		}
 
@@ -178,7 +201,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 		pipeline = "image-to-image"
 		cap = core.Capability_ImageToImage
 		modelID = *v.ModelId
-		submitFn = func(ctx context.Context) (*worker.ImageResponse, error) {
+		submitFn = func(ctx context.Context) (interface{}, error) {
 			return orch.ImageToImage(ctx, v)
 		}
 
@@ -197,7 +220,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 		pipeline = "upscale"
 		cap = core.Capability_Upscale
 		modelID = *v.ModelId
-		submitFn = func(ctx context.Context) (*worker.ImageResponse, error) {
+		submitFn = func(ctx context.Context) (interface{}, error) {
 			return orch.Upscale(ctx, v)
 		}
 
@@ -216,7 +239,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 		pipeline = "image-to-video"
 		cap = core.Capability_ImageToVideo
 		modelID = *v.ModelId
-		submitFn = func(ctx context.Context) (*worker.ImageResponse, error) {
+		submitFn = func(ctx context.Context) (interface{}, error) {
 			return orch.ImageToVideo(ctx, v)
 		}
 
@@ -233,6 +256,20 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 		frames := int64(25)
 
 		outPixels = height * width * int64(frames)
+	case worker.AudioToTextMultipartRequestBody:
+		pipeline = "audio-to-text"
+		cap = core.Capability_AudioToText
+		modelID = *v.ModelId
+		submitFn = func(ctx context.Context) (interface{}, error) {
+			return orch.AudioToText(ctx, v)
+		}
+
+		outPixels, err = common.CalculateAudioDuration(v.Audio)
+		if err != nil {
+			respondWithError(w, "Unable to calculate duration", http.StatusBadRequest)
+			return
+		}
+		outPixels *= 1000 // Convert to milliseconds
 	default:
 		respondWithError(w, "Unknown request type", http.StatusBadRequest)
 		return
@@ -272,7 +309,6 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 
 	start := time.Now()
 	resp, err := submitFn(ctx)
-	err = errors.New("fake error for testing")
 	if err != nil {
 		if monitor.Enabled {
 			monitor.AIProcessingError(err.Error(), pipeline, modelID, sender.Hex())
@@ -297,9 +333,14 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request
 		case worker.ImageToImageMultipartRequestBody:
 			latencyScore = CalculateImageToImageLatencyScore(took, v, outPixels)
 		case worker.ImageToVideoMultipartRequestBody:
-			latencyScore = CalculateImageToVideoLatencyScore(took, outPixels)
+			latencyScore = CalculateImageToVideoLatencyScore(took, v, outPixels)
 		case worker.UpscaleMultipartRequestBody:
-			latencyScore = CalculateUpscaleLatencyScore(took, outPixels)
+			latencyScore = CalculateUpscaleLatencyScore(took, v, outPixels)
+		case worker.AudioToTextMultipartRequestBody:
+			durationSeconds, err := common.CalculateAudioDuration(v.Audio)
+			if err == nil {
+				latencyScore = CalculateAudioToTextLatencyScore(took, durationSeconds)
+			}
 		}
 
 		var pricePerAIUnit float64