From e0d7db1a10cfd87f3bf3ffc58e054159552a5edf Mon Sep 17 00:00:00 2001 From: Max Holland Date: Thu, 19 Oct 2023 18:02:29 +0100 Subject: [PATCH] Support for VOD thumbnails --- clients/catalyst.go | 11 +++--- go.mod | 2 +- go.sum | 4 +-- task/upload.go | 83 +++++++++++++++++++++++---------------------- task/upload_test.go | 35 ++++++++++++++++++- 5 files changed, 85 insertions(+), 50 deletions(-) diff --git a/clients/catalyst.go b/clients/catalyst.go index e87bf29..1fa4e7a 100644 --- a/clients/catalyst.go +++ b/clients/catalyst.go @@ -59,11 +59,12 @@ type OutputLocation struct { } type OutputsRequest struct { - SourceMp4 bool `json:"source_mp4"` - HLS string `json:"hls"` - MP4 string `json:"mp4"` - FMP4 string `json:"fragmented_mp4"` - Clip string `json:"clip"` + SourceMp4 bool `json:"source_mp4"` + HLS string `json:"hls"` + MP4 string `json:"mp4"` + FMP4 string `json:"fragmented_mp4"` + Clip string `json:"clip"` + Thumbnails string `json:"thumbnails"` } type CatalystOptions struct { diff --git a/go.mod b/go.mod index 18e1f49..8a450dc 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/golang/glog v1.1.2 github.com/julienschmidt/httprouter v1.3.0 github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4 - github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0 + github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc github.com/livepeer/go-tools v0.3.2 github.com/livepeer/livepeer-data v0.7.5-0.20230927031152-b938ac1dc665 github.com/peterbourgon/ff v1.7.1 diff --git a/go.sum b/go.sum index cc71da2..b0b6e2f 100644 --- a/go.sum +++ b/go.sum @@ -281,8 +281,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4 h1:UfiMdEDGa88yqYD9+i1+ldAex9Kf1+3jbq+wBrmZccM= github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4/go.mod h1:Ybiub5AGDrDfvyh1MWdIa551LAwhx/6lSpbQlgb1W1Q= -github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0 h1:mVzmSN/dmYOtoaVhpfbDSF59quCLO6Dyvfm84z2RVqU= -github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw= +github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc h1:fN5jindUaI3PZXn6Ff8EXG8Vtvi4gxCja2fS8YI6Sw0= +github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw= github.com/livepeer/go-tools v0.3.2 h1:5pOUrOmkkGbbcWnpCt2yrSD6cD85G4GcpO4B25NpMJM= github.com/livepeer/go-tools v0.3.2/go.mod h1:qs31y68b3PQPmSr8nR8l5WQiIWI623z6pqOccqebjos= github.com/livepeer/livepeer-data v0.7.5-0.20230927031152-b938ac1dc665 h1:EXlI922Fsv9lyIw1LQ7pZN6slCuYya8NQrCFWN8INg4= diff --git a/task/upload.go b/task/upload.go index 6bb2ec8..5c7a6e4 100644 --- a/task/upload.go +++ b/task/upload.go @@ -219,12 +219,10 @@ func TaskTranscodeFile(tctx *TaskContext) (*TaskHandlerOutput, error) { getOutputLocations: func() ([]clients.OutputLocation, error) { _, outputLocation, err := outputLocations( params.Storage.URL, - isEnabled(params.Outputs.HLS.Path), - params.Outputs.HLS.Path, - isEnabled(params.Outputs.MP4.Path), - params.Outputs.MP4.Path, - isEnabled(params.Outputs.FMP4.Path), - params.Outputs.FMP4.Path, + outLoc(isEnabled(params.Outputs.HLS.Path), params.Outputs.HLS.Path), + outLoc(isEnabled(params.Outputs.MP4.Path), params.Outputs.MP4.Path), + outLoc(isEnabled(params.Outputs.FMP4.Path), params.Outputs.FMP4.Path), + outLocation{}, false, ) return outputLocation, err @@ -677,7 +675,11 @@ func uploadTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.Outpu } else { mp4 = OUTPUT_ONLY_SHORT } - outputNames, outputLocations, err := outputLocations(outURL, OUTPUT_ENABLED, playbackId, mp4, playbackId, "", "", !isEncryptionEnabled(*tctx.Task.Params.Upload)) + thumbsEnabled := "" + if tctx.Task.Params.Upload.Thumbnails { + thumbsEnabled = OUTPUT_ENABLED + } + outputNames, outputLocations, err := outputLocations(outURL, outLoc(OUTPUT_ENABLED, playbackId), outLoc(mp4, playbackId), outLocation{}, outLoc(thumbsEnabled, playbackId), !isEncryptionEnabled(*tctx.Task.Params.Upload)) if err != nil { return nil, nil, err } @@ -718,7 +720,7 @@ func clipTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.OutputL return nil, nil, fmt.Errorf("error parsing object store URL %w", err) } - outputNames, outputLocations, err := outputLocations(outURL, OUTPUT_ENABLED, playbackId, OUTPUT_ENABLED, playbackId, "", "", false) + outputNames, outputLocations, err := outputLocations(outURL, outLoc(OUTPUT_ENABLED, playbackId), outLoc(OUTPUT_ENABLED, playbackId), outLocation{}, outLocation{}, false) if err != nil { return nil, nil, err @@ -738,14 +740,24 @@ func clipTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.OutputL return outputNames, outputLocations, nil } +func outLoc(enabled, path string) outLocation { + return outLocation{ + enabled: enabled, + path: path, + } +} + +type outLocation struct { + enabled string + path string +} + func outputLocations( - outURL, + outURL string, hls, - hlsRelPath, mp4, - mp4RelPath, fmp4, - fmp4RelPath string, + thumbnails outLocation, sourceCopy bool, ) ([]OutputName, []clients.OutputLocation, error) { url, err := url.Parse(outURL) @@ -755,46 +767,35 @@ func outputLocations( names, locations := []OutputName{OutputNameOSPlaylistHLS, OutputNameEmpty}, []clients.OutputLocation{ - { - Type: "object_store", - URL: url.JoinPath(hlsRelPath).String(), - Outputs: &clients.OutputsRequest{ - HLS: hls, - }, - }, - { - Type: "object_store", - URL: url.JoinPath(mp4RelPath).String(), - Outputs: &clients.OutputsRequest{ - MP4: mp4, - }, - }, + outputLocation(url, hls.path, &clients.OutputsRequest{HLS: hls.enabled}), + outputLocation(url, mp4.path, &clients.OutputsRequest{MP4: mp4.enabled}), } - if fmp4 == OUTPUT_ENABLED { + if fmp4.enabled == OUTPUT_ENABLED { names, locations = append(names, OutputNameEmpty), - append(locations, clients.OutputLocation{ - Type: "object_store", - URL: url.JoinPath(fmp4RelPath).String(), - Outputs: &clients.OutputsRequest{ - FMP4: fmp4, - }, - }) + append(locations, outputLocation(url, fmp4.path, &clients.OutputsRequest{FMP4: fmp4.enabled})) } if sourceCopy { names, locations = append(names, OutputNameOSSourceMP4), - append(locations, clients.OutputLocation{ - Type: "object_store", - URL: url.JoinPath(videoFileName(hlsRelPath)).String(), - Outputs: &clients.OutputsRequest{ - SourceMp4: true, - }, - }) + append(locations, outputLocation(url, videoFileName(hls.path), &clients.OutputsRequest{SourceMp4: true})) + } + if thumbnails.enabled == OUTPUT_ENABLED { + names, locations = + append(names, OutputNameEmpty), + append(locations, outputLocation(url, thumbnails.path, &clients.OutputsRequest{Thumbnails: thumbnails.enabled})) } return names, locations, nil } +func outputLocation(url *url.URL, path string, request *clients.OutputsRequest) clients.OutputLocation { + return clients.OutputLocation{ + Type: "object_store", + URL: url.JoinPath(path).String(), + Outputs: request, + } +} + func catalystTaskAttemptID(task *api.Task) string { // Simplest way to identify unique runs of a given task. We should think of // something more sophisticated in the future. diff --git a/task/upload_test.go b/task/upload_test.go index 14bf93e..586fbda 100644 --- a/task/upload_test.go +++ b/task/upload_test.go @@ -160,6 +160,8 @@ func TestOutputLocations(t *testing.T) { hlsRelPath string mp4 string mp4RelPath string + thumbs string + thumbsRelPath string sourceCopy bool expectedOutputLocations []clients.OutputLocation hasError bool @@ -244,10 +246,41 @@ func TestOutputLocations(t *testing.T) { }, }, }, + { + name: "Thumbnails", + outURL: "s3+https://user:pass@host.com/outbucket", + hls: "enabled", + hlsRelPath: "video/hls", + thumbs: "enabled", + thumbsRelPath: "video/thumbs", + expectedOutputLocations: []clients.OutputLocation{ + { + Type: "object_store", + URL: "s3+https://user:pass@host.com/outbucket/video/hls", + Outputs: &clients.OutputsRequest{ + HLS: "enabled", + }, + }, + { + Type: "object_store", + URL: "s3+https://user:pass@host.com/outbucket", + Outputs: &clients.OutputsRequest{ + MP4: "", + }, + }, + { + Type: "object_store", + URL: "s3+https://user:pass@host.com/outbucket/video/thumbs", + Outputs: &clients.OutputsRequest{ + Thumbnails: "enabled", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, gotOutputLocations, err := outputLocations(tt.outURL, tt.hls, tt.hlsRelPath, tt.mp4, tt.mp4RelPath, "", "", tt.sourceCopy) + _, gotOutputLocations, err := outputLocations(tt.outURL, outLoc(tt.hls, tt.hlsRelPath), outLoc(tt.mp4, tt.mp4RelPath), outLocation{}, outLoc(tt.thumbs, tt.thumbsRelPath), tt.sourceCopy) if tt.hasError { require.Error(t, err)