Skip to content

Commit

Permalink
Support for VOD thumbnails
Browse files Browse the repository at this point in the history
  • Loading branch information
@mjh1
mjh1 committed Oct 20, 2023
1 parent d8fa8b6 commit e0d7db1
Show file tree
Hide file tree
Showing 5 changed files with 85 additions and 50 deletions.
11 changes: 6 additions & 5 deletions clients/catalyst.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,12 @@ type OutputLocation struct {
}

type OutputsRequest struct {
SourceMp4 bool `json:"source_mp4"`
HLS string `json:"hls"`
MP4 string `json:"mp4"`
FMP4 string `json:"fragmented_mp4"`
Clip string `json:"clip"`
SourceMp4 bool `json:"source_mp4"`
HLS string `json:"hls"`
MP4 string `json:"mp4"`
FMP4 string `json:"fragmented_mp4"`
Clip string `json:"clip"`
Thumbnails string `json:"thumbnails"`
}

type CatalystOptions struct {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/golang/glog v1.1.2
github.com/julienschmidt/httprouter v1.3.0
github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4
github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0
github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc
github.com/livepeer/go-tools v0.3.2
github.com/livepeer/livepeer-data v0.7.5-0.20230927031152-b938ac1dc665
github.com/peterbourgon/ff v1.7.1
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n
github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo=
github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4 h1:UfiMdEDGa88yqYD9+i1+ldAex9Kf1+3jbq+wBrmZccM=
github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4/go.mod h1:Ybiub5AGDrDfvyh1MWdIa551LAwhx/6lSpbQlgb1W1Q=
github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0 h1:mVzmSN/dmYOtoaVhpfbDSF59quCLO6Dyvfm84z2RVqU=
github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw=
github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc h1:fN5jindUaI3PZXn6Ff8EXG8Vtvi4gxCja2fS8YI6Sw0=
github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw=
github.com/livepeer/go-tools v0.3.2 h1:5pOUrOmkkGbbcWnpCt2yrSD6cD85G4GcpO4B25NpMJM=
github.com/livepeer/go-tools v0.3.2/go.mod h1:qs31y68b3PQPmSr8nR8l5WQiIWI623z6pqOccqebjos=
github.com/livepeer/livepeer-data v0.7.5-0.20230927031152-b938ac1dc665 h1:EXlI922Fsv9lyIw1LQ7pZN6slCuYya8NQrCFWN8INg4=
Expand Down
83 changes: 42 additions & 41 deletions task/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,12 +219,10 @@ func TaskTranscodeFile(tctx *TaskContext) (*TaskHandlerOutput, error) {
getOutputLocations: func() ([]clients.OutputLocation, error) {
_, outputLocation, err := outputLocations(
params.Storage.URL,
isEnabled(params.Outputs.HLS.Path),
params.Outputs.HLS.Path,
isEnabled(params.Outputs.MP4.Path),
params.Outputs.MP4.Path,
isEnabled(params.Outputs.FMP4.Path),
params.Outputs.FMP4.Path,
outLoc(isEnabled(params.Outputs.HLS.Path), params.Outputs.HLS.Path),
outLoc(isEnabled(params.Outputs.MP4.Path), params.Outputs.MP4.Path),
outLoc(isEnabled(params.Outputs.FMP4.Path), params.Outputs.FMP4.Path),
outLocation{},

Check warning on line 225 in task/upload.go

View check run for this annotation

Codecov / codecov/patch

task/upload.go#L222-L225 

Added lines #L222 - L225 were not covered by tests
false,
)
return outputLocation, err
Expand Down Expand Up @@ -677,7 +675,11 @@ func uploadTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.Outpu
} else {
mp4 = OUTPUT_ONLY_SHORT
}
outputNames, outputLocations, err := outputLocations(outURL, OUTPUT_ENABLED, playbackId, mp4, playbackId, "", "", !isEncryptionEnabled(*tctx.Task.Params.Upload))
thumbsEnabled := ""
if tctx.Task.Params.Upload.Thumbnails {
thumbsEnabled = OUTPUT_ENABLED
}
outputNames, outputLocations, err := outputLocations(outURL, outLoc(OUTPUT_ENABLED, playbackId), outLoc(mp4, playbackId), outLocation{}, outLoc(thumbsEnabled, playbackId), !isEncryptionEnabled(*tctx.Task.Params.Upload))

Check warning on line 682 in task/upload.go

View check run for this annotation

Codecov / codecov/patch

task/upload.go#L678-L682 

Added lines #L678 - L682 were not covered by tests
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -718,7 +720,7 @@ func clipTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.OutputL
return nil, nil, fmt.Errorf("error parsing object store URL %w", err)
}

outputNames, outputLocations, err := outputLocations(outURL, OUTPUT_ENABLED, playbackId, OUTPUT_ENABLED, playbackId, "", "", false)
outputNames, outputLocations, err := outputLocations(outURL, outLoc(OUTPUT_ENABLED, playbackId), outLoc(OUTPUT_ENABLED, playbackId), outLocation{}, outLocation{}, false)

Check warning on line 723 in task/upload.go

View check run for this annotation

Codecov / codecov/patch

task/upload.go#L723 

Added line #L723 was not covered by tests

if err != nil {
return nil, nil, err
Expand All @@ -738,14 +740,24 @@ func clipTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.OutputL
return outputNames, outputLocations, nil
}

func outLoc(enabled, path string) outLocation {
return outLocation{
enabled: enabled,
path: path,
}
}

type outLocation struct {
enabled string
path string
}

func outputLocations(
outURL,
outURL string,
hls,
hlsRelPath,
mp4,
mp4RelPath,
fmp4,
fmp4RelPath string,
thumbnails outLocation,
sourceCopy bool,
) ([]OutputName, []clients.OutputLocation, error) {
url, err := url.Parse(outURL)
Expand All @@ -755,46 +767,35 @@ func outputLocations(
names, locations :=
[]OutputName{OutputNameOSPlaylistHLS, OutputNameEmpty},
[]clients.OutputLocation{
{
Type: "object_store",
URL: url.JoinPath(hlsRelPath).String(),
Outputs: &clients.OutputsRequest{
HLS: hls,
},
},
{
Type: "object_store",
URL: url.JoinPath(mp4RelPath).String(),
Outputs: &clients.OutputsRequest{
MP4: mp4,
},
},
outputLocation(url, hls.path, &clients.OutputsRequest{HLS: hls.enabled}),
outputLocation(url, mp4.path, &clients.OutputsRequest{MP4: mp4.enabled}),
}
if fmp4 == OUTPUT_ENABLED {
if fmp4.enabled == OUTPUT_ENABLED {
names, locations =
append(names, OutputNameEmpty),
append(locations, clients.OutputLocation{
Type: "object_store",
URL: url.JoinPath(fmp4RelPath).String(),
Outputs: &clients.OutputsRequest{
FMP4: fmp4,
},
})
append(locations, outputLocation(url, fmp4.path, &clients.OutputsRequest{FMP4: fmp4.enabled}))

Check warning on line 776 in task/upload.go

View check run for this annotation

Codecov / codecov/patch

task/upload.go#L776 

Added line #L776 was not covered by tests
}
if sourceCopy {
names, locations =
append(names, OutputNameOSSourceMP4),
append(locations, clients.OutputLocation{
Type: "object_store",
URL: url.JoinPath(videoFileName(hlsRelPath)).String(),
Outputs: &clients.OutputsRequest{
SourceMp4: true,
},
})
append(locations, outputLocation(url, videoFileName(hls.path), &clients.OutputsRequest{SourceMp4: true}))
}
if thumbnails.enabled == OUTPUT_ENABLED {
names, locations =
append(names, OutputNameEmpty),
append(locations, outputLocation(url, thumbnails.path, &clients.OutputsRequest{Thumbnails: thumbnails.enabled}))
}
return names, locations, nil
}

func outputLocation(url *url.URL, path string, request *clients.OutputsRequest) clients.OutputLocation {
return clients.OutputLocation{
Type: "object_store",
URL: url.JoinPath(path).String(),
Outputs: request,
}
}

func catalystTaskAttemptID(task *api.Task) string {
// Simplest way to identify unique runs of a given task. We should think of
// something more sophisticated in the future.
Expand Down
35 changes: 34 additions & 1 deletion task/upload_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,8 @@ func TestOutputLocations(t *testing.T) {
hlsRelPath string
mp4 string
mp4RelPath string
thumbs string
thumbsRelPath string
sourceCopy bool
expectedOutputLocations []clients.OutputLocation
hasError bool
Expand Down Expand Up @@ -244,10 +246,41 @@ func TestOutputLocations(t *testing.T) {
},
},
},
{
name: "Thumbnails",
outURL: "s3+https://user:[email protected]/outbucket",
hls: "enabled",
hlsRelPath: "video/hls",
thumbs: "enabled",
thumbsRelPath: "video/thumbs",
expectedOutputLocations: []clients.OutputLocation{
{
Type: "object_store",
URL: "s3+https://user:[email protected]/outbucket/video/hls",
Outputs: &clients.OutputsRequest{
HLS: "enabled",
},
},
{
Type: "object_store",
URL: "s3+https://user:[email protected]/outbucket",
Outputs: &clients.OutputsRequest{
MP4: "",
},
},
{
Type: "object_store",
URL: "s3+https://user:[email protected]/outbucket/video/thumbs",
Outputs: &clients.OutputsRequest{
Thumbnails: "enabled",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, gotOutputLocations, err := outputLocations(tt.outURL, tt.hls, tt.hlsRelPath, tt.mp4, tt.mp4RelPath, "", "", tt.sourceCopy)
_, gotOutputLocations, err := outputLocations(tt.outURL, outLoc(tt.hls, tt.hlsRelPath), outLoc(tt.mp4, tt.mp4RelPath), outLocation{}, outLoc(tt.thumbs, tt.thumbsRelPath), tt.sourceCopy)

if tt.hasError {
require.Error(t, err)
Expand Down

0 comments on commit e0d7db1

Please sign in to comment.