From 3da5f8c2430cd5be2f8271891d9d24e232aedf72 Mon Sep 17 00:00:00 2001 From: Nico Xiang <11958087+nicoxiang@users.noreply.github.com> Date: Fri, 9 Sep 2022 14:56:49 +0800 Subject: [PATCH] Geek class (#61) * feat: aliyun vod download --- README.md | 37 ++- cmd/root.go | 49 ++- go.mod | 1 + go.sum | 3 + internal/geektime/client.go | 149 ++++++++- internal/pkg/crypto/aes.go | 67 ++++ internal/pkg/crypto/hmac.go | 15 + internal/pkg/crypto/rsa.go | 37 +++ internal/pkg/geektime/domain.go | 1 + internal/pkg/m3u8/m3u8.go | 66 ++++ internal/pkg/m3u8/tsparser.go | 211 +++++++++++++ internal/video/video.go | 286 +++++++++++------- internal/video/vod/struct_play_info.go | 31 ++ .../struct_play_info_list_in_get_play_info.go | 7 + internal/video/vod/struct_video_base.go | 16 + internal/video/vod/vod.go | 128 ++++++++ 16 files changed, 963 insertions(+), 141 deletions(-) create mode 100644 internal/pkg/crypto/aes.go create mode 100644 internal/pkg/crypto/hmac.go create mode 100644 internal/pkg/crypto/rsa.go create mode 100644 internal/pkg/m3u8/m3u8.go create mode 100644 internal/pkg/m3u8/tsparser.go create mode 100644 internal/video/vod/struct_play_info.go create mode 100644 internal/video/vod/struct_play_info_list_in_get_play_info.go create mode 100644 internal/video/vod/struct_video_base.go create mode 100644 internal/video/vod/vod.go diff --git a/README.md b/README.md index 4378bf9..eb5490c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # geektime-downloader -geektime-downloader 支持下载专栏为 PDF/Markdown 文档和下载视频课。 +geektime-downloader 支持下载极客时间专栏(PDF/Markdown)和视频课，及训练营视频。 [![go report card](https://goreportcard.com/badge/github.com/nicoxiang/geektime-downloader "go report card")](https://goreportcard.com/report/github.com/nicoxiang/geektime-downloader) [![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT) @@ -56,14 +56,25 @@ Flags: -h, --help help for geektime-downloader --output int 专栏的输出内容(1pdf,2markdown,4audio)可自由组合 (default 1) -u, --phone string 你的极客时间账号(手机号) - -q, --quality string 下载视频清晰度(ld标清,sd高清,hd超清) (default "sd") + -q, --quality string 下载视频清晰度(ld标清,sd高清,hd超清) (default "sd") + --university 是否下载训练营的内容 ``` ## Note 1. 文件下载目标位置可以通过 help 查看。默认情况下 Windows 位于 %USERPROFILE%/geektime-downloader 下；Unix, 包括 macOS, 位于 $HOME/geektime-downloader 下 -2. 如何查看课程 ID? +2. 如何下载训练营课程? + +在命令行后追加 --university 参数，其余操作和普通课程相同，训练营暂时只支持下载视频课程 + +``` +> geektime-downloader.exe -u "phone number" --university +``` + +3. 如何查看课程 ID? + +普通课程： 打开极客时间[课程列表页](https://time.geekbang.org/resource)，选择你想要查看的课程，在新打开的课程详情 Tab 页，查看 URL 最后的数字，例如下面的链接中 100056701 就是课程 ID： @@ -71,18 +82,22 @@ Flags: https://time.geekbang.org/column/intro/100056701 ``` -3. Ctrl + C 退出程序 +训练营课程： -4. 如何下载 Markdown 格式和文章音频? +打开极客时间[训练营课程列表页](https://u.geekbang.org/schedule)，选择你想要查看的课程，在新打开的课程详情 Tab 页，查看 URL lesson/后的数字，例如下面的链接中 419 就是课程 ID： -默认情况下载专栏的输出内容只有 PDF，可以通过 --output 参数按需选择是否需要下载 Markdown 格式和文章音频。比如 --output 3 就是下载 PDF 和 Markdown；--output 6 就是下载 Markdown 和音频；--output 7 就是下载所有。 +``` +https://u.geekbang.org/lesson/419?article=535616 +``` -Markdown 格式虽然显示效果上不及 PDF，但优势为可以显示完整的代码块（PDF 代码块在水平方向太长时会有缺失）并保留了原文中的超链接。 +4. Ctrl + C 退出程序 -5. 如果选择下载所有后中断程序，可重新进入程序继续下载 +5. 如何下载专栏的 Markdown 格式和文章音频? -6. 通过密码登录的情况下，为了避免多次登录账户，会在目录 [UserConfigDir](https://pkg.go.dev/os#UserConfigDir)/geektime-downloader 下存放用户的登录 cookie，如果不是在自己的电脑上执行，请在使用完毕程序后手动删除 +默认情况下载专栏的输出内容只有 PDF，可以通过 --output 参数按需选择是否需要下载 Markdown 格式和文章音频。比如 --output 3 就是下载 PDF 和 Markdown；--output 6 就是下载 Markdown 和音频；--output 7 就是下载所有。 + +Markdown 格式虽然显示效果上不及 PDF，但优势为可以显示完整的代码块（PDF 代码块在水平方向太长时会有缺失）并保留了原文中的超链接。 -## Inspired by +6. 如果选择下载所有后中断程序，可重新进入程序继续下载 -* [geektime-dl](https://github.com/mmzou/geektime-dl) +7. 通过密码登录的情况下，为了避免多次登录账户，会在目录 [UserConfigDir](https://pkg.go.dev/os#UserConfigDir)/geektime-downloader 下存放用户的登录 cookie，如果不是在自己的电脑上执行，请在使用完毕程序后手动删除 diff --git a/cmd/root.go b/cmd/root.go index 9aacafb..f953134 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,6 +40,7 @@ var ( currentProduct geektime.Product quality string downloadComments bool + university bool columnOutputType int ) @@ -54,6 +55,7 @@ func init() { rootCmd.Flags().StringVarP(&downloadFolder, "folder", "f", defaultDownloadFolder, "专栏和视频课的下载目标位置") rootCmd.Flags().StringVarP(&quality, "quality", "q", "sd", "下载视频清晰度(ld标清,sd高清,hd超清)") rootCmd.Flags().BoolVar(&downloadComments, "comments", true, "是否需要专栏的第一页评论") + rootCmd.Flags().BoolVar(&university, "university", false, "是否下载训练营的内容") rootCmd.Flags().IntVar(&columnOutputType, "output", 1, "专栏的输出内容(1pdf,2markdown,4audio)可自由组合") rootCmd.MarkFlagsMutuallyExclusive("phone", "gcid") @@ -155,7 +157,11 @@ func productOps(ctx context.Context) { options[1] = option{"下载当前专栏所有文章", 1} options[2] = option{"选择文章", 2} } else if isVideo() { - options[1] = option{"下载当前视频课所有视频", 1} + s1 := "下载当前视频课所有视频" + if currentProduct.Type == geektime.ProductTypeUniversityVideo { + s1 = "下载当前训练营所有视频" + } + options[1] = option{s1, 1} options[2] = option{"选择视频", 2} } templates := &promptui.SelectTemplates{ @@ -302,7 +308,7 @@ func handleDownloadAll(ctx context.Context) { } checkError(err) - + increasePDFCount(total, &i) r := rand.Intn(2000) time.Sleep(time.Duration(r) * time.Millisecond) @@ -313,10 +319,15 @@ func handleDownloadAll(ctx context.Context) { if _, ok := downloaded[fileName]; ok { continue } - videoInfo, err := geektime.GetVideoInfo(a.AID, quality) - checkError(err) - err = video.DownloadVideo(ctx, videoInfo.M3U8URL, a.Title, projectDir, int64(videoInfo.Size), concurrency) - checkError(err) + if currentProduct.Type == geektime.ProductTypeNormalVideo { + videoInfo, err := geektime.GetVideoInfo(a.AID, quality) + checkError(err) + err = video.DownloadHLSStandardEncryptVideo(ctx, videoInfo.M3U8URL, a.Title, projectDir, int64(videoInfo.Size), concurrency) + checkError(err) + } else if currentProduct.Type == geektime.ProductTypeUniversityVideo { + err = video.DownloadAliyunVodEncryptVideo(ctx, a.AID, currentProduct, projectDir, quality, concurrency) + checkError(err) + } } } selectProduct(ctx) @@ -341,7 +352,14 @@ func loadArticles() { func loadProduct(ctx context.Context, productID int) { sp.Prefix = "[ 正在加载课程信息... ]" sp.Start() - p, err := geektime.GetColumnInfo(productID) + var p geektime.Product + var err error + if university { + p, err = geektime.GetMyClassProduct(productID) + } else { + p, err = geektime.GetColumnInfo(productID) + } + if err != nil { sp.Stop() checkError(err) @@ -402,19 +420,24 @@ func downloadArticle(ctx context.Context, article geektime.Article, projectDir s sp.Stop() } else if isVideo() { - videoInfo, err := geektime.GetVideoInfo(article.AID, quality) - checkError(err) - err = video.DownloadVideo(ctx, videoInfo.M3U8URL, article.Title, projectDir, int64(videoInfo.Size), concurrency) - checkError(err) + if currentProduct.Type == geektime.ProductTypeNormalVideo { + videoInfo, err := geektime.GetVideoInfo(article.AID, quality) + checkError(err) + err = video.DownloadHLSStandardEncryptVideo(ctx, videoInfo.M3U8URL, article.Title, projectDir, int64(videoInfo.Size), concurrency) + checkError(err) + } else if currentProduct.Type == geektime.ProductTypeUniversityVideo { + err := video.DownloadAliyunVodEncryptVideo(ctx, article.AID, currentProduct, projectDir, quality, concurrency) + checkError(err) + } } } func isColumn() bool { - return currentProduct.Type == "c1" + return currentProduct.Type == geektime.ProductTypeColumn } func isVideo() bool { - return currentProduct.Type == "c3" + return currentProduct.Type == geektime.ProductTypeNormalVideo || currentProduct.Type == geektime.ProductTypeUniversityVideo } // Sets the bit at pos in the integer n. diff --git a/go.mod b/go.mod index 4597a9c..03bbd77 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/chromedp/chromedp v0.8.5 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/go-resty/resty/v2 v2.7.0 + github.com/google/uuid v1.3.0 github.com/spf13/cobra v1.5.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) diff --git a/go.sum b/go.sum index 90cfed8..5391eb3 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -109,6 +111,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/geektime/client.go b/internal/geektime/client.go index 6098f0c..212f765 100644 --- a/internal/geektime/client.go +++ b/internal/geektime/client.go @@ -20,11 +20,23 @@ const ( ArticleV1Path = "/serv/v1/article" // ColumnInfoV3Path ... ColumnInfoV3Path = "/serv/v3/column/info" + // PlayAuthV1Path ... + PlayAuthV1Path = "/serv/v1/video/play-auth" + // MyClassInfoV1Path ... + MyClassInfoV1Path = "/serv/v1/myclass/info" + + // ProductTypeColumn c1 column + ProductTypeColumn = "c1" + // ProductTypeNormalVideo c3 normal video + ProductTypeNormalVideo = "c3" + // ProductTypeUniversityVideo u university video + ProductTypeUniversityVideo = "u" ) var ( - geekTimeClient *resty.Client - accountClient *resty.Client + geekTimeClient *resty.Client + accountClient *resty.Client + ugeekTimeClient *resty.Client // SiteCookies ... SiteCookies []*http.Cookie ) @@ -68,6 +80,12 @@ type ArticleInfo struct { AudioDownloadURL string } +// PlayAuthInfo ... +type PlayAuthInfo struct { + PlayAuth string + VideoID string +} + // ColumnResponse ... type ColumnResponse struct { Code int `json:"code"` @@ -78,6 +96,53 @@ type ColumnResponse struct { } `json:"data"` } +// PlayAuthResponse ... +type PlayAuthResponse struct { + Code int `json:"code"` + Data struct { + PlayAuth string `json:"play_auth"` + VID string `json:"vid"` + } `json:"data"` +} + +// MyClassInfoResponse ... +type MyClassInfoResponse struct { + Code int `json:"code"` + Data struct { + ClassType int `json:"class_type"` + Title string `json:"title"` + Lessons []struct { + ChapterName string `json:"chapter_name"` + BeginTime int `json:"begin_time"` + ChapterID int `json:"chapter_id"` + IndexNo int `json:"index_no"` + Articles []struct { + ArticleID int `json:"article_id"` + ArticleTitle string `json:"article_title"` + IndexNo int `json:"index_no"` + IsRead bool `json:"is_read"` + IsFinish bool `json:"is_finish"` + // HasNotes bool `json:"has_notes"` + // IsRequired int `json:"is_required"` + VideoTime int `json:"video_time"` + // LearnTime int `json:"learn_time"` + // LearnStatus int `json:"learn_status"` + // MaxOffset int `json:"max_offset"` + // ArticleMaxOffset int `json:"article_max_offset"` + // VideoMaxOffset int `json:"video_max_offset"` + // ArticleLen int `json:"article_len"` + // VideoLen int `json:"video_len"` + // Ctime int `json:"ctime"` + // Exercises []interface{} `json:"exercises"` + } `json:"articles"` + } `json:"lessons"` + } `json:"data"` + Error struct { + Code int `json:"code"` + Msg string `json:"msg"` + } `json:"error"` +} + // VideoResponse ... type VideoResponse struct { Code int `json:"code"` @@ -125,6 +190,14 @@ func InitClient(cookies []*http.Cookie) { SetHeader(pgt.OriginHeaderName, pgt.GeekBang). SetLogger(logger.DiscardLogger{}) + ugeekTimeClient = resty.New(). + SetBaseURL(pgt.GeekBangUniversity). + SetCookies(cookies). + SetTimeout(10*time.Second). + SetHeader(pgt.UserAgentHeaderName, pgt.UserAgentHeaderValue). + SetHeader(pgt.OriginHeaderName, pgt.GeekBangUniversity). + SetLogger(logger.DiscardLogger{}) + SiteCookies = cookies } @@ -327,3 +400,75 @@ func Auth() error { // status code 452 return pgt.ErrAuthFailed } + +// GetMyClassProduct ... +func GetMyClassProduct(classID int) (Product, error) { + var p Product + var resp MyClassInfoResponse + _, err := ugeekTimeClient.R().SetBody( + map[string]interface{}{ + "class_id": classID, + }). + SetResult(&resp). + Post(MyClassInfoV1Path) + + if err != nil { + return p, err + } + + if resp.Code != 0 { + if resp.Error.Code == -5001 { + p.Access = false + return p, nil + } + return p, ErrGeekTimeAPIBadCode{PlayAuthV1Path, resp.Code, ""} + } + + p = Product{ + Access: true, + ID: classID, + Title: resp.Data.Title, + Type: ProductTypeUniversityVideo, + } + var articles []Article + for _, lesson := range resp.Data.Lessons { + for _, article := range lesson.Articles { + // ONLY download university video lessons + if article.VideoTime > 0 { + articles = append(articles, Article{ + AID: article.ArticleID, + Title: article.ArticleTitle, + }) + } + } + } + p.Articles = articles + + return p, nil +} + +// GetPlayAuth ... +func GetPlayAuth(articleID, classID int) (PlayAuthInfo, error) { + var info PlayAuthInfo + var result PlayAuthResponse + _, err := ugeekTimeClient.R().SetBody( + map[string]interface{}{ + "article_id": articleID, + "class_id": classID, + }). + SetResult(&result). + Post(PlayAuthV1Path) + + if err != nil { + return info, err + } + + if result.Code != 0 { + return info, ErrGeekTimeAPIBadCode{PlayAuthV1Path, result.Code, ""} + } + + return PlayAuthInfo{ + PlayAuth: result.Data.PlayAuth, + VideoID: result.Data.VID, + }, nil +} diff --git a/internal/pkg/crypto/aes.go b/internal/pkg/crypto/aes.go new file mode 100644 index 0000000..c38a99b --- /dev/null +++ b/internal/pkg/crypto/aes.go @@ -0,0 +1,67 @@ +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "encoding/base64" + "fmt" +) + +// GetAESDecryptKey get 'aliyun private encyption method' decrypt key +// * * r0 = cmMeyfzJWyZcSwyH //rand +// * * r1 = md5(rand) +// * * r1 = r1.substring(8,24) +// * * iv = base64(r1).getBytes(); +// * * key1 = aes.decrypt(rnd,iv,iv) +// * * seed1 = md5(r0+key1) +// * * seed1 = seed1.substring(8,24) +// * * k2 = base64(seed1).getBytes(); +// * * key2 = aes.decrypt(plain,k2,iv); +// * * result = hex.encodeHexStr(base64.decode(key2)) +// @param cr client random string +// @param sr server response random string +// @param plainText server response plain text +func GetAESDecryptKey(cr, sr, plainText string) string { + crMD5 := fmt.Sprintf("%x", md5.Sum([]byte(cr))) + t1 := crMD5[8:24] + iv := []byte(t1) + sd, _ := base64.StdEncoding.DecodeString(sr) + dc1 := AESDecryptCBC(sd, iv, iv) + r2 := cr + string(dc1) + r2MD5 := fmt.Sprintf("%x", md5.Sum([]byte(r2))) + t2 := r2MD5[8:24] + key2 := []byte(t2) + pd, _ := base64.StdEncoding.DecodeString(plainText) + d2c := AESDecryptCBC(pd, key2, iv) + b, _ := base64.StdEncoding.DecodeString(string(d2c)) + return fmt.Sprintf("%x", b) +} + +// AESDecryptCBC ... +func AESDecryptCBC(encrypted, key, iv []byte) (decrypted []byte) { + block, _ := aes.NewCipher(key) + blockSize := block.BlockSize() + blockMode := cipher.NewCBCDecrypter(block, iv[:blockSize]) + decrypted = make([]byte, len(encrypted)) + blockMode.CryptBlocks(decrypted, encrypted) + decrypted = pkcs5UnPadding(decrypted) + return decrypted +} + +// AESDecryptECB ... +func AESDecryptECB(encrypted, key []byte) (decrypted []byte) { + block, _ := aes.NewCipher(key) + decrypted = make([]byte, len(encrypted)) + size := 16 + for bs, be := 0, size; bs < len(encrypted); bs, be = bs+size, be+size { + block.Decrypt(decrypted[bs:be], encrypted[bs:be]) + } + return decrypted +} + +func pkcs5UnPadding(origData []byte) []byte { + length := len(origData) + unpadding := int(origData[length-1]) + return origData[:(length - unpadding)] +} diff --git a/internal/pkg/crypto/hmac.go b/internal/pkg/crypto/hmac.go new file mode 100644 index 0000000..415d01e --- /dev/null +++ b/internal/pkg/crypto/hmac.go @@ -0,0 +1,15 @@ +package crypto + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" +) + +// HmacSHA1Signature ... +func HmacSHA1Signature(accessKeySecret, stringToSign string) string { + key := accessKeySecret + "&" + mac := hmac.New(sha1.New, []byte(key)) + mac.Write([]byte(stringToSign)) + return base64.StdEncoding.EncodeToString(mac.Sum(nil)) +} diff --git a/internal/pkg/crypto/rsa.go b/internal/pkg/crypto/rsa.go new file mode 100644 index 0000000..1a179e1 --- /dev/null +++ b/internal/pkg/crypto/rsa.go @@ -0,0 +1,37 @@ +package crypto + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" +) + +const ( + // PublicKeyStr ... + PublicKeyStr = ` +-----BEGIN PUBLIC KEY----- +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAIcLeIt2wmIyXckgNhCGpMTAZyBGO+nk0/IdOrhIdfRR +gBLHdydsftMVPNHrRuPKQNZRslWE1vvgx80w9lCllIUCAwEAAQ== +-----END PUBLIC KEY-----` +) + +// RSAEncrypt ... +func RSAEncrypt(origData []byte) (string, error) { + block, _ := pem.Decode([]byte(PublicKeyStr)) + if block == nil { + return "", errors.New("rsa encrypt public key error") + } + pubInterface, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return "", err + } + pub := pubInterface.(*rsa.PublicKey) + data, err := rsa.EncryptPKCS1v15(rand.Reader, pub, origData) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(data), nil +} diff --git a/internal/pkg/geektime/domain.go b/internal/pkg/geektime/domain.go index e096396..a9c849c 100644 --- a/internal/pkg/geektime/domain.go +++ b/internal/pkg/geektime/domain.go @@ -4,5 +4,6 @@ package geektime const ( GeekBang = "https://time.geekbang.org" GeekBangAccount = "https://account.geekbang.org" + GeekBangUniversity = "https://u.geekbang.org" GeekBangCookieDomain = ".geekbang.org" ) diff --git a/internal/pkg/m3u8/m3u8.go b/internal/pkg/m3u8/m3u8.go new file mode 100644 index 0000000..c1928ba --- /dev/null +++ b/internal/pkg/m3u8/m3u8.go @@ -0,0 +1,66 @@ +package m3u8 + +import ( + "bufio" + "regexp" + "strings" + "time" + + "github.com/go-resty/resty/v2" + pgt "github.com/nicoxiang/geektime-downloader/internal/pkg/geektime" + "github.com/nicoxiang/geektime-downloader/internal/pkg/logger" +) + +var ( + client *resty.Client + // regex pattern for extracting `key=value` parameters from a line + linePattern = regexp.MustCompile(`([a-zA-Z-]+)=("[^"]+"|[^",]+)`) +) + +func init() { + client = resty.New(). + SetRetryCount(1). + SetTimeout(10*time.Second). + SetHeader(pgt.UserAgentHeaderName, pgt.UserAgentHeaderValue). + SetLogger(logger.DiscardLogger{}) +} + +// Parse do m3u8 url GET request, and extract ts file names and decrypt key from that +func Parse(m3u8url string) (tsFileNames []string, keyURI string, err error) { + m3u8Resp, err := client.R().SetDoNotParseResponse(true).Get(m3u8url) + if err != nil { + return nil, "", err + } + defer m3u8Resp.RawBody().Close() + s := bufio.NewScanner(m3u8Resp.RawBody()) + var lines []string + for s.Scan() { + lines = append(lines, s.Text()) + } + + gotKeyURI := false + + for _, line := range lines { + // geektime video ONLY has one EXT-X-KEY + if strings.HasPrefix(line, "#EXT-X-KEY") && !gotKeyURI { + // ONLY Method and URI, IV not present + params := parseLineParameters(line) + keyURI, gotKeyURI = params["URI"], true + } + if !strings.HasPrefix(line, "#") && strings.HasSuffix(line, ".ts") { + tsFileNames = append(tsFileNames, line) + } + } + + return +} + +// parseLineParameters extra parameters in string `line` +func parseLineParameters(line string) map[string]string { + r := linePattern.FindAllStringSubmatch(line, -1) + params := make(map[string]string) + for _, arr := range r { + params[arr[1]] = strings.Trim(arr[2], "\"") + } + return params +} diff --git a/internal/pkg/m3u8/tsparser.go b/internal/pkg/m3u8/tsparser.go new file mode 100644 index 0000000..47b15a1 --- /dev/null +++ b/internal/pkg/m3u8/tsparser.go @@ -0,0 +1,211 @@ +package m3u8 + +// copy from +// https://github.com/SweetInk/lagou-course-downloader/blob/master/src/main/java/online/githuboy/lagou/course/decrypt/alibaba/TSParser.java +// https://github.com/lbbniu/aliyun-m3u8-downloader/blob/main/pkg/parse/aliyun/tsparser.go + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/nicoxiang/geektime-downloader/internal/pkg/crypto" +) + +const ( + packetLength = 188 + syncByte byte = 0x47 + payloadStartMask byte = 0x40 + atfMask byte = 0x30 + atfReserve byte = 0x00 + atfPayloadOnly byte = 0x01 + atfFieldOnly byte = 0x02 + atfFiledFollowPayload byte = 0x03 +) + +// TSParser ... +type TSParser struct { + stream *tsStream +} + +type tsPesFragment struct { + packets []*tsPacket +} + +type tsStream struct { + data []byte + key []byte + packets []*tsPacket + videos []*tsPesFragment + audios []*tsPesFragment +} + +type tsHeader struct { + syncByte byte //8 + transportErrorIndicator byte //1 + payloadUnitStartIndicator byte //1 + pid int //13 + transportScramblingControl byte //2 + adaptationFiled byte //2 + continuityCounter byte //4 + hasError bool + isPayloadStart bool + hasAdaptationFieldField bool + hasPayload bool +} + +type tsPacket struct { + header tsHeader + packNo int + startOffset int + headerLength int // 4 + atfLength int + pesOffset int + pesHeaderLength int + payloadStartOffset int + payloadRelativeOffset int // 0 + payloadLength int // 0 + payload []byte +} + +func newTSPacket() *tsPacket { + return &tsPacket{ + headerLength: 4, + } +} + +// NewTSParser ... +func NewTSParser(data []byte, key string) *TSParser { + hexKey, _ := hex.DecodeString(key) + stream := &tsStream{ + data: data, + key: hexKey, + } + stream.parseTS() + return &TSParser{ + stream: stream, + } +} + +// Decrypt ... +func (p *TSParser) Decrypt() []byte { + p.decryptPES(p.stream.data, p.stream.videos, p.stream.key) + p.decryptPES(p.stream.data, p.stream.audios, p.stream.key) + return p.stream.data +} + +func (p *TSParser) decryptPES(byteBuf []byte, pesFragments []*tsPesFragment, key []byte) { + for _, pes := range pesFragments { + buffer := &bytes.Buffer{} + for _, packet := range pes.packets { + if nil == packet.payload { + panic("payload is null") + } + buffer.Write(packet.payload) + } + length := buffer.Len() + all := buffer.Bytes() + buffer.Reset() + if length%16 > 0 { + newLength := 16 * (length / 16) + decrypt := crypto.AESDecryptECB(all[:newLength], key) + buffer.Write(decrypt) + buffer.Write(all[newLength:]) + } else { + decrypt := crypto.AESDecryptECB(all, key) + buffer.Write(decrypt) + } + //Rewrite decrypted bytes to byteBuf + for _, packet := range pes.packets { + payloadLength := packet.payloadLength + payloadStartOffset := packet.payloadStartOffset + buffer.Read(byteBuf[payloadStartOffset : payloadStartOffset+payloadLength]) + } + } +} + +func (pes *tsPesFragment) add(packet *tsPacket) { + pes.packets = append(pes.packets, packet) +} + +func (stream *tsStream) parseTS() { + byteBuf := bytes.NewReader(stream.data) + length := byteBuf.Len() + if length%packetLength != 0 { + panic("not a ts package") + } + var pes *tsPesFragment + packNums := length / packetLength + for packageNo := 0; packageNo < packNums; packageNo++ { + buffer := make([]byte, packetLength) + byteBuf.Read(buffer) + packet := stream.parseTSPacket(buffer, packageNo, packageNo*packetLength) + switch packet.header.pid { + // video data + case 0x100: + if packet.header.isPayloadStart { + if nil != pes { + stream.videos = append(stream.videos, pes) + } + pes = new(tsPesFragment) + } + pes.add(packet) + //audio data + case 0x101: + if packet.header.isPayloadStart { + if nil != pes { + stream.audios = append(stream.audios, pes) + } + pes = new(tsPesFragment) + } + pes.add(packet) + } + stream.packets = append(stream.packets, packet) + } +} + +func (stream *tsStream) parseTSPacket(buffer []byte, packNo, offset int) *tsPacket { + if buffer[0] != syncByte { + panic(fmt.Sprintf("Invalid ts package in :%d offset: %d", packNo, offset)) + } + header := tsHeader{} + header.syncByte = buffer[0] + if buffer[1]&0x80 > 0 { + header.transportErrorIndicator = 1 + } + if buffer[1]&payloadStartMask > 0 { + header.payloadUnitStartIndicator = 1 + } + if buffer[1]&0x20 > 0 { + header.transportErrorIndicator = 1 + } + header.pid = int(buffer[1]&0x1F)<<8 | int(buffer[2]&0xFF) + header.transportScramblingControl = ((buffer[3] & 0xC0) >> 6) & 0xFF + header.adaptationFiled = ((buffer[3] & atfMask) >> 4) & 0xFF + header.continuityCounter = (buffer[3] & 0x0F) & 0xFF + header.hasError = header.transportErrorIndicator != 0 + header.isPayloadStart = header.payloadUnitStartIndicator != 0 + header.hasAdaptationFieldField = header.adaptationFiled == atfFieldOnly || header.adaptationFiled == atfFiledFollowPayload + header.hasPayload = header.adaptationFiled == atfPayloadOnly || header.adaptationFiled == atfFiledFollowPayload + packet := newTSPacket() + packet.header = header + packet.packNo = packNo + packet.startOffset = offset + if header.hasAdaptationFieldField { + atfLength := buffer[4] & 0xFF + packet.headerLength++ + packet.atfLength = int(atfLength) + } + if header.isPayloadStart { + packet.pesOffset = packet.startOffset + packet.headerLength + packet.atfLength + // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension + packet.pesHeaderLength = int(6 + 3 + buffer[packet.headerLength+packet.atfLength+8]&0xFF) + } + packet.payloadRelativeOffset = packet.headerLength + packet.atfLength + packet.pesHeaderLength + packet.payloadStartOffset = int(packet.startOffset + packet.payloadRelativeOffset) + packet.payloadLength = packetLength - packet.payloadRelativeOffset + if packet.payloadLength > 0 { + packet.payload = buffer[packet.payloadRelativeOffset:packetLength] + } + return packet +} diff --git a/internal/video/video.go b/internal/video/video.go index 49483cf..457c803 100644 --- a/internal/video/video.go +++ b/internal/video/video.go @@ -2,23 +2,24 @@ package video import ( "context" - "crypto/aes" - "crypto/cipher" "errors" "fmt" "io/ioutil" "os" "path/filepath" - "sort" - "strconv" "strings" "time" "github.com/cheggaaa/pb/v3" "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "github.com/nicoxiang/geektime-downloader/internal/geektime" + "github.com/nicoxiang/geektime-downloader/internal/pkg/crypto" "github.com/nicoxiang/geektime-downloader/internal/pkg/filenamify" pgt "github.com/nicoxiang/geektime-downloader/internal/pkg/geektime" "github.com/nicoxiang/geektime-downloader/internal/pkg/logger" + "github.com/nicoxiang/geektime-downloader/internal/pkg/m3u8" + "github.com/nicoxiang/geektime-downloader/internal/video/vod" "golang.org/x/sync/errgroup" ) @@ -28,70 +29,113 @@ const ( TSExtension = ".ts" ) +// EncryptType enum +type EncryptType int + +const ( + // AliyunVodEncrypt ... + AliyunVodEncrypt EncryptType = iota + // HLSStandardEncrypt ... + HLSStandardEncrypt +) + var ( + // make simple api call client *resty.Client + // used to download video + downloadClient *resty.Client ) func init() { client = resty.New(). + SetHeader(pgt.UserAgentHeaderName, pgt.UserAgentHeaderValue). SetRetryCount(1). - SetTimeout(10*time.Second). + SetTimeout(10 * time.Second). + SetLogger(logger.DiscardLogger{}) + + downloadClient = resty.New(). SetHeader(pgt.UserAgentHeaderName, pgt.UserAgentHeaderValue). - SetHeader(pgt.OriginHeaderName, pgt.GeekBang). + SetRetryCount(0). + SetTimeout(time.Minute). SetLogger(logger.DiscardLogger{}) } -// ByNumericalFilename implement sort interface, order by file name suffix number -type ByNumericalFilename []os.FileInfo - -func (nf ByNumericalFilename) Len() int { return len(nf) } -func (nf ByNumericalFilename) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] } -func (nf ByNumericalFilename) Less(i, j int) bool { - // Use path names - pathA := nf[i].Name() - pathB := nf[j].Name() - - // Grab integer value of each filename by parsing the string and slicing off - // the extension - a, err1 := strconv.ParseInt(pathA[0:strings.LastIndex(pathA, ".")], 10, 64) - b, err2 := strconv.ParseInt(pathB[0:strings.LastIndex(pathB, ".")], 10, 64) - - // If any were not numbers sort lexographically - if err1 != nil || err2 != nil { - return pathA < pathB - } - - // Which integer is smaller? - return a < b +// GetPlayInfoResponse is the response struct for api GetPlayInfo +type GetPlayInfoResponse struct { + RequestID string `json:"RequestId" xml:"RequestId"` + VideoBase vod.VideoBase `json:"VideoBase" xml:"VideoBase"` + PlayInfoList vod.PlayInfoListInGetPlayInfo `json:"PlayInfoList" xml:"PlayInfoList"` } -// DownloadVideo ... -func DownloadVideo(ctx context.Context, m3u8url, title, projectDir string, size int64, concurrency int) (err error) { - resetRestyOptions() - - i := strings.LastIndex(m3u8url, "/") - tsURLPrefix := m3u8url[:i+1] - filenamifyTitle := filenamify.Filenamify(title) - - // Stage1: Make m3u8 URL call and resolve - decryptkmsURL, tsFileNames, err := readM3U8File(ctx, m3u8url) +// DownloadAliyunVodEncryptVideo ... +func DownloadAliyunVodEncryptVideo(ctx context.Context, + articleID int, + currentProduct geektime.Product, + projectDir string, + quality string, + concurrency int) error { + playAuthInfo, err := geektime.GetPlayAuth(articleID, currentProduct.ID) if err != nil { - return + return err + } + clientRand := uuid.NewString() + playInfoURL, err := vod.BuildVodGetPlayInfoURL(playAuthInfo.PlayAuth, playAuthInfo.VideoID, clientRand) + if err != nil { + return err + } + playInfo, err := getPlayInfo(playInfoURL, quality) + if err != nil { + return err } - if decryptkmsURL == "" || len(tsFileNames) == 0 { - return errors.New("unexpected m3u8 response format") + tsURLPrefix := extractTSURLPrefix(playInfo.PlayURL) + // just ignore keyURI in m3u8, aliyun private vod use another decrypt method + tsFileNames, _, err := m3u8.Parse(playInfo.PlayURL) + if err != nil { + return err } + decryptKey := crypto.GetAESDecryptKey(clientRand, playInfo.Rand, playInfo.Plaintext) + title := getUniversityVideoTitle(articleID, currentProduct) + return download(ctx, tsURLPrefix, title, projectDir, tsFileNames, []byte(decryptKey), playInfo.Size, AliyunVodEncrypt, concurrency) +} - // Stage2: Get decrypt key - key, err := getDecryptKey(ctx, decryptkmsURL) +// DownloadHLSStandardEncryptVideo ... +func DownloadHLSStandardEncryptVideo(ctx context.Context, m3u8url, title, projectDir string, size int64, concurrency int) (err error) { + tsURLPrefix := extractTSURLPrefix(m3u8url) + tsFileNames, keyURI, err := m3u8.Parse(m3u8url) if err != nil { - return + return err } - if key == nil { - return errors.New("unexpected decrypt key response") + var decryptKey []byte + // Old version keyURI + // https://misc.geekbang.org/serv/v1/decrypt/decryptkms/?Ciphertext=longlongstring + if strings.HasPrefix(keyURI, "https://") || strings.HasPrefix(keyURI, "http://") { + resp, err := client.R(). + SetContext(ctx). + SetHeader(pgt.OriginHeaderName, pgt.GeekBang). + Get(keyURI) + decryptKey = resp.Body() + if err != nil { + return err + } + } else { + return errors.New("unexpected m3u8 keyURI") } - // Stage3: Make temp ts folder and download temp ts files + return download(ctx, tsURLPrefix, title, projectDir, tsFileNames, decryptKey, size, HLSStandardEncrypt, concurrency) +} + +func download(ctx context.Context, + tsURLPrefix, + title, + projectDir string, + tsFileNames []string, + decryptKey []byte, + size int64, + videoEncryptType EncryptType, + concurrency int) (err error) { + + // Make temp ts folder and download temp ts files + filenamifyTitle := filenamify.Filenamify(title) tempVideoDir := filepath.Join(projectDir, filenamifyTitle) if err = os.MkdirAll(tempVideoDir, os.ModePerm); err != nil { return @@ -108,7 +152,7 @@ func DownloadVideo(ctx context.Context, m3u8url, title, projectDir string, size bar := newBar(size, fmt.Sprintf("[正在下载 %s] ", filenamifyTitle)) bar.Start() - setVideoDownloadClientOptions(tempVideoDir) + downloadClient.SetOutputDirectory(tempVideoDir) for i := 0; i < concurrency; i++ { g.Go(func() error { @@ -126,8 +170,8 @@ func DownloadVideo(ctx context.Context, m3u8url, title, projectDir string, size return } - // Stage4: Read temp ts files, decrypt and merge into the one final video file - err = mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir, key) + // Read temp ts files, decrypt and merge into the one final video file + err = mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir, decryptKey, videoEncryptType) return } @@ -137,45 +181,37 @@ func writeToTempVideoFile(ctx context.Context, bar *pb.ProgressBar, tsURLPrefix string) (err error) { for tsFileName := range tsFileNames { - resp, err := client.R(). - SetContext(ctx). - SetOutput(tsFileName). - Get(tsURLPrefix + tsFileName) + + // fix error: http2: server sent GOAWAY and closed the connection; LastStreamID=1999 + // resty retry not work, because error comes from io read, not request + err := retry(5, 700*time.Millisecond, func() error { + resp, err := downloadClient.R(). + SetContext(ctx). + SetHeader(pgt.OriginHeaderName, pgt.GeekBang). + SetOutput(tsFileName). + Get(tsURLPrefix + tsFileName) + if err != nil { + return err + } + addBarValue(bar, resp.Size()) + return nil + }) + if err != nil { for range tsFileNames { } return err } - addBarValue(bar, resp.Size()) - } - return nil -} -func readM3U8File(ctx context.Context, url string) (decryptkmsURL string, tsFileNames []string, err error) { - resp, err := client.R().SetContext(ctx).Get(url) - if err != nil { - return - } - s := string(resp.Body()) - lines := strings.Split(s, "\n") - for _, line := range lines { - if strings.HasPrefix(line, "#EXT-X-KEY") { - i := strings.LastIndex(line, "URI=") - decryptkmsURL = line[i+5 : len(line)-1] - } - if !strings.HasPrefix(line, "#") && strings.HasSuffix(line, ".ts") { - tsFileNames = append(tsFileNames, line) - } } - return + return nil } -func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte) error { +func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte, videoEncryptType EncryptType) error { tempTSFiles, err := ioutil.ReadDir(tempVideoDir) if err != nil { return err } - sort.Sort(ByNumericalFilename(tempTSFiles)) fullPath := filepath.Join(projectDir, filenamifyTitle+TSExtension) finalVideoFile, err := os.OpenFile(fullPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) defer func() { @@ -189,45 +225,28 @@ func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte) if err != nil { return err } - aes128 := aesDecryptCBC(f, key, make([]byte, 16)) - // https://en.wikipedia.org/wiki/MPEG_transport_stream - for j := 0; j < len(aes128); j++ { - if aes128[j] == syncByte { - aes128 = aes128[j:] - break + switch videoEncryptType { + case HLSStandardEncrypt: + aes128 := crypto.AESDecryptCBC(f, key, make([]byte, 16)) + // https://en.wikipedia.org/wiki/MPEG_transport_stream + for j := 0; j < len(aes128); j++ { + if aes128[j] == syncByte { + aes128 = aes128[j:] + break + } } + f = aes128 + case AliyunVodEncrypt: + tsParser := m3u8.NewTSParser(f, string(key)) + f = tsParser.Decrypt() } - if _, err := finalVideoFile.Write(aes128); err != nil { + if _, err := finalVideoFile.Write(f); err != nil { return err } } return nil } -func getDecryptKey(ctx context.Context, decryptkmsURL string) (key []byte, err error) { - keyResp, err := client.R().SetContext(ctx).Get(decryptkmsURL) - if err != nil { - return - } - return keyResp.Body(), nil -} - -func aesDecryptCBC(encrypted, key, iv []byte) (decrypted []byte) { - block, _ := aes.NewCipher(key) - blockSize := block.BlockSize() - blockMode := cipher.NewCBCDecrypter(block, iv[:blockSize]) - decrypted = make([]byte, len(encrypted)) - blockMode.CryptBlocks(decrypted, encrypted) - decrypted = pkcs5UnPadding(decrypted) - return decrypted -} - -func pkcs5UnPadding(origData []byte) []byte { - length := len(origData) - unpadding := int(origData[length-1]) - return origData[:(length - unpadding)] -} - func newBar(size int64, prefix string) *pb.ProgressBar { bar := pb.New64(size) bar.SetRefreshRate(time.Second) @@ -247,14 +266,51 @@ func addBarValue(bar *pb.ProgressBar, written int64) { } } -func resetRestyOptions() { - client.SetTimeout(10 * time.Second). - SetRetryCount(1). - SetOutputDirectory("") +func getUniversityVideoTitle(articleID int, currentProduct geektime.Product) string { + for _, v := range currentProduct.Articles { + if v.AID == articleID { + return v.Title + } + } + return "" } -func setVideoDownloadClientOptions(tempVideoDir string) { - client.SetTimeout(time.Minute). - SetRetryCount(0). - SetOutputDirectory(tempVideoDir) +func extractTSURLPrefix(m3u8url string) string { + i := strings.LastIndex(m3u8url, "/") + return m3u8url[:i+1] +} + +func getPlayInfo(playInfoURL, quality string) (vod.PlayInfo, error) { + var getPlayInfoResp GetPlayInfoResponse + var playInfo vod.PlayInfo + _, err := client.R(). + SetHeader(pgt.OriginHeaderName, pgt.GeekBangUniversity). + SetResult(&getPlayInfoResp). + Get(playInfoURL) + + if err != nil { + return playInfo, err + } + + playInfoList := getPlayInfoResp.PlayInfoList.PlayInfo + for _, p := range playInfoList { + if strings.EqualFold(p.Definition, quality) { + playInfo = p + } + } + return playInfo, nil +} + +func retry(attempts int, sleep time.Duration, f func() error) (err error) { + for i := 0; i < attempts; i++ { + if i > 0 { + time.Sleep(sleep) + sleep *= 2 + } + err = f() + if err == nil || errors.Is(err, context.Canceled) { + return err + } + } + return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } diff --git a/internal/video/vod/struct_play_info.go b/internal/video/vod/struct_play_info.go new file mode 100644 index 0000000..7e911e5 --- /dev/null +++ b/internal/video/vod/struct_play_info.go @@ -0,0 +1,31 @@ +package vod +// from https://github.com/aliyun/alibaba-cloud-sdk-go + +// PlayInfo is a nested struct in vod response +type PlayInfo struct { + Format string `json:"Format" xml:"Format"` + BitDepth int `json:"BitDepth" xml:"BitDepth"` + NarrowBandType string `json:"NarrowBandType" xml:"NarrowBandType"` + Fps string `json:"Fps" xml:"Fps"` + Encrypt int64 `json:"Encrypt" xml:"Encrypt"` + Rand string `json:"Rand" xml:"Rand"` + StreamType string `json:"StreamType" xml:"StreamType"` + WatermarkID string `json:"WatermarkId" xml:"WatermarkId"` + Size int64 `json:"Size" xml:"Size"` + Definition string `json:"Definition" xml:"Definition"` + Plaintext string `json:"Plaintext" xml:"Plaintext"` + JobID string `json:"JobId" xml:"JobId"` + EncryptType string `json:"EncryptType" xml:"EncryptType"` + PreprocessStatus string `json:"PreprocessStatus" xml:"PreprocessStatus"` + ModificationTime string `json:"ModificationTime" xml:"ModificationTime"` + Bitrate string `json:"Bitrate" xml:"Bitrate"` + CreationTime string `json:"CreationTime" xml:"CreationTime"` + Height int64 `json:"Height" xml:"Height"` + Complexity string `json:"Complexity" xml:"Complexity"` + Duration string `json:"Duration" xml:"Duration"` + HDRType string `json:"HDRType" xml:"HDRType"` + Width int64 `json:"Width" xml:"Width"` + Status string `json:"Status" xml:"Status"` + Specification string `json:"Specification" xml:"Specification"` + PlayURL string `json:"PlayURL" xml:"PlayURL"` +} \ No newline at end of file diff --git a/internal/video/vod/struct_play_info_list_in_get_play_info.go b/internal/video/vod/struct_play_info_list_in_get_play_info.go new file mode 100644 index 0000000..3989243 --- /dev/null +++ b/internal/video/vod/struct_play_info_list_in_get_play_info.go @@ -0,0 +1,7 @@ +package vod +// from https://github.com/aliyun/alibaba-cloud-sdk-go + +// PlayInfoListInGetPlayInfo is a nested struct in vod response +type PlayInfoListInGetPlayInfo struct { + PlayInfo []PlayInfo `json:"PlayInfo" xml:"PlayInfo"` +} \ No newline at end of file diff --git a/internal/video/vod/struct_video_base.go b/internal/video/vod/struct_video_base.go new file mode 100644 index 0000000..f1e463e --- /dev/null +++ b/internal/video/vod/struct_video_base.go @@ -0,0 +1,16 @@ +package vod +// from https://github.com/aliyun/alibaba-cloud-sdk-go + +// VideoBase is a nested struct in vod response +type VideoBase struct { + CreationTime string `json:"CreationTime" xml:"CreationTime"` + Status string `json:"Status" xml:"Status"` + TranscodeMode string `json:"TranscodeMode" xml:"TranscodeMode"` + OutputType string `json:"OutputType" xml:"OutputType"` + VideoID string `json:"VideoId" xml:"VideoId"` + CoverURL string `json:"CoverURL" xml:"CoverURL"` + Duration string `json:"Duration" xml:"Duration"` + Title string `json:"Title" xml:"Title"` + MediaType string `json:"MediaType" xml:"MediaType"` + DanMuURL string `json:"DanMuURL" xml:"DanMuURL"` +} \ No newline at end of file diff --git a/internal/video/vod/vod.go b/internal/video/vod/vod.go new file mode 100644 index 0000000..1216e21 --- /dev/null +++ b/internal/video/vod/vod.go @@ -0,0 +1,128 @@ +package vod + +import ( + "encoding/base64" + "encoding/json" + "net/url" + "sort" + "strings" + "time" + + "github.com/google/uuid" + pc "github.com/nicoxiang/geektime-downloader/internal/pkg/crypto" +) + +var ( + // PlayAuthSign1 ... + PlayAuthSign1 = []int{52, 58, 53, 121, 116, 102} + // PlayAuthSign2 ... + PlayAuthSign2 = []int{90, 91} +) + +// BuildVodGetPlayInfoURL ... +func BuildVodGetPlayInfoURL(playAuth, videoID, clientRand string) (string, error) { + decodedPlayAuth := decodePlayAuth(playAuth) + var jsonMap map[string]string + json.Unmarshal([]byte(decodedPlayAuth), &jsonMap) + + encryptedClientRand, err := pc.RSAEncrypt([]byte(clientRand)) + if err != nil { + return "", err + } + + publicParams := map[string]string{} + publicParams["AccessKeyId"] = jsonMap["AccessKeyId"] + publicParams["SignatureMethod"] = "HMAC-SHA1" + publicParams["SignatureVersion"] = "1.0" + publicParams["SignatureNonce"] = uuid.NewString() + publicParams["Format"] = "JSON" + publicParams["Channel"] = "HTML5" + publicParams["StreamType"] = "video" + publicParams["Rand"] = encryptedClientRand + publicParams["Formats"] = "" + publicParams["Version"] = "2017-03-21" + + privateParams := map[string]string{} + privateParams["Action"] = "GetPlayInfo" + privateParams["AuthInfo"] = jsonMap["AuthInfo"] + privateParams["AuthTimeout"] = "7200" + privateParams["PlayConfig"] = "{}" + privateParams["PlayerVersion"] = "2.8.2" + privateParams["ReAuthInfo"] = "{}" + privateParams["SecurityToken"] = jsonMap["SecurityToken"] + privateParams["VideoId"] = videoID + allParams := getAllParams(publicParams, privateParams) + cqs := getCQS(allParams) + stringToSign := "GET" + "&" + percentEncode("/") + "&" + percentEncode(cqs) + accessKeySecret := jsonMap["AccessKeySecret"] + signature := pc.HmacSHA1Signature(accessKeySecret, stringToSign) + queryString := cqs + "&Signature=" + percentEncode(signature) + return "https://vod.cn-shanghai.aliyuncs.com/?" + queryString, nil +} + +func decodePlayAuth(playAuth string) string { + if isSignedPlayAuth(playAuth) { + playAuth = decodeSignedPlayAuth2B64(playAuth) + } + data, err := base64.StdEncoding.DecodeString(playAuth) + if err != nil { + return "" + } + return string(data) +} + +func isSignedPlayAuth(playAuth string) bool { + signPos1 := time.Now().Year() / 100 + signPos2 := len(playAuth) - 2 + sign1 := getSignStr(PlayAuthSign1) + sign2 := getSignStr(PlayAuthSign2) + r1 := playAuth[signPos1 : signPos1+len(sign1)] + r2 := playAuth[signPos2:] + return sign1 == r1 && r2 == sign2 +} + +func decodeSignedPlayAuth2B64(playAuth string) string { + sign1 := getSignStr(PlayAuthSign1) + sign2 := getSignStr(PlayAuthSign2) + playAuth = strings.Replace(playAuth, sign1, "", 1) + playAuth = playAuth[:len(playAuth)-len(sign2)] + factor := time.Now().Year() / 100 + newCharCodeList := []byte(playAuth) + for i, code := range newCharCodeList { + r := int(code) / factor + z := factor / 10 + if r == z { + newCharCodeList[i] = code + } else { + newCharCodeList[i] = code - 1 + } + } + return string(newCharCodeList) +} + +func getSignStr(sign []int) string { + s := strings.Builder{} + for i, b := range sign { + s.WriteByte(byte(b - i)) + } + return s.String() +} + +func percentEncode(s string) string { + return url.QueryEscape(s) +} + +func getCQS(allParams []string) string { + sort.Strings(allParams) + return strings.Join(allParams, "&") +} + +func getAllParams(publicParams, privateParams map[string]string) (allParams []string) { + for key, value := range publicParams { + allParams = append(allParams, percentEncode(key)+"="+percentEncode(value)) + } + for key, value := range privateParams { + allParams = append(allParams, percentEncode(key)+"="+percentEncode(value)) + } + return allParams +}