Skip to content

Commit

Permalink
Geek class (#61)
Browse files Browse the repository at this point in the history 
* feat: aliyun vod download
  • Loading branch information
@nicoxiang
nicoxiang authored Sep 9, 2022
1 parent 80e8f46 commit 3da5f8c
Show file tree
Hide file tree
Showing 16 changed files with 963 additions and 141 deletions.
37 changes: 26 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# geektime-downloader

geektime-downloader 支持下载专栏为 PDF/Markdown 文档和下载视频课。
geektime-downloader 支持下载极客时间专栏(PDF/Markdown)和视频课，及训练营视频。

[![go report card](https://goreportcard.com/badge/github.com/nicoxiang/geektime-downloader "go report card")](https://goreportcard.com/report/github.com/nicoxiang/geektime-downloader)
[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT)
Expand Down Expand Up @@ -56,33 +56,48 @@ Flags:
-h, --help help for geektime-downloader
--output int 专栏的输出内容(1pdf,2markdown,4audio)可自由组合 (default 1)
-u, --phone string 你的极客时间账号(手机号)
-q, --quality string 下载视频清晰度(ld标清,sd高清,hd超清) (default "sd")
-q, --quality string 下载视频清晰度(ld标清,sd高清,hd超清) (default "sd")
--university 是否下载训练营的内容
```

## Note

1. 文件下载目标位置可以通过 help 查看。默认情况下 Windows 位于 %USERPROFILE%/geektime-downloader 下；Unix, 包括 macOS, 位于 $HOME/geektime-downloader 下

2. 如何查看课程 ID?
2. 如何下载训练营课程?

在命令行后追加 --university 参数，其余操作和普通课程相同，训练营暂时只支持下载视频课程

```
> geektime-downloader.exe -u "phone number" --university
```

3. 如何查看课程 ID?

普通课程：

打开极客时间[课程列表页](https://time.geekbang.org/resource)，选择你想要查看的课程，在新打开的课程详情 Tab 页，查看 URL 最后的数字，例如下面的链接中 100056701 就是课程 ID：

```
https://time.geekbang.org/column/intro/100056701
```

3. Ctrl + C 退出程序
训练营课程：

4. 如何下载 Markdown 格式和文章音频?
打开极客时间[训练营课程列表页](https://u.geekbang.org/schedule)，选择你想要查看的课程，在新打开的课程详情 Tab 页，查看 URL lesson/后的数字，例如下面的链接中 419 就是课程 ID：

默认情况下载专栏的输出内容只有 PDF，可以通过 --output 参数按需选择是否需要下载 Markdown 格式和文章音频。比如 --output 3 就是下载 PDF 和 Markdown；--output 6 就是下载 Markdown 和音频；--output 7 就是下载所有。
```
https://u.geekbang.org/lesson/419?article=535616
```

Markdown 格式虽然显示效果上不及 PDF，但优势为可以显示完整的代码块（PDF 代码块在水平方向太长时会有缺失）并保留了原文中的超链接。
4. Ctrl + C 退出程序

5. 如果选择下载所有后中断程序，可重新进入程序继续下载
5. 如何下载专栏的 Markdown 格式和文章音频?

6. 通过密码登录的情况下，为了避免多次登录账户，会在目录 [UserConfigDir](https://pkg.go.dev/os#UserConfigDir)/geektime-downloader 下存放用户的登录 cookie，如果不是在自己的电脑上执行，请在使用完毕程序后手动删除
默认情况下载专栏的输出内容只有 PDF，可以通过 --output 参数按需选择是否需要下载 Markdown 格式和文章音频。比如 --output 3 就是下载 PDF 和 Markdown；--output 6 就是下载 Markdown 和音频；--output 7 就是下载所有。

Markdown 格式虽然显示效果上不及 PDF，但优势为可以显示完整的代码块（PDF 代码块在水平方向太长时会有缺失）并保留了原文中的超链接。

## Inspired by
6. 如果选择下载所有后中断程序，可重新进入程序继续下载

* [geektime-dl](https://github.com/mmzou/geektime-dl)
7. 通过密码登录的情况下，为了避免多次登录账户，会在目录 [UserConfigDir](https://pkg.go.dev/os#UserConfigDir)/geektime-downloader 下存放用户的登录 cookie，如果不是在自己的电脑上执行，请在使用完毕程序后手动删除
49 changes: 36 additions & 13 deletions cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ var (
currentProduct geektime.Product
quality string
downloadComments bool
university bool
columnOutputType int
)

Expand All @@ -54,6 +55,7 @@ func init() {
rootCmd.Flags().StringVarP(&downloadFolder, "folder", "f", defaultDownloadFolder, "专栏和视频课的下载目标位置")
rootCmd.Flags().StringVarP(&quality, "quality", "q", "sd", "下载视频清晰度(ld标清,sd高清,hd超清)")
rootCmd.Flags().BoolVar(&downloadComments, "comments", true, "是否需要专栏的第一页评论")
rootCmd.Flags().BoolVar(&university, "university", false, "是否下载训练营的内容")
rootCmd.Flags().IntVar(&columnOutputType, "output", 1, "专栏的输出内容(1pdf,2markdown,4audio)可自由组合")

rootCmd.MarkFlagsMutuallyExclusive("phone", "gcid")
Expand Down Expand Up @@ -155,7 +157,11 @@ func productOps(ctx context.Context) {
options[1] = option{"下载当前专栏所有文章", 1}
options[2] = option{"选择文章", 2}
} else if isVideo() {
options[1] = option{"下载当前视频课所有视频", 1}
s1 := "下载当前视频课所有视频"
if currentProduct.Type == geektime.ProductTypeUniversityVideo {
s1 = "下载当前训练营所有视频"
}
options[1] = option{s1, 1}
options[2] = option{"选择视频", 2}
}
templates := &promptui.SelectTemplates{
Expand Down Expand Up @@ -302,7 +308,7 @@ func handleDownloadAll(ctx context.Context) {
}

checkError(err)

increasePDFCount(total, &i)
r := rand.Intn(2000)
time.Sleep(time.Duration(r) * time.Millisecond)
Expand All @@ -313,10 +319,15 @@ func handleDownloadAll(ctx context.Context) {
if _, ok := downloaded[fileName]; ok {
continue
}
videoInfo, err := geektime.GetVideoInfo(a.AID, quality)
checkError(err)
err = video.DownloadVideo(ctx, videoInfo.M3U8URL, a.Title, projectDir, int64(videoInfo.Size), concurrency)
checkError(err)
if currentProduct.Type == geektime.ProductTypeNormalVideo {
videoInfo, err := geektime.GetVideoInfo(a.AID, quality)
checkError(err)
err = video.DownloadHLSStandardEncryptVideo(ctx, videoInfo.M3U8URL, a.Title, projectDir, int64(videoInfo.Size), concurrency)
checkError(err)
} else if currentProduct.Type == geektime.ProductTypeUniversityVideo {
err = video.DownloadAliyunVodEncryptVideo(ctx, a.AID, currentProduct, projectDir, quality, concurrency)
checkError(err)
}
}
}
selectProduct(ctx)
Expand All @@ -341,7 +352,14 @@ func loadArticles() {
func loadProduct(ctx context.Context, productID int) {
sp.Prefix = "[ 正在加载课程信息... ]"
sp.Start()
p, err := geektime.GetColumnInfo(productID)
var p geektime.Product
var err error
if university {
p, err = geektime.GetMyClassProduct(productID)
} else {
p, err = geektime.GetColumnInfo(productID)
}

if err != nil {
sp.Stop()
checkError(err)
Expand Down Expand Up @@ -402,19 +420,24 @@ func downloadArticle(ctx context.Context, article geektime.Article, projectDir s

sp.Stop()
} else if isVideo() {
videoInfo, err := geektime.GetVideoInfo(article.AID, quality)
checkError(err)
err = video.DownloadVideo(ctx, videoInfo.M3U8URL, article.Title, projectDir, int64(videoInfo.Size), concurrency)
checkError(err)
if currentProduct.Type == geektime.ProductTypeNormalVideo {
videoInfo, err := geektime.GetVideoInfo(article.AID, quality)
checkError(err)
err = video.DownloadHLSStandardEncryptVideo(ctx, videoInfo.M3U8URL, article.Title, projectDir, int64(videoInfo.Size), concurrency)
checkError(err)
} else if currentProduct.Type == geektime.ProductTypeUniversityVideo {
err := video.DownloadAliyunVodEncryptVideo(ctx, article.AID, currentProduct, projectDir, quality, concurrency)
checkError(err)
}
}
}

func isColumn() bool {
return currentProduct.Type == "c1"
return currentProduct.Type == geektime.ProductTypeColumn
}

func isVideo() bool {
return currentProduct.Type == "c3"
return currentProduct.Type == geektime.ProductTypeNormalVideo || currentProduct.Type == geektime.ProductTypeUniversityVideo
}

// Sets the bit at pos in the integer n.
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/chromedp/chromedp v0.8.5
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
github.com/go-resty/resty/v2 v2.7.0
github.com/google/uuid v1.3.0
github.com/spf13/cobra v1.5.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
)
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
Expand Down Expand Up @@ -109,6 +111,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
149 changes: 147 additions & 2 deletions internal/geektime/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,23 @@ const (
ArticleV1Path = "/serv/v1/article"
// ColumnInfoV3Path ...
ColumnInfoV3Path = "/serv/v3/column/info"
// PlayAuthV1Path ...
PlayAuthV1Path = "/serv/v1/video/play-auth"
// MyClassInfoV1Path ...
MyClassInfoV1Path = "/serv/v1/myclass/info"

// ProductTypeColumn c1 column
ProductTypeColumn = "c1"
// ProductTypeNormalVideo c3 normal video
ProductTypeNormalVideo = "c3"
// ProductTypeUniversityVideo u university video
ProductTypeUniversityVideo = "u"
)

var (
geekTimeClient *resty.Client
accountClient *resty.Client
geekTimeClient *resty.Client
accountClient *resty.Client
ugeekTimeClient *resty.Client
// SiteCookies ...
SiteCookies []*http.Cookie
)
Expand Down Expand Up @@ -68,6 +80,12 @@ type ArticleInfo struct {
AudioDownloadURL string
}

// PlayAuthInfo ...
type PlayAuthInfo struct {
PlayAuth string
VideoID string
}

// ColumnResponse ...
type ColumnResponse struct {
Code int `json:"code"`
Expand All @@ -78,6 +96,53 @@ type ColumnResponse struct {
} `json:"data"`
}

// PlayAuthResponse ...
type PlayAuthResponse struct {
Code int `json:"code"`
Data struct {
PlayAuth string `json:"play_auth"`
VID string `json:"vid"`
} `json:"data"`
}

// MyClassInfoResponse ...
type MyClassInfoResponse struct {
Code int `json:"code"`
Data struct {
ClassType int `json:"class_type"`
Title string `json:"title"`
Lessons []struct {
ChapterName string `json:"chapter_name"`
BeginTime int `json:"begin_time"`
ChapterID int `json:"chapter_id"`
IndexNo int `json:"index_no"`
Articles []struct {
ArticleID int `json:"article_id"`
ArticleTitle string `json:"article_title"`
IndexNo int `json:"index_no"`
IsRead bool `json:"is_read"`
IsFinish bool `json:"is_finish"`
// HasNotes bool `json:"has_notes"`
// IsRequired int `json:"is_required"`
VideoTime int `json:"video_time"`
// LearnTime int `json:"learn_time"`
// LearnStatus int `json:"learn_status"`
// MaxOffset int `json:"max_offset"`
// ArticleMaxOffset int `json:"article_max_offset"`
// VideoMaxOffset int `json:"video_max_offset"`
// ArticleLen int `json:"article_len"`
// VideoLen int `json:"video_len"`
// Ctime int `json:"ctime"`
// Exercises []interface{} `json:"exercises"`
} `json:"articles"`
} `json:"lessons"`
} `json:"data"`
Error struct {
Code int `json:"code"`
Msg string `json:"msg"`
} `json:"error"`
}

// VideoResponse ...
type VideoResponse struct {
Code int `json:"code"`
Expand Down Expand Up @@ -125,6 +190,14 @@ func InitClient(cookies []*http.Cookie) {
SetHeader(pgt.OriginHeaderName, pgt.GeekBang).
SetLogger(logger.DiscardLogger{})

ugeekTimeClient = resty.New().
SetBaseURL(pgt.GeekBangUniversity).
SetCookies(cookies).
SetTimeout(10*time.Second).
SetHeader(pgt.UserAgentHeaderName, pgt.UserAgentHeaderValue).
SetHeader(pgt.OriginHeaderName, pgt.GeekBangUniversity).
SetLogger(logger.DiscardLogger{})

SiteCookies = cookies
}

Expand Down Expand Up @@ -327,3 +400,75 @@ func Auth() error {
// status code 452
return pgt.ErrAuthFailed
}

// GetMyClassProduct ...
func GetMyClassProduct(classID int) (Product, error) {
var p Product
var resp MyClassInfoResponse
_, err := ugeekTimeClient.R().SetBody(
map[string]interface{}{
"class_id": classID,
}).
SetResult(&resp).
Post(MyClassInfoV1Path)

if err != nil {
return p, err
}

if resp.Code != 0 {
if resp.Error.Code == -5001 {
p.Access = false
return p, nil
}
return p, ErrGeekTimeAPIBadCode{PlayAuthV1Path, resp.Code, ""}
}

p = Product{
Access: true,
ID: classID,
Title: resp.Data.Title,
Type: ProductTypeUniversityVideo,
}
var articles []Article
for _, lesson := range resp.Data.Lessons {
for _, article := range lesson.Articles {
// ONLY download university video lessons
if article.VideoTime > 0 {
articles = append(articles, Article{
AID: article.ArticleID,
Title: article.ArticleTitle,
})
}
}
}
p.Articles = articles

return p, nil
}

// GetPlayAuth ...
func GetPlayAuth(articleID, classID int) (PlayAuthInfo, error) {
var info PlayAuthInfo
var result PlayAuthResponse
_, err := ugeekTimeClient.R().SetBody(
map[string]interface{}{
"article_id": articleID,
"class_id": classID,
}).
SetResult(&result).
Post(PlayAuthV1Path)

if err != nil {
return info, err
}

if result.Code != 0 {
return info, ErrGeekTimeAPIBadCode{PlayAuthV1Path, result.Code, ""}
}

return PlayAuthInfo{
PlayAuth: result.Data.PlayAuth,
VideoID: result.Data.VID,
}, nil
}
Loading

0 comments on commit 3da5f8c

Please sign in to comment.