diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 7395f85..bdbdfb7 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/setup-go@v5 with: - go-version: '1.18.0' + go-version: '1.22.0' - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v4.0.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a7ed76d..38e2de7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.18 + go-version: 1.22 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v4 with: diff --git a/.gitignore b/.gitignore index a7f0cbc..bde7941 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ .vscode dist/ + +.idea diff --git a/README.md b/README.md index baa4f08..fc4161f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,24 @@ # geektime-downloader -geektime-downloader 支持下载极客时间专栏(PDF/Markdown/音频)/视频课/每日一课/大厂实践/训练营视频。 +geektime-downloader 支持下载以下极客时间网站资源。 + +**极客时间** +- [x] 专栏(PDF/Markdown/音频) +- [x] 视频课 +- [x] 每日一课 +- [x] 大厂案例 +- [x] 训练营视频 +- [ ] 线下大会 + +**企业版极客时间** +- [ ] 体系课 +- [ ] 每日一课 +- [ ] 大厂案例 +- [ ] 生态课 +- [x] 训练营视频 + +部分资源暂未支持下载，欢迎PR。 + [![go report card](https://goreportcard.com/badge/github.com/nicoxiang/geektime-downloader "go report card")](https://goreportcard.com/report/github.com/nicoxiang/geektime-downloader) [![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT) @@ -49,15 +67,18 @@ Usage: geektime-downloader [flags] Flags: - --comments 是否需要专栏的第一页评论 (default true) - -f, --folder string 专栏和视频课的下载目标位置 (default "") - --gcess string 极客时间 cookie 值 gcess - --gcid string 极客时间 cookie 值 gcid - -h, --help help for geektime-downloader - --output int 专栏的输出内容(1pdf,2markdown,4audio)可自由组合 (default 1) - -u, --phone string 你的极客时间账号(手机号) - -q, --quality string 下载视频清晰度(ld标清,sd高清,hd超清) (default "sd") - --wait-seconds int Chrome生成PDF前的等待页面加载时间, 单位为秒, 默认8秒 (default 8) + --comments 是否需要专栏的第一页评论 (default true) + --enterprise 是否下载企业版极客时间资源 + -f, --folder string 专栏和视频课的下载目标位置 (default "C:\\Users\\nico\\geektime-downloader") + --gcess string 极客时间 cookie 值 gcess + --gcid string 极客时间 cookie 值 gcid + -h, --help help for geektime-downloader + --interval int 下载资源的间隔时间, 单位为秒, 默认1秒 (default 1) + --output int 专栏的输出内容(1pdf,2markdown,4audio)可自由组合 (default 1) + -u, --phone string 你的极客时间账号(手机号) + --print-pdf-timeout int Chrome生成PDF的超时时间, 单位为秒, 默认60秒 (default 60) + --print-pdf-wait int Chrome生成PDF前的等待页面加载时间, 单位为秒, 默认8秒 (default 8) + -q, --quality string 下载视频清晰度(ld标清,sd高清,hd超清) (default "sd") ``` ## Note @@ -110,14 +131,25 @@ https://time.geekbang.org/opencourse/videointro/100546701 **其他：** -打开极客时间[我的课程-其他](https://time.geekbang.org/dashboard/course)，选择你想要查看的会议，在新打开的会议详情 Tab 页，查看 URL ```course/intro/``` 最后的数字，例如下面的链接中 100551201 就是课程 ID： +打开极客时间[我的课程-其他](https://time.geekbang.org/dashboard/course)，选择你想要查看的课程，在新打开的课程详情 Tab 页，查看 URL ```course/intro/``` 最后的数字，例如下面的链接中 100551201 就是课程 ID： ``` https://time.geekbang.org/course/intro/100551201 ``` +**企业版训练营：** + +选择你想要查看的课程，查看 URL ```mall/product/```后的数字，例如下面的链接中 100618109 就是课程 ID： + +``` +https://b.geekbang.org/mall/product/100618109 +``` + ### 为什么我下载的PDF是空白页? -首先下载课程请保证VPN已关闭。在此前提下如果仍然出现空白页情况，说明后台Chrome网页加载速度较慢，可以尝试加大--wait-seconds参数，保证页面完全加载完成后再开始生成PDF。 +首先下载课程请保证VPN已关闭。在此前提下如果仍然出现空白页情况，说明后台Chrome网页加载速度较慢，可以尝试加大--print-pdf-wait参数，保证页面完全加载完成后再开始生成PDF。 + +### 为什么我下载PDF一直提示超时? +首先下载课程请保证VPN已关闭。在此前提下如果下载持续出现超时，有可能是因为课程章节图片等内容较多，生成速度慢，比如课程《AI 绘画核心技术与实战》中的部分章节，可以尝试加大--print-pdf-timeout参数，并耐心等待。 ### 如何下载专栏的 Markdown 格式和文章音频? diff --git a/cmd/root.go b/cmd/root.go index 3dc3741..ad39eb4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "math" "math/rand" "net/http" @@ -24,29 +23,31 @@ import ( "github.com/nicoxiang/geektime-downloader/internal/markdown" "github.com/nicoxiang/geektime-downloader/internal/pdf" "github.com/nicoxiang/geektime-downloader/internal/pkg/filenamify" + "github.com/nicoxiang/geektime-downloader/internal/pkg/files" "github.com/nicoxiang/geektime-downloader/internal/video" "github.com/spf13/cobra" "golang.org/x/net/html" ) var ( - phone string - gcid string - gcess string - concurrency int - downloadFolder string - sp *spinner.Spinner - selectedProduct geektime.Product - quality string - downloadComments bool - selectedProductType productTypeSelectOption - columnOutputType int - waitSeconds int - interval int - productTypeOptions = make([]productTypeSelectOption, 6) - geektimeClient *geektime.Client - accountClient *geektime.Client - universityClient *geektime.Client + phone string + gcid string + gcess string + concurrency int + downloadFolder string + sp *spinner.Spinner + selectedProduct geektime.Course + quality string + downloadComments bool + selectedProductType productTypeSelectOption + columnOutputType int + printPDFWaitSeconds int + printPDFTimeoutSeconds int + interval int + productTypeOptions []productTypeSelectOption + geektimeClient *geektime.Client + isEnterprise bool + waitRand = rand.New(rand.NewSource(time.Now().UnixNano())) ) type productTypeSelectOption struct { @@ -66,7 +67,7 @@ func init() { userHomeDir, _ := os.UserHomeDir() concurrency = int(math.Ceil(float64(runtime.NumCPU()) / 2.0)) defaultDownloadFolder := filepath.Join(userHomeDir, config.GeektimeDownloaderFolder) - setProductTypeOptions() + rootCmd.Flags().StringVarP(&phone, "phone", "u", "", "你的极客时间账号(手机号)") rootCmd.Flags().StringVar(&gcid, "gcid", "", "极客时间 cookie 值 gcid") rootCmd.Flags().StringVar(&gcess, "gcess", "", "极客时间 cookie 值 gcess") @@ -74,24 +75,29 @@ func init() { rootCmd.Flags().StringVarP(&quality, "quality", "q", "sd", "下载视频清晰度(ld标清,sd高清,hd超清)") rootCmd.Flags().BoolVar(&downloadComments, "comments", true, "是否需要专栏的第一页评论") rootCmd.Flags().IntVar(&columnOutputType, "output", 1, "专栏的输出内容(1pdf,2markdown,4audio)可自由组合") - rootCmd.Flags().IntVar(&waitSeconds, "wait-seconds", 8, "Chrome生成PDF前的等待页面加载时间, 单位为秒, 默认8秒") - rootCmd.Flags().IntVar(&interval, "interval", 0, "下载视频的间隔时间, 单位为秒, 默认0秒") + rootCmd.Flags().IntVar(&printPDFWaitSeconds, "print-pdf-wait", 8, "Chrome生成PDF前的等待页面加载时间, 单位为秒, 默认8秒") + rootCmd.Flags().IntVar(&printPDFTimeoutSeconds, "print-pdf-timeout", 60, "Chrome生成PDF的超时时间, 单位为秒, 默认60秒") + rootCmd.Flags().IntVar(&interval, "interval", 1, "下载资源的间隔时间, 单位为秒, 默认1秒") + rootCmd.Flags().BoolVar(&isEnterprise, "enterprise", false, "是否下载企业版极客时间资源") rootCmd.MarkFlagsMutuallyExclusive("phone", "gcid") rootCmd.MarkFlagsMutuallyExclusive("phone", "gcess") rootCmd.MarkFlagsRequiredTogether("gcid", "gcess") sp = spinner.New(spinner.CharSets[4], 100*time.Millisecond) - accountClient = geektime.NewAccountClient() } func setProductTypeOptions() { - productTypeOptions[0] = productTypeSelectOption{0, "普通课程", 1, []string{"c1", "c3"}, true} - productTypeOptions[1] = productTypeSelectOption{1, "每日一课", 2, []string{"d"}, false} - productTypeOptions[2] = productTypeSelectOption{2, "公开课", 1, []string{"p35", "p29", "p30"}, true} - productTypeOptions[3] = productTypeSelectOption{3, "大厂案例", 4, []string{"q"}, false} - productTypeOptions[4] = productTypeSelectOption{4, "训练营", 5, []string{""}, true} // custom source type, not use - productTypeOptions[5] = productTypeSelectOption{5, "其他", 1, []string{"x", "c6"}, true} + if isEnterprise { + productTypeOptions = append(productTypeOptions, productTypeSelectOption{0, "训练营", 5, []string{"c44"}, true}) //custom source type, not use + } else { + productTypeOptions = append(productTypeOptions, productTypeSelectOption{0, "普通课程", 1, []string{"c1", "c3"}, true}) + productTypeOptions = append(productTypeOptions, productTypeSelectOption{1, "每日一课", 2, []string{"d"}, false}) + productTypeOptions = append(productTypeOptions, productTypeSelectOption{2, "公开课", 1, []string{"p35", "p29", "p30"}, true}) + productTypeOptions = append(productTypeOptions, productTypeSelectOption{3, "大厂案例", 4, []string{"q"}, false}) + productTypeOptions = append(productTypeOptions, productTypeSelectOption{4, "训练营", 5, []string{""}, true}) //custom source type, not use + productTypeOptions = append(productTypeOptions, productTypeSelectOption{5, "其他", 1, []string{"x", "c6"}, true}) + } } var rootCmd = &cobra.Command{ @@ -130,7 +136,7 @@ var rootCmd = &cobra.Command{ checkError(err) sp.Prefix = "[ 正在登录... ]" sp.Start() - readCookies, err = accountClient.Login(phone, pwd) + readCookies, err = geektime.Login(phone, pwd) if err != nil { sp.Stop() checkError(err) @@ -142,11 +148,12 @@ var rootCmd = &cobra.Command{ } // first time auth check - if err := accountClient.Auth(readCookies); err != nil { + if err := geektime.Auth(readCookies); err != nil { checkError(err) } + geektimeClient = geektime.NewClient(readCookies) - universityClient = geektime.NewUniversityClient(readCookies) + setProductTypeOptions() selectProductType(cmd.Context()) }, } @@ -228,14 +235,17 @@ func letInputProductID(ctx context.Context) { func loadProduct(ctx context.Context, productID int) { sp.Prefix = "[ 正在加载课程信息... ]" sp.Start() - var p geektime.Product + var p geektime.Course var err error if isUniversity() { - p, err = universityClient.MyClassProduct(productID) // university don't need check product type // if input invalid id, access mark is 0 + p, err = geektimeClient.UniversityCourseInfo(productID) + } else if isEnterprise { + // TODO: check enterprise course type + p, err = geektimeClient.EnterpriseCourseInfo(productID) } else { - p, err = geektimeClient.ColumnInfo(productID) + p, err = geektimeClient.CourseInfo(productID) if err == nil { c := checkProductType(p.Type) // if check product type fail, re-input product @@ -295,7 +305,6 @@ func productOps(ctx context.Context) { } func selectArticle(ctx context.Context) { - loadArticles() items := []geektime.Article{ { AID: -1, @@ -337,189 +346,137 @@ func handleSelectArticle(ctx context.Context, index int) { } func handleDownloadAll(ctx context.Context) { - loadArticles() projectDir, err := mkDownloadProjectDir(downloadFolder, phone, gcid, selectedProduct.Title) checkError(err) - downloaded, err := findDownloadedArticleFileNames(projectDir) - checkError(err) if isText() { - rand.Seed(time.Now().UnixNano()) fmt.Printf("正在下载专栏 《%s》 中的所有文章\n", selectedProduct.Title) total := len(selectedProduct.Articles) var i int - needDownloadPDF := columnOutputType&1 == 1 - needDownloadMD := (columnOutputType>>1)&1 == 1 - needDownloadAudio := (columnOutputType>>2)&1 == 1 - - for _, a := range selectedProduct.Articles { - fileName := filenamify.Filenamify(a.Title) - var b int - if _, exists := downloaded[fileName+pdf.PDFExtension]; exists { - b = setBit(b, 0) - } - if _, exists := downloaded[fileName+markdown.MDExtension]; exists { - b = setBit(b, 1) - } - if _, exists := downloaded[fileName+audio.MP3Extension]; exists { - b = setBit(b, 2) - } - - if b == columnOutputType { - increasePDFCount(total, &i) - continue - } - - articleInfo, err := geektimeClient.V1ArticleInfo(a.AID) - checkError(err) - - hasVideo, videoURL := getVideoURLFromArticleContent(articleInfo.Data.ArticleContent) - - if hasVideo && videoURL != "" { - err = video.DownloadMP4(ctx, a.Title, projectDir, []string{videoURL}) - } - - if len(articleInfo.Data.InlineVideoSubtitles) > 0 { - videoURLs := make([]string, len(articleInfo.Data.InlineVideoSubtitles)) - for i, v := range articleInfo.Data.InlineVideoSubtitles { - videoURLs[i] = v.VideoURL - } - err = video.DownloadMP4(ctx, a.Title, projectDir, videoURLs) - } - - if needDownloadPDF { - err = pdf.PrintArticlePageToPDF(ctx, - a.AID, - projectDir, - a.Title, - geektimeClient.Cookies, - downloadComments, - waitSeconds, - ) - if err != nil { - checkError(err) - } - } - - if needDownloadMD { - err = markdown.Download(ctx, - articleInfo.Data.ArticleContent, - a.Title, - projectDir, - a.AID) + for _, article := range selectedProduct.Articles { + skipped := downloadTextArticle(ctx, article, projectDir, false) + increaseDownloadedTextArticleCount(total, &i) + if !skipped { + waitRandomTime() } - - if needDownloadAudio { - err = audio.DownloadAudio(ctx, articleInfo.Data.AudioDownloadURL, projectDir, a.Title) - } - - checkError(err) - - increasePDFCount(total, &i) - waitRandomTime() } } else { - for _, a := range selectedProduct.Articles { - fileName := filenamify.Filenamify(a.Title) + video.TSExtension - if _, ok := downloaded[fileName]; ok { - continue - } - if isUniversity() { - err := video.DownloadUniversityVideo(ctx, universityClient, a.AID, selectedProduct, projectDir, quality, concurrency) - checkError(err) - } else { - err := video.DownloadArticleVideo(ctx, geektimeClient, a.AID, selectedProductType.SourceType, projectDir, quality, concurrency) - checkError(err) + for _, article := range selectedProduct.Articles { + skipped := downloadVideoArticle(ctx, article, projectDir, false) + if !skipped { + waitRandomTime() } - waitRandomTime() } } selectProductType(ctx) } -func increasePDFCount(total int, i *int) { - (*i)++ +func increaseDownloadedTextArticleCount(total int, i *int) { + *i++ fmt.Printf("\r已完成下载%d/%d", *i, total) } -func loadArticles() { - if !isUniversity() && len(selectedProduct.Articles) <= 0 { - sp.Prefix = "[ 正在加载文章列表... ]" +func downloadArticle(ctx context.Context, article geektime.Article, projectDir string) { + if isText() { + sp.Prefix = fmt.Sprintf("[ 正在下载 《%s》... ]", article.Title) sp.Start() - articles, err := geektimeClient.ColumnArticles(strconv.Itoa(selectedProduct.ID)) - checkError(err) - selectedProduct.Articles = articles - sp.Stop() + defer sp.Stop() + downloadTextArticle(ctx, article, projectDir, true) + } else { + downloadVideoArticle(ctx, article, projectDir, true) } } -func downloadArticle(ctx context.Context, article geektime.Article, projectDir string) { - if isText() { - needDownloadPDF := columnOutputType&1 == 1 - needDownloadMD := (columnOutputType>>1)&1 == 1 - needDownloadAudio := (columnOutputType>>2)&1 == 1 +func downloadTextArticle(ctx context.Context, article geektime.Article, projectDir string, overwrite bool) bool { + needDownloadPDF := columnOutputType&1 == 1 + needDownloadMD := (columnOutputType>>1)&1 == 1 + needDownloadAudio := (columnOutputType>>2)&1 == 1 + skipped := true + + articleInfo, err := geektimeClient.V1ArticleInfo(article.AID) + checkError(err) - articleInfo, err := geektimeClient.V1ArticleInfo(article.AID) + hasVideo, videoURL := getVideoURLFromArticleContent(articleInfo.Data.ArticleContent) + if hasVideo && videoURL != "" { + err = video.DownloadMP4(ctx, article.Title, projectDir, []string{videoURL}, overwrite) checkError(err) + } - sp.Prefix = fmt.Sprintf("[ 正在下载 《%s》... ]", article.Title) - hasVideo, videoURL := getVideoURLFromArticleContent(articleInfo.Data.ArticleContent) - if len(articleInfo.Data.InlineVideoSubtitles) > 0 || hasVideo && videoURL != "" { - sp.Prefix = fmt.Sprintf("[ 正在下载 《%s》, 该文章中包含视频, 请耐心等待... ]", article.Title) + if len(articleInfo.Data.InlineVideoSubtitles) > 0 { + videoURLs := make([]string, len(articleInfo.Data.InlineVideoSubtitles)) + for i, v := range articleInfo.Data.InlineVideoSubtitles { + videoURLs[i] = v.VideoURL } - sp.Start() + err = video.DownloadMP4(ctx, article.Title, projectDir, videoURLs, overwrite) + checkError(err) + } - if hasVideo && videoURL != "" { - err = video.DownloadMP4(ctx, article.Title, projectDir, []string{videoURL}) + if needDownloadPDF { + innerSkipped, err := pdf.PrintArticlePageToPDF(ctx, + article.AID, + projectDir, + article.Title, + geektimeClient.Cookies, + downloadComments, + printPDFWaitSeconds, + printPDFTimeoutSeconds, + overwrite, + ) + if !innerSkipped { + skipped = false } + checkError(err) + } - if len(articleInfo.Data.InlineVideoSubtitles) > 0 { - videoURLs := make([]string, len(articleInfo.Data.InlineVideoSubtitles)) - for i, v := range articleInfo.Data.InlineVideoSubtitles { - videoURLs[i] = v.VideoURL - } - err = video.DownloadMP4(ctx, article.Title, projectDir, videoURLs) + if needDownloadMD { + innerSkipped, err := markdown.Download(ctx, + articleInfo.Data.ArticleContent, + article.Title, + projectDir, + article.AID, + overwrite) + if !innerSkipped { + skipped = false } + checkError(err) + } - if needDownloadPDF { - checkError(err) - err = pdf.PrintArticlePageToPDF(ctx, - article.AID, - projectDir, - article.Title, - geektimeClient.Cookies, - downloadComments, - waitSeconds, - ) - if err != nil { - sp.Stop() - checkError(err) - } + if needDownloadAudio { + innerSkipped, err := audio.DownloadAudio(ctx, articleInfo.Data.AudioDownloadURL, projectDir, article.Title, overwrite) + if !innerSkipped { + skipped = false } + checkError(err) + } + return skipped +} - if needDownloadMD { - err = markdown.Download(ctx, - articleInfo.Data.ArticleContent, - article.Title, - projectDir, - article.AID) - } +func downloadVideoArticle(ctx context.Context, article geektime.Article, projectDir string, overwrite bool) bool { + dir := projectDir + var err error + // add sub dir + if article.SectionTitle != "" { + dir, err = mkDownloadProjectSectionDir(projectDir, article.SectionTitle) + checkError(err) + } - if needDownloadAudio { - err = audio.DownloadAudio(ctx, articleInfo.Data.AudioDownloadURL, projectDir, article.Title) - } + fileName := filenamify.Filenamify(article.Title) + video.TSExtension + fullPath := filepath.Join(dir, fileName) + if files.CheckFileExists(fullPath) && !overwrite { + return true + } - sp.Stop() + if isUniversity() { + err = video.DownloadUniversityVideo(ctx, geektimeClient, article.AID, selectedProduct, dir, quality, concurrency) + checkError(err) + } else if isEnterprise { + err = video.DownloadEnterpriseArticleVideo(ctx, geektimeClient, article.AID, dir, quality, concurrency) checkError(err) } else { - if isUniversity() { - err := video.DownloadUniversityVideo(ctx, universityClient, article.AID, selectedProduct, projectDir, quality, concurrency) - checkError(err) - } else { - err := video.DownloadArticleVideo(ctx, geektimeClient, article.AID, selectedProductType.SourceType, projectDir, quality, concurrency) - checkError(err) - } + err = video.DownloadArticleVideo(ctx, geektimeClient, article.AID, selectedProductType.SourceType, dir, quality, concurrency) + checkError(err) } + return false } func isText() bool { @@ -527,13 +484,7 @@ func isText() bool { } func isUniversity() bool { - return selectedProductType.Index == 4 -} - -// Sets the bit at pos in the integer n. -func setBit(n int, pos uint) int { - n |= (1 << pos) - return n + return selectedProductType.Index == 4 && !isEnterprise } func readCookiesFromInput() []*http.Cookie { @@ -556,21 +507,6 @@ func readCookiesFromInput() []*http.Cookie { return cookies } -func findDownloadedArticleFileNames(projectDir string) (map[string]struct{}, error) { - files, err := ioutil.ReadDir(projectDir) - res := make(map[string]struct{}, len(files)) - if err != nil { - return res, err - } - if len(files) == 0 { - return res, nil - } - for _, f := range files { - res[f.Name()] = struct{}{} - } - return res, nil -} - func mkDownloadProjectDir(downloadFolder, phone, gcid, projectName string) (string, error) { userName := phone if gcid != "" { @@ -584,6 +520,15 @@ func mkDownloadProjectDir(downloadFolder, phone, gcid, projectName string) (stri return path, nil } +func mkDownloadProjectSectionDir(downloadFolder, sectionName string) (string, error) { + path := filepath.Join(downloadFolder, filenamify.Filenamify(sectionName)) + err := os.MkdirAll(path, os.ModePerm) + if err != nil { + return "", err + } + return path, nil +} + func checkProductType(productType string) bool { for _, pt := range selectedProductType.AcceptProductTypes { if pt == productType { @@ -633,8 +578,8 @@ func getVideoURLFromArticleContent(content string) (hasVideo bool, videoURL stri // waitRandomTime wait interval seconds of time plus a 2000ms max jitter func waitRandomTime() { - r := interval*1000 + rand.Intn(2000) - time.Sleep(time.Duration(r) * time.Millisecond) + randomMillis := interval*1000 + waitRand.Intn(2000) + time.Sleep(time.Duration(randomMillis) * time.Millisecond) } // Execute ... diff --git a/go.mod b/go.mod index 165549c..0dc854e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/nicoxiang/geektime-downloader -go 1.18 +go 1.22 require ( github.com/JohannesKaufmann/html-to-markdown v1.5.0 diff --git a/internal/audio/audio.go b/internal/audio/audio.go index 3cbd426..b9d3bb5 100644 --- a/internal/audio/audio.go +++ b/internal/audio/audio.go @@ -8,6 +8,7 @@ import ( "github.com/nicoxiang/geektime-downloader/internal/geektime" "github.com/nicoxiang/geektime-downloader/internal/pkg/downloader" "github.com/nicoxiang/geektime-downloader/internal/pkg/filenamify" + "github.com/nicoxiang/geektime-downloader/internal/pkg/files" ) const ( @@ -16,14 +17,18 @@ const ( ) // DownloadAudio ... -func DownloadAudio(ctx context.Context, downloadAudioURL, dir, title string) error { +func DownloadAudio(ctx context.Context, downloadAudioURL, dir, title string, overwrite bool) (bool, error) { if downloadAudioURL == "" { - return nil + return false, nil } filenamifyTitle := filenamify.Filenamify(title) dst := filepath.Join(dir, filenamifyTitle+MP3Extension) + if files.CheckFileExists(dst) && !overwrite { + return true, nil + } + headers := make(map[string]string, 2) headers[geektime.Origin] = geektime.DefaultBaseURL headers[geektime.UserAgent] = geektime.DefaultUserAgent @@ -34,5 +39,5 @@ func DownloadAudio(ctx context.Context, downloadAudioURL, dir, title string) err _ = os.Remove(dst) } - return err + return false, err } diff --git a/internal/config/config.go b/internal/config/config.go index 78f01d9..3878fff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,7 +2,6 @@ package config import ( "errors" - "io/ioutil" "net/http" "os" "path/filepath" @@ -26,7 +25,7 @@ func init() { // ReadCookieFromConfigFile read cookies from app config file. func ReadCookieFromConfigFile(phone string) ([]*http.Cookie, error) { dir := filepath.Join(userConfigDir, GeektimeDownloaderFolder) - files, err := ioutil.ReadDir(dir) + files, err := os.ReadDir(dir) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, nil @@ -45,7 +44,7 @@ func ReadCookieFromConfigFile(phone string) ([]*http.Cookie, error) { fullName := filepath.Join(userConfigDir, GeektimeDownloaderFolder, fi.Name()) var cookies []*http.Cookie - data, err := ioutil.ReadFile(fullName) + data, err := os.ReadFile(fullName) if err != nil { return nil, err } @@ -79,9 +78,9 @@ func WriteCookieToConfigFile(phone string, cookies []*http.Cookie) error { if err := os.MkdirAll(dir, os.ModePerm); err != nil { return err } - removeConfig(dir, phone) + _ = removeConfig(dir, phone) - file, err := ioutil.TempFile(dir, phone) + file, err := os.CreateTemp(dir, phone) if err != nil { return err } @@ -103,7 +102,7 @@ func RemoveConfig(phone string) error { } func removeConfig(dir, phone string) error { - files, err := ioutil.ReadDir(dir) + files, err := os.ReadDir(dir) if err != nil { return err } @@ -121,7 +120,7 @@ func removeConfig(dir, phone string) error { } } } - return nil; + return nil } func writeOnelineConfig(sb strings.Builder, cookie *http.Cookie) strings.Builder { diff --git a/internal/geektime/account.go b/internal/geektime/account.go new file mode 100644 index 0000000..3272f3c --- /dev/null +++ b/internal/geektime/account.go @@ -0,0 +1,121 @@ +package geektime + +import ( + "fmt" + "net/http" + "time" + + "github.com/go-resty/resty/v2" + "github.com/nicoxiang/geektime-downloader/internal/pkg/logger" +) + +const ( + // GeekBangAccountBaseURL ... + GeekBangAccountBaseURL = "https://account.geekbang.org" + // LoginPath ... + LoginPath = "/account/ticket/login" + // V1AuthPath ... + V1AuthPath = "/serv/v1/user/auth" +) + +// Login call geektime login api and return auth cookies +func Login(phone, password string) ([]*http.Cookie, error) { + var res struct { + Code int `json:"code"` + Data struct { + UID int `json:"uid"` + Name string `json:"nickname"` + } `json:"data"` + Error struct { + Code int `json:"code"` + Msg string `json:"msg"` + } `json:"error"` + } + + client := resty.New(). + SetTimeout(DefaultTimeout). + SetHeader(UserAgent, DefaultUserAgent). + SetLogger(logger.DiscardLogger{}) + + logger.Infof("Login request start") + + resp, err := client.R(). + SetHeader(Origin, DefaultBaseURL). + SetBody(map[string]interface{}{ + "country": 86, + "appid": 1, + "platform": 3, + "cellphone": phone, + "password": password, + }). + SetResult(&res). + Post(GeekBangAccountBaseURL + LoginPath) + + if err != nil { + return nil, err + } + + if resp.RawResponse.StatusCode != 200 || res.Code != 0 { + logger.Warnf("Login request end, status code: %d, response body: %s", + resp.RawResponse.StatusCode, + resp.String(), + ) + } + + if res.Code == 0 { + var cookies []*http.Cookie + for _, c := range resp.Cookies() { + if c.Name == GCID || c.Name == GCESS { + cookies = append(cookies, c) + } + } + return cookies, nil + } else if res.Error.Code == -3031 { + return nil, ErrWrongPassword + } else if res.Error.Code == -3005 { + return nil, ErrTooManyLoginAttemptTimes + } + return nil, ErrGeekTimeAPIBadCode{LoginPath, resp.String()} +} + +// Auth check if current user login is expired or login in another device +func Auth(cs []*http.Cookie) error { + var res struct { + Code int `json:"code"` + } + t := fmt.Sprintf("%v", time.Now().Round(time.Millisecond).UnixNano()/(int64(time.Millisecond)/int64(time.Nanosecond))) + params := make(map[string]string, 2) + params["t"] = t + params["v_t"] = t + + client := resty.New(). + SetTimeout(DefaultTimeout). + SetHeader(UserAgent, DefaultUserAgent). + SetLogger(logger.DiscardLogger{}) + + logger.Infof("Auth request start") + + resp, err := client.R(). + SetQueryParams(params). + SetCookies(cs). + SetHeader(Origin, DefaultBaseURL). + SetResult(&res). + Get(GeekBangAccountBaseURL + V1AuthPath) + + if err != nil { + return err + } + + if resp.RawResponse.StatusCode != 200 || res.Code != 0 { + logger.Warnf("Auth request end, status code: %d, response body: %s", + resp.RawResponse.StatusCode, + resp.String(), + ) + + // result Code -1 + // {\"error\":{\"msg\":\"未登录\",\"code\":-2000} + return ErrAuthFailed + } + + return nil +} diff --git a/internal/geektime/client.go b/internal/geektime/client.go new file mode 100644 index 0000000..65923c6 --- /dev/null +++ b/internal/geektime/client.go @@ -0,0 +1,136 @@ +package geektime + +import ( + "errors" + "fmt" + "net/http" + "reflect" + "time" + + "github.com/go-resty/resty/v2" + "github.com/nicoxiang/geektime-downloader/internal/pkg/logger" +) + +const ( + DefaultTimeout = 10 * time.Second + // Origin ... + Origin = "Origin" + // UserAgent ... + UserAgent = "User-Agent" + // DefaultUserAgent ... + DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36" +) + +// A Client manages communication with the Geektime API. +type Client struct { + RestyClient *resty.Client + Cookies []*http.Cookie +} + +// ErrGeekTimeAPIBadCode ... +type ErrGeekTimeAPIBadCode struct { + Path string + ResponseString string +} + +// Error implements error interface +func (e ErrGeekTimeAPIBadCode) Error() string { + return fmt.Sprintf("请求极客时间接口 %s 失败, ResponseBody: %s", e.Path, e.ResponseString) +} + +var ( + // ErrWrongPassword ... + ErrWrongPassword = errors.New("密码错误, 请尝试重新登录") + // ErrTooManyLoginAttemptTimes ... + ErrTooManyLoginAttemptTimes = errors.New("密码输入错误次数过多，已触发验证码校验，请稍后再试") + // ErrGeekTimeRateLimit ... + ErrGeekTimeRateLimit = errors.New("已触发限流, 你可以选择重新登录/重新获取 cookie, 或者稍后再试, 然后生成剩余的文章") + // ErrAuthFailed ... + ErrAuthFailed = errors.New("当前账户在其他设备登录或者登录已经过期, 请尝试重新登录") +) + +// NewClient returns a new Geektime API client. +func NewClient(cs []*http.Cookie) *Client { + restyClient := resty.New(). + SetCookies(cs). + SetRetryCount(1). + SetTimeout(DefaultTimeout). + SetHeader(UserAgent, DefaultUserAgent). + SetLogger(logger.DiscardLogger{}) + + c := &Client{RestyClient: restyClient, Cookies: cs} + return c +} + +// newRequest new http request +func (c *Client) newRequest( + method string, + baseURL string, + path string, + params map[string]string, + body interface{}, + result interface{}) *resty.Request { + r := c.RestyClient.R() + r.Method = method + r.URL = baseURL + path + r.SetHeader(Origin, baseURL) + if len(params) > 0 { + r.SetQueryParams(params) + } + if body != nil { + r.SetBody(body) + } + r.SetResult(result) + return r +} + +// do perform http request +func do(request *resty.Request) (*resty.Response, error) { + logger.Infof("Http request start, method: %s, url: %s, request body: %v", + request.Method, + request.URL, + request.Body, + ) + resp, err := request.Execute(request.Method, request.URL) + + if err != nil { + return nil, err + } + + statusCode := resp.RawResponse.StatusCode + + logger.Infof("Http request end, method: %s, url: %s, status code: %d", + resp.RawResponse.Request.Method, + resp.RawResponse.Request.URL, + resp.RawResponse.StatusCode, + ) + + if statusCode != 200 { + logNotOkResponse(resp) + if statusCode == 451 { + return nil, ErrGeekTimeRateLimit + } else if statusCode == 452 { + return nil, ErrAuthFailed + } + } + + rv := reflect.ValueOf(request.Result) + f := reflect.Indirect(rv).FieldByName("Code") + code := int(f.Int()) + + if code == 0 { + return resp, nil + } + + logNotOkResponse(resp) + //未登录或者已失效 + if code == -3050 || code == -2000 { + return nil, ErrAuthFailed + } + + return nil, ErrGeekTimeAPIBadCode{request.URL, resp.String()} +} + +func logNotOkResponse(resp *resty.Response) { + logger.Warnf("Http request not ok, response body: %s", resp.String()) +} diff --git a/internal/geektime/enterprise.go b/internal/geektime/enterprise.go new file mode 100644 index 0000000..edaf157 --- /dev/null +++ b/internal/geektime/enterprise.go @@ -0,0 +1,138 @@ +package geektime + +import ( + "strconv" + + "github.com/go-resty/resty/v2" + "github.com/nicoxiang/geektime-downloader/internal/geektime/response" +) + +const ( + // GeekBangEnterpriseBaseURL is geekbang enterprise base URL + GeekBangEnterpriseBaseURL = "https://b.geekbang.org" + // V1EnterpriseCourseInfoPath used in enterprise course product info + V1EnterpriseCourseInfoPath = "/app/v1/course/info" + // V1EnterpriseArticlesInfoPath used in enterprise course articles info + V1EnterpriseArticlesInfoPath = "/app/v1/course/articles" + // V1EnterpriseArticleDetailPath used in enterprise course article detail info + V1EnterpriseArticleDetailPath = "/app/v1/article/detail" + // V1EnterpriseVideoPlayAuthPath used in enterprise course video play auth + V1EnterpriseVideoPlayAuthPath = "/app/v1/source_auth/video_play_auth" +) + +// EnterpriseCourseInfo get enterprise course info +func (c *Client) EnterpriseCourseInfo(id int) (Course, error) { + var p Course + var err error + p, err = c.enterpriseCourseInfo(id) + if err != nil { + return p, err + } + + var articles []Article + articles, err = c.enterpriseCourseArticles(id) + if err != nil { + return p, err + } + p.Articles = articles + + return p, nil +} + +// V1EnterpriseArticleDetail get enterprise article detail +func (c *Client) V1EnterpriseArticleDetail(articleID string) (response.V1EnterpriseArticlesDetailResponse, error) { + var res response.V1EnterpriseArticlesDetailResponse + r := c.newRequest( + resty.MethodPost, + GeekBangEnterpriseBaseURL, + V1EnterpriseArticleDetailPath, + nil, + map[string]interface{}{ + "article_id": articleID, + }, + &res, + ) + if _, err := do(r); err != nil { + return response.V1EnterpriseArticlesDetailResponse{}, err + } + return res, nil +} + +// EnterpriseVideoPlayAuth get enterprise play auth string +func (c *Client) EnterpriseVideoPlayAuth(articleID, videoID string) (string, error) { + var res response.V3VideoPlayAuthResponse + r := c.newRequest( + resty.MethodPost, + GeekBangEnterpriseBaseURL, + V1EnterpriseVideoPlayAuthPath, + nil, + map[string]interface{}{ + "aid": articleID, + "video_id": videoID, + }, + &res, + ) + if _, err := do(r); err != nil { + return "", err + } + return res.Data.PlayAuth, nil +} + +func (c *Client) enterpriseCourseInfo(productID int) (Course, error) { + var res response.V1EnterpriseProductInfoResponse + + r := c.newRequest( + resty.MethodPost, + GeekBangEnterpriseBaseURL, + V1EnterpriseCourseInfoPath, + nil, + map[string]interface{}{ + "id": productID, + }, + &res, + ) + + if _, err := do(r); err != nil { + return Course{}, err + } + + return Course{ + Access: res.Data.Extra.IsMyCourse, + ID: productID, + Title: res.Data.Title, + Type: "", + IsVideo: true, + }, nil +} + +func (c *Client) enterpriseCourseArticles(productID int) ([]Article, error) { + var res response.V1EnterpriseArticlesResponse + r := c.newRequest( + resty.MethodPost, + GeekBangEnterpriseBaseURL, + V1EnterpriseArticlesInfoPath, + nil, + map[string]interface{}{ + "id": productID, + }, + &res, + ) + + if _, err := do(r); err != nil { + return nil, err + } + + var articles []Article + + for _, sections := range res.Data.List { + for _, a := range sections.ArticleList { + articleID, _ := strconv.Atoi(a.Article.ID) + articles = append(articles, Article{ + AID: articleID, + SectionTitle: sections.Title, + Title: a.Article.Title, + }) + } + } + return articles, nil +} diff --git a/internal/geektime/geektime.go b/internal/geektime/geektime.go index 4f2e55f..9053e08 100644 --- a/internal/geektime/geektime.go +++ b/internal/geektime/geektime.go @@ -1,40 +1,20 @@ package geektime import ( - "errors" - "fmt" - "net/http" - "reflect" "strconv" - "time" "github.com/go-resty/resty/v2" "github.com/nicoxiang/geektime-downloader/internal/geektime/response" - "github.com/nicoxiang/geektime-downloader/internal/pkg/logger" ) const ( // DefaultBaseURL ... DefaultBaseURL = "https://time.geekbang.org" - // DefaultUserAgent ... - DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36" - // Origin ... - Origin = "Origin" - // UserAgent ... - UserAgent = "User-Agent" - // GeekBangUniversityBaseURL ... - GeekBangUniversityBaseURL = "https://u.geekbang.org" - // GeekBangAccountBaseURL ... - GeekBangAccountBaseURL = "https://account.geekbang.org" - // LoginPath ... - LoginPath = "/account/ticket/login" - // V1AuthPath ... - V1AuthPath = "/serv/v1/user/auth" + // V1ColumnArticlesPath get all articles summary info in one column V1ColumnArticlesPath = "/serv/v1/column/articles" // V1ArticlePath used in normal column V1ArticlePath = "/serv/v1/article" - // V3ColumnInfoPath used in get normal column/video info V3ColumnInfoPath = "/serv/v3/column/info" // V3ProductInfoPath used in get daily lesson, qconplus product info @@ -44,11 +24,6 @@ const ( // V3VideoPlayAuthPath used in normal video, daily lesson, qconplus video play auth V3VideoPlayAuthPath = "/serv/v3/source_auth/video_play_auth" - // UniversityV1VideoPlayAuthPath used in university video play auth - UniversityV1VideoPlayAuthPath = "/serv/v1/video/play-auth" - // UniversityV1MyClassInfoPath get university class info and all articles info in it - UniversityV1MyClassInfoPath = "/serv/v1/myclass/info" - // GeekBangCookieDomain ... GeekBangCookieDomain = ".geekbang.org" @@ -58,19 +33,8 @@ const ( GCESS = "GCESS" ) -var ( - // ErrWrongPassword ... - ErrWrongPassword = errors.New("密码错误, 请尝试重新登录") - // ErrTooManyLoginAttemptTimes ... - ErrTooManyLoginAttemptTimes = errors.New("密码输入错误次数过多，已触发验证码校验，请稍后再试") - // ErrGeekTimeRateLimit ... - ErrGeekTimeRateLimit = errors.New("已触发限流, 你可以选择重新登录/重新获取 cookie, 或者稍后再试, 然后生成剩余的文章") - // ErrAuthFailed ... - ErrAuthFailed = errors.New("当前账户在其他设备登录或者登录已经过期, 请尝试重新登录") -) - -// Product ... -type Product struct { +// Course ... +type Course struct { Access bool ID int Title string @@ -81,220 +45,36 @@ type Product struct { // Article ... type Article struct { - AID int - Title string -} - -// ErrGeekTimeAPIBadCode ... -type ErrGeekTimeAPIBadCode struct { - Path string - ResponseString string -} - -// Error implements error interface -func (e ErrGeekTimeAPIBadCode) Error() string { - return fmt.Sprintf("请求极客时间接口 %s 失败, ResponseBody: %s", e.Path, e.ResponseString) -} - -// A Client manages communication with the Geektime API. -type Client struct { - HTTPClient *resty.Client - BaseURL string - Cookies []*http.Cookie -} - -// NewClient returns a new Geektime API client. -func NewClient(cs []*http.Cookie) *Client { - httpClient := resty.New(). - SetCookies(cs). - SetRetryCount(1). - SetTimeout(10*time.Second). - SetHeader("User-Agent", DefaultUserAgent). - SetLogger(logger.DiscardLogger{}) - - c := &Client{HTTPClient: httpClient, BaseURL: DefaultBaseURL, Cookies: cs} - return c -} - -// NewAccountClient ... -func NewAccountClient() *Client { - httpClient := resty.New(). - SetRetryCount(1). - SetTimeout(10*time.Second). - SetHeader("User-Agent", DefaultUserAgent). - SetLogger(logger.DiscardLogger{}) - - c := &Client{HTTPClient: httpClient, BaseURL: GeekBangAccountBaseURL} - return c -} - -// NewUniversityClient ... -func NewUniversityClient(cs []*http.Cookie) *Client { - httpClient := resty.New(). - SetCookies(cs). - SetRetryCount(1). - SetTimeout(10*time.Second). - SetHeader("User-Agent", DefaultUserAgent). - SetLogger(logger.DiscardLogger{}) - - c := &Client{HTTPClient: httpClient, BaseURL: GeekBangUniversityBaseURL, Cookies: cs} - return c + AID int + SectionTitle string + Title string } -// Login call geektime login api and return auth cookies -func (c *Client) Login(phone, password string) ([]*http.Cookie, error) { - var res struct { - Code int `json:"code"` - Data struct { - UID int `json:"uid"` - Name string `json:"nickname"` - } `json:"data"` - Error struct { - Code int `json:"code"` - Msg string `json:"msg"` - } `json:"error"` - } - - r := c.newRequest(resty.MethodPost, - LoginPath, - nil, - map[string]interface{}{ - "country": 86, - "appid": 1, - "platform": 3, - "cellphone": phone, - "password": password, - }, - &res, - ) - - logger.Infof("Login request start") - resp, err := r.Execute(r.Method, r.URL) - +// CourseInfo get narmal geektime course info +func (c *Client) CourseInfo(productID int) (Course, error) { + var p Course + var err error + p, err = c.columnInfo(productID) if err != nil { - return nil, err - } - - if resp.RawResponse.StatusCode != 200 || res.Code != 0 { - logger.Warnf("Login request end, status code: %d, response body: %s", - resp.RawResponse.StatusCode, - resp.String(), - ) - } - - if res.Code == 0 { - var cookies []*http.Cookie - for _, c := range resp.Cookies() { - if c.Name == GCID || c.Name == GCESS { - cookies = append(cookies, c) - } - } - return cookies, nil - } else if res.Error.Code == -3031 { - return nil, ErrWrongPassword - } else if res.Error.Code == -3005 { - return nil, ErrTooManyLoginAttemptTimes - } - return nil, ErrGeekTimeAPIBadCode{LoginPath, resp.String()} -} - -// Auth check if current user login is expired or login in another device -func (c *Client) Auth(cs []*http.Cookie) error { - var res struct { - Code int `json:"code"` + return p, err } - t := fmt.Sprintf("%v", time.Now().Round(time.Millisecond).UnixNano()/(int64(time.Millisecond)/int64(time.Nanosecond))) - c.HTTPClient.SetCookies(cs) - params := make(map[string]string, 2) - params["t"] = t - params["v_t"] = t - r := c.newRequest(resty.MethodGet, - V1AuthPath, - params, - nil, - res, - ) - r.SetHeader(Origin, DefaultBaseURL) - - logger.Infof("Auth request start") - resp, err := r.Execute(r.Method, r.URL) + var articles []Article + articles, err = c.columnArticles(productID) if err != nil { - return err - } - - if resp.RawResponse.StatusCode != 200 || res.Code != 0 { - logger.Warnf("Auth request end, status code: %d, response body: %s", - resp.RawResponse.StatusCode, - resp.String(), - ) - - // result Code -1 - // {\"error\":{\"msg\":\"未登录\",\"code\":-2000} - return ErrAuthFailed - } - - return nil -} - -// ColumnInfo get normal column info, like v3 product info -func (c *Client) ColumnInfo(productID int) (Product, error) { - var res response.V3ColumnInfoResponse - r := c.newRequest(resty.MethodPost, - V3ColumnInfoPath, - nil, - map[string]interface{}{ - "product_id": productID, - "with_recommend_article": true, - }, - &res, - ) - if _, err := do(r); err != nil { - return Product{}, err - } - - return Product{ - Access: res.Data.Extra.Sub.AccessMask > 0, - ID: res.Data.ID, - Type: res.Data.Type, - Title: res.Data.Title, - IsVideo: res.Data.IsVideo, - }, nil -} - -// ColumnArticles call geektime api to get article list -func (c *Client) ColumnArticles(cid string) ([]Article, error) { - res := &response.V1ColumnArticlesResponse{} - r := c.newRequest(resty.MethodPost, - V1ColumnArticlesPath, - nil, - map[string]interface{}{ - "cid": cid, - "order": "earliest", - "prev": 0, - "sample": false, - "size": 500, //get all articles - }, - res, - ) - if _, err := do(r); err != nil { - return nil, err + return p, err } + p.Articles = articles - var articles []Article - for _, v := range res.Data.List { - articles = append(articles, Article{ - AID: v.ID, - Title: v.ArticleTitle, - }) - } - return articles, nil + return p, nil } // V1ArticleInfo ... func (c *Client) V1ArticleInfo(articleID int) (response.V1ArticleResponse, error) { var res response.V1ArticleResponse - r := c.newRequest(resty.MethodPost, + r := c.newRequest( + resty.MethodPost, + DefaultBaseURL, V1ArticlePath, nil, map[string]interface{}{ @@ -314,7 +94,9 @@ func (c *Client) V1ArticleInfo(articleID int) (response.V1ArticleResponse, error // ProductInfo used to get daily lesson or qconplus product info func (c *Client) ProductInfo(productID int) (response.V3ProductInfoResponse, error) { var res response.V3ProductInfoResponse - r := c.newRequest(resty.MethodPost, + r := c.newRequest( + resty.MethodPost, + DefaultBaseURL, V3ProductInfoPath, nil, map[string]interface{}{ @@ -331,7 +113,9 @@ func (c *Client) ProductInfo(productID int) (response.V3ProductInfoResponse, err // V3ArticleInfo used to get daily lesson or qconplus article info func (c *Client) V3ArticleInfo(articleID int) (response.V3ArticleInfoResponse, error) { var res response.V3ArticleInfoResponse - r := c.newRequest(resty.MethodPost, + r := c.newRequest( + resty.MethodPost, + DefaultBaseURL, V3ArticleInfoPath, nil, map[string]interface{}{ @@ -348,7 +132,9 @@ func (c *Client) V3ArticleInfo(articleID int) (response.V3ArticleInfoResponse, e // VideoPlayAuth get play auth string func (c *Client) VideoPlayAuth(articleID, sourceType int, videoID string) (string, error) { var res response.V3VideoPlayAuthResponse - r := c.newRequest(resty.MethodPost, + r := c.newRequest( + resty.MethodPost, + DefaultBaseURL, V3VideoPlayAuthPath, nil, map[string]interface{}{ @@ -364,133 +150,60 @@ func (c *Client) VideoPlayAuth(articleID, sourceType int, videoID string) (strin return res.Data.PlayAuth, nil } -// UniversityVideoPlayAuth ... -func (c *Client) UniversityVideoPlayAuth(articleID, classID int) (response.V1VideoPlayAuthResponse, error) { - var res response.V1VideoPlayAuthResponse - r := c.newRequest(resty.MethodPost, - UniversityV1VideoPlayAuthPath, +// columnInfo get normal column info, like v3 product info +func (c *Client) columnInfo(productID int) (Course, error) { + var res response.V3ColumnInfoResponse + r := c.newRequest( + resty.MethodPost, + DefaultBaseURL, + V3ColumnInfoPath, nil, map[string]interface{}{ - "article_id": articleID, - "class_id": classID, + "product_id": productID, + "with_recommend_article": true, }, &res, ) if _, err := do(r); err != nil { - return response.V1VideoPlayAuthResponse{}, err + return Course{}, err } - return res, nil -} -// MyClassProduct ... -func (c *Client) MyClassProduct(classID int) (Product, error) { - var p Product + return Course{ + Access: res.Data.Extra.Sub.AccessMask > 0, + ID: res.Data.ID, + Type: res.Data.Type, + Title: res.Data.Title, + IsVideo: res.Data.IsVideo, + }, nil +} - var res response.V1MyClassInfoResponse - r := c.newRequest(resty.MethodPost, - UniversityV1MyClassInfoPath, +// columnArticles call geektime api to get article list +func (c *Client) columnArticles(cid int) ([]Article, error) { + res := &response.V1ColumnArticlesResponse{} + r := c.newRequest( + resty.MethodPost, + DefaultBaseURL, + V1ColumnArticlesPath, nil, map[string]interface{}{ - "class_id": classID, + "cid": strconv.Itoa(cid), + "order": "earliest", + "prev": 0, + "sample": false, + "size": 500, //get all articles }, - &res, - ) - - resp, err := do(r) - if err != nil { - return p, err - } - - if res.Code != 0 { - if res.Error.Code == -5001 { - p.Access = false - return p, nil - } - return p, ErrGeekTimeAPIBadCode{UniversityV1MyClassInfoPath, resp.String()} - } - - p = Product{ - Access: true, - ID: classID, - Title: res.Data.Title, - Type: "", - IsVideo: true, - } - var articles []Article - for _, lesson := range res.Data.Lessons { - for _, article := range lesson.Articles { - // ONLY download university video lessons - if article.VideoTime > 0 { - articles = append(articles, Article{ - AID: article.ArticleID, - Title: article.ArticleTitle, - }) - } - } - } - p.Articles = articles - - return p, nil -} - -func (c *Client) newRequest(method, url string, params map[string]string, body interface{}, res interface{}) *resty.Request { - r := c.HTTPClient.R() - r.Method = method - r.URL = c.BaseURL + url - r.SetHeader(Origin, c.BaseURL) - if len(params) > 0 { - r.SetQueryParams(params) - } - if body != nil { - r.SetBody(body) - } - r.SetResult(res) - return r -} - -func do(r *resty.Request) (*resty.Response, error) { - logger.Infof("Http request start, method: %s, url: %s", - r.Method, - r.URL, + res, ) - resp, err := r.Execute(r.Method, r.URL) - - if err != nil { + if _, err := do(r); err != nil { return nil, err } - statusCode := resp.RawResponse.StatusCode - if statusCode != 200 { - logNotOkResponse(resp) - if statusCode == 451 { - return nil, ErrGeekTimeRateLimit - } else if statusCode == 452 { - return nil, ErrAuthFailed - } - } - - rv := reflect.ValueOf(r.Result) - f := reflect.Indirect(rv).FieldByName("Code") - code := int(f.Int()) - - if code == 0 { - return resp, nil - } - - logNotOkResponse(resp) - //未登录或者已失效 - if code == -3050 || code == -2000 { - return nil, ErrAuthFailed + var articles []Article + for _, v := range res.Data.List { + articles = append(articles, Article{ + AID: v.ID, + Title: v.ArticleTitle, + }) } - - return nil, ErrGeekTimeAPIBadCode{r.URL, resp.String()} -} - -func logNotOkResponse(resp *resty.Response) { - logger.Warnf("Http request end, method: %s, url: %s, status code: %d, response body: %s", - resp.RawResponse.Request.Method, - resp.RawResponse.Request.URL, - resp.RawResponse.StatusCode, - resp.String(), - ) + return articles, nil } diff --git a/internal/geektime/response/struct_v1_enterprise_article_info.go b/internal/geektime/response/struct_v1_enterprise_article_info.go new file mode 100644 index 0000000..b78b3a5 --- /dev/null +++ b/internal/geektime/response/struct_v1_enterprise_article_info.go @@ -0,0 +1,195 @@ +package response + +type V1EnterpriseArticlesDetailResponse struct { + Code int `json:"code"` + Data struct { + ID string `json:"id"` + Time string `json:"time"` + Type string `json:"type"` + FavoriteID int `json:"favorite_id"` + DiscussionNumber int `json:"discussion_number"` + ColumnTitle string `json:"column_title"` + Rights bool `json:"rights"` + Show bool `json:"show"` + RichType int `json:"rich_type"` + PID int `json:"pid"` + SKU int `json:"sku"` + Action string `json:"action"` + Score int `json:"score"` + IsRequired bool `json:"is_required"` + URI string `json:"uri"` + ColumnType int `json:"column_type"` + EnterpriseID string `json:"enterprise_id"` + NodeType int `json:"node_type"` + Published int `json:"published"` + ArtStatus int `json:"art_status"` + SKUStatus int `json:"sku_status"` + IsSell int `json:"is_sell"` + Name string `json:"name"` + ProductType string `json:"product_type"` + ArticleSource int `json:"article_source"` + ArticleVendorID int `json:"article_vendor_id"` + Author struct { + Name string `json:"name"` + Avatar string `json:"avatar"` + Info string `json:"info"` + Intro string `json:"intro"` + } `json:"author"` + Article struct { + ID string `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + ContentMD string `json:"content_md"` + CTime int `json:"ctime"` + PosterWxlite string `json:"poster_wxlite"` + CoverHidden int `json:"cover_hidden"` + Subtitle string `json:"subtitle"` + Summary string `json:"summary"` + CouldPreview bool `json:"could_preview"` + BCouldPreview bool `json:"b_could_preview"` + ContentJSON string `json:"content_json"` + ContentJSONShort string `json:"content_json_short"` + InlineVideo struct { + Rights []interface{} `json:"rights"` + Preview []interface{} `json:"preview"` + } `json:"inline_video"` + Cover struct { + ColumnCover string `json:"column_cover"` + Default string `json:"default"` + CoverID int `json:"cover_id"` + CoverStatus int `json:"cover_status"` + SKUCover struct { + Ratio16 string `json:"ratio_16"` + Ratio16URL string `json:"ratio_16_url"` + Ratio4 string `json:"ratio_4"` + Ratio4URL string `json:"ratio_4_url"` + Ratio1 string `json:"ratio_1"` + Ratio1URL string `json:"ratio_1_url"` + ShowCover int `json:"show_cover"` + } `json:"sku_cover"` + } `json:"cover"` + Share struct { + Title string `json:"title"` + Content string `json:"content"` + Cover string `json:"cover"` + Poster string `json:"poster"` + } `json:"share"` + Relation struct { + PrevID string `json:"prev_id"` + PrevChapterTitle string `json:"prev_chapter_title"` + PrevArticleTitle string `json:"prev_article_title"` + NextID string `json:"next_id"` + NextChapterTitle string `json:"next_chapter_title"` + NextArticleTitle string `json:"next_article_title"` + } `json:"relation"` + } `json:"article"` + Chapter struct { + SourceID int `json:"source_id"` + Title string `json:"title"` + SKU string `json:"sku"` + Score string `json:"score"` + PChapterSourceID string `json:"pchapter_source_id"` + PChapterTitle string `json:"p_chapter_title"` + ChapterStatus int `json:"chapter_status"` + } `json:"chapter"` + Audio struct { + URL string `json:"url"` + DownloadURL string `json:"download_url"` + Size int `json:"size"` + Title string `json:"title"` + Time string `json:"time"` + MD5 string `json:"md5"` + Dubber string `json:"dubber"` + ID string `json:"id"` + Status int `json:"status"` + } `json:"audio"` + Video struct { + ID string `json:"id"` + MD5 string `json:"md5"` + URL string `json:"url"` + Cover struct { + Type int `json:"type"` + ID int `json:"id"` + URL string `json:"url"` + } `json:"cover"` + Width int `json:"width"` + Height int `json:"height"` + Size int `json:"size"` + Time string `json:"time"` + HLSMedias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"hls_medias"` + HLSVid string `json:"hls_vid"` + Version int `json:"version"` + Medias interface{} `json:"medias"` + MediaOpen string `json:"media_open"` + CouldPreview int `json:"could_preview"` + Preview struct { + Duration int `json:"duration"` + Medias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"medias"` + } `json:"preview"` + Subtitles struct { + Rights interface{} `json:"rights"` + Preview interface{} `json:"preview"` + } `json:"subtitles"` + Status int `json:"status"` + } `json:"video"` + Files []interface{} `json:"files"` + Extra struct { + Process struct { + ArticleID string `json:"article_id"` + LearnPercent int `json:"learn_percent"` + ArticleOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"article_offset"` + AudioOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"audio_offset"` + VideoOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"video_offset"` + } `json:"process"` + IsLast bool `json:"is_last"` + Fav struct { + HasDone bool `json:"has_done"` + TotalCount int `json:"total_count"` + FavID int `json:"fav_id"` + FavType int `json:"fav_type"` + } `json:"fav"` + IsShow bool `json:"IsShow"` + Attachments []interface{} `json:"attachments"` + } `json:"extra"` + AnyreadTotal int `json:"anyread_total"` + AnyreadUsed int `json:"anyread_used"` + AnyreadHit bool `json:"anyread_hit"` + } `json:"data"` + Error interface{} `json:"error"` + Extra struct { + Cost float64 `json:"cost"` + RequestID string `json:"request-id"` + } `json:"extra"` +} diff --git a/internal/geektime/response/struct_v1_enterprise_articles.go b/internal/geektime/response/struct_v1_enterprise_articles.go new file mode 100644 index 0000000..3386f32 --- /dev/null +++ b/internal/geektime/response/struct_v1_enterprise_articles.go @@ -0,0 +1,208 @@ +package response + +type V1EnterpriseArticlesResponse struct { + Code int `json:"code"` + Data struct { + List []struct { + ID int `json:"id"` + Title string `json:"title"` + Count int `json:"count"` + Score int `json:"score"` + IsLast bool `json:"is_last"` + ArticleList []struct { + ID string `json:"id"` + Time string `json:"time"` + Type string `json:"type"` + FavoriteID int `json:"favorite_id"` + DiscussionNumber int `json:"discussion_number"` + ColumnTitle string `json:"column_title"` + Rights bool `json:"rights"` + Show bool `json:"show"` + RichType int `json:"rich_type"` + PID int `json:"pid"` + SKU int `json:"sku"` + Action string `json:"action"` + Score int `json:"score"` + IsRequired bool `json:"is_required"` + URI string `json:"uri"` + ColumnType int `json:"column_type"` + EnterpriseID string `json:"enterprise_id"` + NodeType int `json:"node_type"` + Published int `json:"published"` + ArtStatus int `json:"art_status"` + SKUStatus int `json:"sku_status"` + IsSell int `json:"is_sell"` + Name string `json:"name"` + ProductType string `json:"product_type"` + ArticleSource int `json:"article_source"` + ArticleVendorID int `json:"article_vendor_id"` + Author struct { + Name string `json:"name"` + Avatar string `json:"avatar"` + Info string `json:"info"` + Intro string `json:"intro"` + } `json:"author"` + Article struct { + ID string `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + ContentMD string `json:"content_md"` + CTime int `json:"ctime"` + PosterWxlite string `json:"poster_wxlite"` + CoverHidden int `json:"cover_hidden"` + Subtitle string `json:"subtitle"` + Summary string `json:"summary"` + CouldPreview bool `json:"could_preview"` + BCouldPreview bool `json:"b_could_preview"` + ContentJSON string `json:"content_json"` + ContentJSONShort string `json:"content_json_short"` + InlineVideo struct { + Rights []interface{} `json:"rights"` + Preview []interface{} `json:"preview"` + } `json:"inline_video"` + Cover struct { + ColumnCover string `json:"column_cover"` + Default string `json:"default"` + CoverID int `json:"cover_id"` + CoverStatus int `json:"cover_status"` + SKUCover struct { + Ratio16 string `json:"ratio_16"` + Ratio16URL string `json:"ratio_16_url"` + Ratio4 string `json:"ratio_4"` + Ratio4URL string `json:"ratio_4_url"` + Ratio1 string `json:"ratio_1"` + Ratio1URL string `json:"ratio_1_url"` + ShowCover int `json:"show_cover"` + } `json:"sku_cover"` + } `json:"cover"` + Share struct { + Title string `json:"title"` + Content string `json:"content"` + Cover string `json:"cover"` + Poster string `json:"poster"` + } `json:"share"` + Relation struct { + PrevID string `json:"prev_id"` + PrevChapterTitle string `json:"prev_chapter_title"` + PrevArticleTitle string `json:"prev_article_title"` + NextID string `json:"next_id"` + NextChapterTitle string `json:"next_chapter_title"` + NextArticleTitle string `json:"next_article_title"` + } `json:"relation"` + } `json:"article"` + Chapter struct { + SourceID int `json:"source_id"` + Title string `json:"title"` + SKU string `json:"sku"` + Score string `json:"score"` + PChapterSourceID string `json:"pchapter_source_id"` + PChapterTitle string `json:"p_chapter_title"` + ChapterStatus int `json:"chapter_status"` + } `json:"chapter"` + Audio struct { + URL string `json:"url"` + DownloadURL string `json:"download_url"` + Size int `json:"size"` + Title string `json:"title"` + Time string `json:"time"` + MD5 string `json:"md5"` + Dubber string `json:"dubber"` + ID string `json:"id"` + Status int `json:"status"` + } `json:"audio"` + Video struct { + ID string `json:"id"` + MD5 string `json:"md5"` + URL string `json:"url"` + Cover struct { + Type int `json:"type"` + ID int `json:"id"` + URL string `json:"url"` + } `json:"cover"` + Width int `json:"width"` + Height int `json:"height"` + Size int `json:"size"` + Time string `json:"time"` + HlsMedias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"hls_medias"` + HlsVid string `json:"hls_vid"` + Version int `json:"version"` + Medias interface{} `json:"medias"` + MediaOpen string `json:"media_open"` + CouldPreview int `json:"could_preview"` + Preview struct { + Duration int `json:"duration"` + Medias []struct { + Quality string `json:"quality"` + Size int `json:"size"` + URL string `json:"url"` + } `json:"medias"` + } `json:"preview"` + Subtitles struct { + Rights interface{} `json:"rights"` + Preview []interface{} `json:"preview"` + } `json:"subtitles"` + Status int `json:"status"` + } `json:"video"` + Files []interface{} `json:"files"` + Extra struct { + Process struct { + ArticleID string `json:"article_id"` + LearnPercent int `json:"learn_percent"` + ArticleOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"article_offset"` + AudioOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"audio_offset"` + VideoOffset struct { + CurOffset int `json:"cur_offset"` + MaxOffset int `json:"max_offset"` + Length int `json:"length"` + Version int `json:"version"` + Process int `json:"process"` + LearnTime int `json:"learn_time"` + LearnStatus int `json:"learn_status"` + } `json:"video_offset"` + } `json:"process"` + IsLast bool `json:"is_last"` + Fav struct { + HasDone bool `json:"has_done"` + TotalCount int `json:"total_count"` + FavID int `json:"fav_id"` + FavType int `json:"fav_type"` + } `json:"fav"` + IsShow bool `json:"IsShow"` + Attachments []interface{} `json:"attachments"` + } `json:"extra"` + AnyreadTotal int `json:"anyread_total"` + AnyreadUsed int `json:"anyread_used"` + AnyreadHit bool `json:"anyread_hit"` + } `json:"article_list"` + } `json:"list"` + HasChapter bool `json:"has_chapter"` + IsShow bool `json:"is_show"` + AnyreadTotal int `json:"anyread_total"` + AnyreadUsed int `json:"anyread_used"` + + Extra struct { + Cost float64 `json:"cost"` + RequestID string `json:"request-id"` + } `json:"extra"` + } `json:"data"` +} diff --git a/internal/geektime/response/struct_v1_enterprise_product_info.go b/internal/geektime/response/struct_v1_enterprise_product_info.go new file mode 100644 index 0000000..3096a24 --- /dev/null +++ b/internal/geektime/response/struct_v1_enterprise_product_info.go @@ -0,0 +1,166 @@ +package response + +type V1EnterpriseProductInfoResponse struct { + Code int `json:"code"` + Data struct { + ID int `json:"id"` + SKU int `json:"sku"` + Title string `json:"title"` + SubTitle string `json:"sub_title"` + ProductType string `json:"product_type"` + ColumnType int `json:"column_type"` + CourseType int `json:"course_type"` + UpdateFreq string `json:"update_frequency"` + Author struct { + Name string `json:"name"` + Intro string `json:"intro"` + Info string `json:"info"` + Avatar string `json:"avatar"` + BriefHTML string `json:"brief_html"` + Brief string `json:"brief"` + } `json:"author"` + Cover struct { + Square string `json:"square"` + Rectangle string `json:"rectangle"` + Horizontal string `json:"horizontal"` + LectureHorizontal string `json:"lecture_horizontal"` + LearnHorizontal string `json:"learn_horizontal"` + Transparent string `json:"transparent"` + Color string `json:"color"` + Cover string `json:"cover"` + RectCover string `json:"rect_cover"` + Ratio1 string `json:"ratio_1"` + Ratio4 string `json:"ratio_4"` + Ratio16 string `json:"ratio_16"` + CoverID int `json:"cover_id"` + CoverStatus int `json:"cover_status"` + } `json:"cover"` + TeachTypeList []int `json:"teach_type_list"` + TeachTypeNameList []string `json:"teach_type_name_list"` + Article struct { + Count int `json:"count"` + CountReq int `json:"count_req"` + CountPub int `json:"count_pub"` + FirstArticleID string `json:"first_article_id"` + TotalLength int `json:"total_length"` + TotalTimeStr string `json:"total_time_str"` + TotalTimeHourStr string `json:"total_time_hour_str"` + } `json:"article"` + SEO struct { + Keywords []string `json:"keywords"` + } `json:"seo"` + Category struct { + CategoryID int `json:"category_id"` + Name string `json:"name"` + PID int `json:"pid"` + } `json:"category"` + Path struct { + Desc string `json:"desc"` + DescHTML string `json:"desc_html"` + } `json:"path"` + DL struct { + Article struct { + ArticleID string `json:"article_id"` + Duration string `json:"duration"` + Hot int `json:"hot"` + CouldPreview bool `json:"could_preview"` + DurationSeconds int `json:"duration_seconds"` + } `json:"article"` + CollectionIDs interface{} `json:"collection_ids"` + } `json:"dl"` + Share struct { + PicURL string `json:"pic_url"` + Title string `json:"title"` + PicName string `json:"pic_name"` + Content string `json:"content"` + } `json:"share"` + IsFinish bool `json:"is_finish"` + Unit string `json:"unit"` + BannerCover string `json:"banner_cover"` + CatalogPicURL string `json:"catalog_pic_url"` + Extra struct { + Fav struct { + HasDone bool `json:"has_done"` + TotalCount int `json:"total_count"` + FavID int `json:"fav_id"` + FavType int `json:"fav_type"` + } `json:"fav"` + IsSVIP bool `json:"is_svip"` + IsMyCourse bool `json:"is_my_course"` + Rate struct { + ArticleCount int `json:"article_count"` + ArticleCountReq int `json:"article_count_req"` + IsFinished bool `json:"is_finished"` + RatePercent int `json:"rate_percent"` + VideoSeconds int `json:"video_seconds"` + LastArticleID string `json:"last_article_id"` + LastChapterID int `json:"last_chapter_id"` + HasLearn bool `json:"has_learn"` + } `json:"rate"` + StudyCount int `json:"study_count"` + Modules []struct { + Name string `json:"name"` + IsTop bool `json:"is_top"` + Title string `json:"title"` + Type string `json:"type"` + Content string `json:"content"` + } `json:"modules"` + TplType int `json:"tpl_type"` + CollectionType int `json:"collection_type"` + WithVideo bool `json:"with_video"` + PIDs []interface{} `json:"pids"` + Labels []interface{} `json:"labels"` + CategoryIDs []interface{} `json:"category_ids"` + Group struct { + Title string `json:"title"` + Description string `json:"description"` + StartTime int `json:"start_time"` + EndTime int `json:"end_time"` + QRCodeShow bool `json:"qrcode_show"` + QRCodeURL string `json:"qrcode_url"` + } `json:"group"` + VIP struct { + Show bool `json:"show"` + EndTime int `json:"end_time"` + } `json:"vip"` + CourseStatus int `json:"course_status"` + CID int `json:"cid"` + RelatedVIPSkus []struct { + ColumnTitle string `json:"column_title"` + DisplayType int `json:"display_type"` + EsPrice int `json:"es_price"` + EsSaleMaxLimit int `json:"es_sale_max_limit"` + EsSaleMinLimit int `json:"es_sale_min_limit"` + SKU int `json:"sku"` + Status int `json:"status"` + VIPDays int `json:"vip_days"` + VIPTitle string `json:"vip_title"` + } `json:"related_vip_skus"` + } `json:"extra"` + Intro string `json:"intro"` + IntroHTML string `json:"intro_html"` + BgColor string `json:"bgcolor"` + IsIncludePreview bool `json:"is_include_preview"` + ShowChapter bool `json:"show_chapter"` + DisplayType int `json:"display_type"` + IntroBGStyle int `json:"intro_bg_style"` + Sort int `json:"sort"` + CTime int `json:"ctime"` + SalePrice int `json:"sale_price"` + SaleLimit int `json:"sale_limit"` + Status int `json:"status"` + IsJoinSVIP int `json:"is_join_svip"` + IsJoinColumnVIP int `json:"is_join_column_vip"` + IsJoinCVIP int `json:"is_join_cvip"` + NeedGraduate int `json:"need_graduate"` + AuthorSignatureURL string `json:"author_signature_url"` + IsFreebie int `json:"is_freebie"` + IsDtai int `json:"is_dtai"` + } `json:"data"` + Error struct { + } `json:"error"` + Extra struct { + Cost float64 `json:"cost"` + RequestID string `json:"request-id"` + } `json:"extra"` +} diff --git a/internal/geektime/university.go b/internal/geektime/university.go new file mode 100644 index 0000000..b4aa55b --- /dev/null +++ b/internal/geektime/university.go @@ -0,0 +1,89 @@ +package geektime + +import ( + "github.com/go-resty/resty/v2" + "github.com/nicoxiang/geektime-downloader/internal/geektime/response" +) + +const ( + // GeekBangUniversityBaseURL ... + GeekBangUniversityBaseURL = "https://u.geekbang.org" + + // UniversityV1VideoPlayAuthPath used in university video play auth + UniversityV1VideoPlayAuthPath = "/serv/v1/video/play-auth" + // UniversityV1MyClassInfoPath get university class info and all articles info in it + UniversityV1MyClassInfoPath = "/serv/v1/myclass/info" +) + +// UniversityCourseInfo get university class info +func (c *Client) UniversityCourseInfo(classID int) (Course, error) { + var p Course + + var res response.V1MyClassInfoResponse + r := c.newRequest( + resty.MethodPost, + GeekBangUniversityBaseURL, + UniversityV1MyClassInfoPath, + nil, + map[string]interface{}{ + "class_id": classID, + }, + &res, + ) + + resp, err := do(r) + if err != nil { + return p, err + } + + if res.Code != 0 { + if res.Error.Code == -5001 { + p.Access = false + return p, nil + } + return p, ErrGeekTimeAPIBadCode{UniversityV1MyClassInfoPath, resp.String()} + } + + p = Course{ + Access: true, + ID: classID, + Title: res.Data.Title, + Type: "", + IsVideo: true, + } + var articles []Article + for _, lesson := range res.Data.Lessons { + for _, article := range lesson.Articles { + // ONLY download university video lessons + if article.VideoTime > 0 { + articles = append(articles, Article{ + AID: article.ArticleID, + Title: article.ArticleTitle, + }) + } + } + } + p.Articles = articles + + return p, nil +} + +// UniversityVideoPlayAuth get university play auth string +func (c *Client) UniversityVideoPlayAuth(articleID, classID int) (response.V1VideoPlayAuthResponse, error) { + var res response.V1VideoPlayAuthResponse + r := c.newRequest( + resty.MethodPost, + GeekBangUniversityBaseURL, + UniversityV1VideoPlayAuthPath, + nil, + map[string]interface{}{ + "article_id": articleID, + "class_id": classID, + }, + &res, + ) + if _, err := do(r); err != nil { + return response.V1VideoPlayAuthResponse{}, err + } + return res, nil +} diff --git a/internal/markdown/markdown.go b/internal/markdown/markdown.go index e8217cf..77fec91 100644 --- a/internal/markdown/markdown.go +++ b/internal/markdown/markdown.go @@ -16,6 +16,7 @@ import ( "github.com/nicoxiang/geektime-downloader/internal/geektime" "github.com/nicoxiang/geektime-downloader/internal/pkg/downloader" "github.com/nicoxiang/geektime-downloader/internal/pkg/filenamify" + "github.com/nicoxiang/geektime-downloader/internal/pkg/files" ) var ( @@ -37,17 +38,23 @@ func (ms *markdownString) ReplaceAll(o, n string) { ms.s = strings.ReplaceAll(ms.s, o, n) } -// Download ... -func Download(ctx context.Context, html, title, dir string, aid int) error { +// Download article as markdown +func Download(ctx context.Context, html, title, dir string, aid int, overwrite bool) (bool, error) { select { case <-ctx.Done(): - return context.Canceled + return false, context.Canceled default: } + + fullName := path.Join(dir, filenamify.Filenamify(title)+MDExtension) + if files.CheckFileExists(fullName) && !overwrite { + return true, nil + } + // step1: convert to md string markdown, err := getDefaultConverter().ConvertString(html) if err != nil { - return err + return false, err } // step2: download images var ss = &markdownString{s: markdown} @@ -63,23 +70,22 @@ func Download(ctx context.Context, html, title, dir string, aid int) error { err = writeImageFile(ctx, imageURLs, dir, imagesFolder, ss) if err != nil { - return err + return false, err } - fullName := path.Join(dir, filenamify.Filenamify(title)+MDExtension) f, err := os.Create(fullName) defer func() { _ = f.Close() }() if err != nil { - return err + return false, err } // step3: write md file _, err = f.WriteString("# " + title + "\n" + ss.s) if err != nil { - return err + return false, err } - return nil + return false, nil } func findAllImages(md string) (images []string) { diff --git a/internal/markdown/markdown_test.go b/internal/markdown/markdown_test.go index 58c7d28..e6daeff 100644 --- a/internal/markdown/markdown_test.go +++ b/internal/markdown/markdown_test.go @@ -14,7 +14,7 @@ func TestDownLoad_SpecialHtml(t *testing.T) { content := "这里，让我们一起看一个XSStrike的使用示例，来加深对它的理解。

首先，我们来看看它的用法。

\"图片\"

其中比较重要的配置项，我将它们列举如下：

-h                #提示信息\n-u                 #目标地址\n-data             #通过post方式上传数据\n--headers          #配置请求头信息，包括cookie等\n
sudo python3 xsstrike.py -u 'http://b6b7183d85ac4d36bb9449cb938ef977.app.mituan.zone/level1.php?name=test' \n

这段代码就是用参数u配置了一个目标地址，其中在请求中通过get方式上传了参数name，这样XSStrike可以识别到这个通过get方式上传的参数，可以看到应用有如下输出：

\"图片\"

从输出中，我们可以知道它会首先判断是否有WAF存在，然后对参数进行测试，获取到页面的响应，并据此生成payload。这和我们之前学习的sqlmap非常类似，因为它们本质上其实都是注入检测工具。

生成payload之后，XSStrike会将它们按照Confidence的值从大到小进行排序，之后按照顺序逐一对它们进行检测。这里你可能会好奇Confidence是什么，事实上，它代表的是XSStrike开发人员对于这个payload成功的信心，它的取值范围为0-10，值越高代表注入成功的可能性就越大。

之后XSStrike根据注入的payload以及它们响应的内容，会给这个payload生成一个评分即Efficiency，这个评分越高，代表这个payload实现XSS攻击的成功率越大。如果评分高于90，就会将这个payload标记为成功，并将它输出在命令行中，否则就会认为这个payload无效。

到这里，你已经学会了XSS攻击的检测方法，接下来让我们进入到XSS攻击防御方案的学习之中。

# 原始代码\n<script>alert(1)</script>\n# 混淆后的代码\n[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]][([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]((!![]+[])[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+([][[]]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+!+[]]+(+[![]]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+!+[]]]+(!![]+[])[!+[]+!+[]+!+[]]+(+(!+[]+!+[]+!+[]+[+!+[]]))[(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([]+[])[([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]][([][[]]+[])[+!+[]]+(![]+[])[+!+[]]+((+[])[([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]+[])[+!+[]+[+!+[]]]+(!![]+[])[!+[]+!+[]+!+[]]]](!+[]+!+[]+!+[]+[!+[]+!+[]])+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]])()(([]+[])[([![]]+[][[]])[+!+[]+[+[]]]+(!![]+[])[+[]]+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(![]+[])[!+[]+!+[]+!+[]]]()[+[]]+(![]+[])[!+[]+!+[]+!+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(+(!+[]+!+[]+[+!+[]]+[+!+[]]))[(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([]+[])[([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]][([][[]]+[])[+!+[]]+(![]+[])[+!+[]]+((+[])[([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]+[])[+!+[]+[+!+[]]]+(!![]+[])[!+[]+!+[]+!+[]]]](!+[]+!+[]+!+[]+[+!+[]])[+!+[]]+(!![]+[])[+[]]+([]+[])[([![]]+[][[]])[+!+[]+[+[]]]+(!![]+[])[+[]]+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(![]+[])[!+[]+!+[]+!+[]]]()[!+[]+!+[]]+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+!+[]]+(!![]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[+!+[]+[!+[]+!+[]+!+[]]]+[+!+[]]+([+[]]+![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[!+[]+!+[]+[+[]]]+([]+[])[([![]]+[][[]])[+!+[]+[+[]]]+(!![]+[])[+[]]+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(![]+[])[!+[]+!+[]+!+[]]]()[+[]]+(![]+[+[]])[([![]]+[][[]])[+!+[]+[+[]]]+(!![]+[])[+[]]+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(![]+[])[!+[]+!+[]+!+[]]]()[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]+!+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(+(!+[]+!+[]+[+!+[]]+[+!+[]]))[(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([]+[])[([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]][([][[]]+[])[+!+[]]+(![]+[])[+!+[]]+((+[])[([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]+[])[+!+[]+[+!+[]]]+(!![]+[])[!+[]+!+[]+!+[]]]](!+[]+!+[]+!+[]+[+!+[]])[+!+[]]+(!![]+[])[+[]]+([]+[])[([![]]+[][[]])[+!+[]+[+[]]]+(!![]+[])[+[]]+(![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+([![]]+[][[]])[+!+[]+[+[]]]+([][(![]+[])[+[]]+(![]+[])[!+[]+!+[]]+(![]+[])[+!+[]]+(!![]+[])[+[]]]+[])[!+[]+!+[]+!+[]]+(![]+[])[!+[]+!+[]+!+[]]]()[!+[]+!+[]])\n

这个例子是一个JavaScript代码混淆示例，我们可以将一个非常明显的JavaScript转化为一堆乱码，神奇的是这串乱码和特征明显的JavaScript语句具有一样的功能。这样攻击者就可以将一个很容易被黑名单、白名单以及WAF检测出来的负载改为了难以被检测出来的负载，从而成功发起XSS攻击，实现自己想要的恶意行为。" - err := Download(ctx, content, "失效的输入检测（上）：攻击者有哪些绕过方案？", p, 100101501) + _, err := Download(ctx, content, "失效的输入检测（上）：攻击者有哪些绕过方案？", p, 100101501, true) if err != nil { t.Error(err) } diff --git a/internal/pdf/pdf.go b/internal/pdf/pdf.go index 872c4c0..b63d5e2 100644 --- a/internal/pdf/pdf.go +++ b/internal/pdf/pdf.go @@ -1,7 +1,9 @@ package pdf import ( + "bufio" "context" + "github.com/nicoxiang/geektime-downloader/internal/pkg/logger" "net/http" "os" "path/filepath" @@ -16,6 +18,7 @@ import ( "github.com/chromedp/chromedp/device" "github.com/nicoxiang/geektime-downloader/internal/geektime" "github.com/nicoxiang/geektime-downloader/internal/pkg/filenamify" + "github.com/nicoxiang/geektime-downloader/internal/pkg/files" ) // PDFExtension ... @@ -28,14 +31,23 @@ func PrintArticlePageToPDF(ctx context.Context, title string, cookies []*http.Cookie, downloadComments bool, - waitSeconds int, -) error { + printPDFWaitSeconds int, + printPDFTimeoutSeconds int, + overwrite bool, +) (bool, error) { rateLimit := false + + fileName := filepath.Join(dir, filenamify.Filenamify(title)+PDFExtension) + + if files.CheckFileExists(fileName) && !overwrite { + return true, nil + } + // new tab ctx, cancel := chromedp.NewContext(ctx) defer cancel() - ctx, cancel = context.WithTimeout(ctx, time.Minute) + ctx, cancel = context.WithTimeout(ctx, time.Duration(printPDFTimeoutSeconds)*time.Second) defer cancel() chromedp.ListenTarget(ctx, func(ev interface{}) { @@ -49,30 +61,25 @@ func PrintArticlePageToPDF(ctx context.Context, } }) - var buf []byte err := chromedp.Run(ctx, chromedp.Tasks{ chromedp.Emulate(device.IPadPro11), setCookies(cookies), chromedp.Navigate(geektime.DefaultBaseURL + `/column/article/` + strconv.Itoa(aid)), - chromedp.Sleep(time.Duration(waitSeconds) * time.Second), + chromedp.Sleep(time.Duration(printPDFWaitSeconds) * time.Second), hideRedundantElements(downloadComments), - printToPDF(&buf), + printToPDF(fileName), }, ) if err != nil { if rateLimit { - return geektime.ErrGeekTimeRateLimit + return false, geektime.ErrGeekTimeRateLimit } - return err + return false, err } - fileName := filepath.Join(dir, filenamify.Filenamify(title)+PDFExtension) - if err := os.WriteFile(fileName, buf, 0666); err != nil { - return err - } - return nil + return false, nil } func setCookies(cookies []*http.Cookie) chromedp.ActionFunc { @@ -171,18 +178,45 @@ func hideRedundantElements(downloadComments bool) chromedp.ActionFunc { }) } -func printToPDF(res *[]byte) chromedp.ActionFunc { +func printToPDF(fileName string) chromedp.ActionFunc { return chromedp.ActionFunc(func(ctx context.Context) error { - data, _, err := page.PrintToPDF(). + _, stream, err := page.PrintToPDF(). WithMarginTop(0.4). WithMarginBottom(0.4). WithMarginLeft(0.4). WithMarginRight(0.4). + WithTransferMode(page.PrintToPDFTransferModeReturnAsStream). Do(ctx) if err != nil { return err } - *res = data + + reader := &streamReader{ + ctx: ctx, + handle: stream, + r: nil, + pos: 0, + eof: false, + } + + defer func() { + _ = reader.Close() + }() + + file, _ := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666) + + defer func() { + _ = file.Close() + }() + + buffer := bufio.NewReader(reader) + + _, err = buffer.WriteTo(file) + if err != nil { + logger.Error(err, "write result to output path") + return err + } + return nil }) } diff --git a/internal/pdf/stream.go b/internal/pdf/stream.go new file mode 100644 index 0000000..f4681ba --- /dev/null +++ b/internal/pdf/stream.go @@ -0,0 +1,117 @@ +package pdf + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "github.com/chromedp/cdproto/cdp" + "io" + "strings" + + cdprotoio "github.com/chromedp/cdproto/io" +) + +// Credits: https://raw.githubusercontent.com/mafredri/cdp/3c5eab7ffc5cbee667b0a813ce470ac423792811/protocol/io/stream_reader.go. +type streamReader struct { + ctx context.Context + handle cdprotoio.StreamHandle + r io.Reader + pos int + eof bool +} + +// Read a chunk of the stream. +func (reader *streamReader) Read(p []byte) (n int, err error) { + if reader.r != nil { + // Continue reading from buffer. + return reader.read(p) + } + + if reader.eof { + return 0, io.EOF + } + + if len(p) == 0 { + return 0, nil + } + + // Chromium might have an off-by-one when deciding the maximum size (at + // least for base64 encoded data), usually it will overflow. We subtract + // one to make sure it fits into p. + size := len(p) - 1 + if size < 1 { + // Safety-check to avoid crashing Chrome (e.g. via SetSize(-1)). + size = 1 + } + + reply, err := reader.next(reader.pos, size) + if err != nil { + return 0, err + } + + reader.eof = reply.EOF + + switch { + case reply.Base64encoded: + b := []byte(reply.Data) + size := base64.StdEncoding.DecodedLen(len(b)) + + // Safety-check for fast-path to avoid panics. + if len(p) >= size { + n, err = base64.StdEncoding.Decode(p, b) + reader.pos += n + + return n, err + } + + reader.r = base64.NewDecoder(base64.StdEncoding, bytes.NewReader(b)) + default: + reader.r = strings.NewReader(reply.Data) + } + + return reader.read(p) +} + +// Close closes the stream, discard any temporary backing storage. +func (reader *streamReader) Close() error { + err := cdprotoio.Close(reader.handle).Do(reader.ctx) + if err == nil { + return nil + } + + return fmt.Errorf("close Chromium stream: %w", err) +} + +func (reader *streamReader) next(pos, size int) (cdprotoio.ReadReturns, error) { + params := cdprotoio. + Read(reader.handle). + WithOffset(int64(pos)). + WithSize(int64(size)) + + var res cdprotoio.ReadReturns + err := cdp.Execute(reader.ctx, cdprotoio.CommandRead, params, &res) + + if err == nil { + return res, nil + } + + return res, fmt.Errorf("execute IO.read command: %w", err) +} + +func (reader *streamReader) read(p []byte) (n int, err error) { + n, err = reader.r.Read(p) + reader.pos += n + + if !reader.eof && err == io.EOF { + reader.r = nil + err = nil + } + + return n, err +} + +// Interface guards. +var ( + _ io.Reader = (*streamReader)(nil) +) diff --git a/internal/pkg/files/files.go b/internal/pkg/files/files.go new file mode 100644 index 0000000..b863be5 --- /dev/null +++ b/internal/pkg/files/files.go @@ -0,0 +1,12 @@ +package files + +import ( + "errors" + "os" +) + +// CheckFileExists check if file exists +func CheckFileExists(filePath string) bool { + _, error := os.Stat(filePath) + return !errors.Is(error, os.ErrNotExist) +} diff --git a/internal/pkg/logger/logger.go b/internal/pkg/logger/logger.go index 18a38e5..1689254 100644 --- a/internal/pkg/logger/logger.go +++ b/internal/pkg/logger/logger.go @@ -41,14 +41,14 @@ func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) { return []byte(message), nil } -func init(){ +func init() { userConfigDir, _ := os.UserConfigDir() - logFilePath := filepath.Join(userConfigDir, GeektimeLogFolder, GeektimeLogFolder + ".log") + logFilePath := filepath.Join(userConfigDir, GeektimeLogFolder, GeektimeLogFolder+".log") logger.SetReportCaller(true) logger.SetLevel(logrus.InfoLevel) logger.SetFormatter(&customFormatter{}) - logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err == nil { logger.Out = logFile } else { @@ -68,10 +68,10 @@ func Warnf(format string, args ...interface{}) { } // Error wrapper logrus log.Error -func Error(err error, args ...interface{}){ - if err != nil{ +func Error(err error, args ...interface{}) { + if err != nil { logger.WithError(err).Error(args...) - }else{ + } else { logger.Error(args...) } -} \ No newline at end of file +} diff --git a/internal/pkg/m3u8/m3u8.go b/internal/pkg/m3u8/m3u8.go index bd1b2a1..7538484 100644 --- a/internal/pkg/m3u8/m3u8.go +++ b/internal/pkg/m3u8/m3u8.go @@ -15,7 +15,7 @@ var ( // Parse do m3u8 url GET request, and extract ts file names and check if it's encrypt video func Parse(client *geektime.Client, m3u8url string) (tsFileNames []string, isVodEncryptVideo bool, err error) { - m3u8Resp, err := client.HTTPClient.R().SetDoNotParseResponse(true).Get(m3u8url) + m3u8Resp, err := client.RestyClient.R().SetDoNotParseResponse(true).Get(m3u8url) if err != nil { return nil, false, err } diff --git a/internal/video/video.go b/internal/video/video.go index 45f3b19..0be0525 100644 --- a/internal/video/video.go +++ b/internal/video/video.go @@ -3,11 +3,11 @@ package video import ( "context" "fmt" - "io/ioutil" "net/url" "os" "path" "path/filepath" + "strconv" "strings" "time" @@ -17,12 +17,14 @@ import ( "github.com/nicoxiang/geektime-downloader/internal/pkg/crypto" "github.com/nicoxiang/geektime-downloader/internal/pkg/downloader" "github.com/nicoxiang/geektime-downloader/internal/pkg/filenamify" + "github.com/nicoxiang/geektime-downloader/internal/pkg/files" "github.com/nicoxiang/geektime-downloader/internal/pkg/m3u8" "github.com/nicoxiang/geektime-downloader/internal/video/vod" ) const ( // syncByte = uint8(71) //0x47 + // TSExtension ... TSExtension = ".ts" ) @@ -76,15 +78,43 @@ func DownloadArticleVideo(ctx context.Context, concurrency) } +// DownloadEnterpriseArticleVideo download enterprise video +func DownloadEnterpriseArticleVideo(ctx context.Context, + client *geektime.Client, + articleID int, + projectDir string, + quality string, + concurrency int, +) error { + articleInfo, err := client.V1EnterpriseArticleDetail(strconv.Itoa(articleID)) + if err != nil { + return err + } + if articleInfo.Data.Video.ID == "" { + return nil + } + playAuth, err := client.EnterpriseVideoPlayAuth(strconv.Itoa(articleID), articleInfo.Data.Video.ID) + if err != nil { + return err + } + return downloadAliyunVodEncryptVideo(ctx, + client, + playAuth, + articleInfo.Data.Article.Title, + projectDir, + quality, + articleInfo.Data.Video.ID, + concurrency) +} + // DownloadUniversityVideo ... func DownloadUniversityVideo(ctx context.Context, client *geektime.Client, articleID int, - currentProduct geektime.Product, + currentProduct geektime.Course, projectDir string, quality string, concurrency int) error { - playAuthInfo, err := client.UniversityVideoPlayAuth(articleID, currentProduct.ID) if err != nil { return err @@ -133,8 +163,8 @@ func downloadAliyunVodEncryptVideo(ctx context.Context, return download(ctx, tsURLPrefix, videoTitle, projectDir, tsFileNames, []byte(decryptKey), playInfo.Size, isVodEncryptVideo, concurrency) } -// DownloadMP4 ... -func DownloadMP4(ctx context.Context, title, projectDir string, mp4URLs []string) (err error) { +// DownloadMP4 download MP4 resources in article +func DownloadMP4(ctx context.Context, title, projectDir string, mp4URLs []string, overwrite bool) (err error) { filenamifyTitle := filenamify.Filenamify(title) videoDir := filepath.Join(projectDir, "videos", filenamifyTitle) if err = os.MkdirAll(videoDir, os.ModePerm); err != nil { @@ -145,6 +175,10 @@ func DownloadMP4(ctx context.Context, title, projectDir string, mp4URLs []string u, _ := url.Parse(mp4URL) dst := filepath.Join(videoDir, path.Base(u.Path)) + if files.CheckFileExists(dst) && !overwrite { + continue + } + headers := make(map[string]string, 2) headers[geektime.Origin] = geektime.DefaultBaseURL headers[geektime.UserAgent] = geektime.DefaultUserAgent @@ -185,12 +219,12 @@ func download(ctx context.Context, for _, tsFileName := range tsFileNames { u := tsURLPrefix + tsFileName dst := filepath.Join(tempVideoDir, tsFileName) - + headers := make(map[string]string, 2) headers[geektime.Origin] = geektime.DefaultBaseURL headers[geektime.UserAgent] = geektime.DefaultUserAgent - - fileSize, err := downloader.DownloadFileConcurrently(ctx, dst, u, headers, 5) + + fileSize, err := downloader.DownloadFileConcurrently(ctx, dst, u, headers, concurrency) if err != nil { return err } @@ -207,7 +241,7 @@ func download(ctx context.Context, } func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte, isVodEncryptVideo bool) error { - tempTSFiles, err := ioutil.ReadDir(tempVideoDir) + tempTSFiles, err := os.ReadDir(tempVideoDir) if err != nil { return err } @@ -220,7 +254,7 @@ func mergeTSFiles(tempVideoDir, filenamifyTitle, projectDir string, key []byte, return err } for _, tempTSFile := range tempTSFiles { - f, err := ioutil.ReadFile(filepath.Join(tempVideoDir, tempTSFile.Name())) + f, err := os.ReadFile(filepath.Join(tempVideoDir, tempTSFile.Name())) if err != nil { return err } @@ -267,7 +301,7 @@ func addBarValue(bar *pb.ProgressBar, written int64) { } } -func getUniversityVideoTitle(articleID int, currentProduct geektime.Product) string { +func getUniversityVideoTitle(articleID int, currentProduct geektime.Course) string { for _, v := range currentProduct.Articles { if v.AID == articleID { return v.Title @@ -284,7 +318,7 @@ func extractTSURLPrefix(m3u8url string) string { func getPlayInfo(client *geektime.Client, playInfoURL, quality string) (vod.PlayInfo, error) { var getPlayInfoResp GetPlayInfoResponse var playInfo vod.PlayInfo - _, err := client.HTTPClient.R(). + _, err := client.RestyClient.R(). SetResult(&getPlayInfoResp). Get(playInfoURL)