From 14ad9b316ecb628932d32749e10ba9e463811576 Mon Sep 17 00:00:00 2001 From: boratanrikulu Date: Sat, 11 Jul 2020 13:00:02 +0300 Subject: [PATCH 1/2] Fix translate and author issues --- models/lyric.go | 64 ++++++++++++++++++++++++++++++++++-------------- views/songs.html | 6 ++--- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/models/lyric.go b/models/lyric.go index 4904b45..7679cef 100644 --- a/models/lyric.go +++ b/models/lyric.go @@ -46,6 +46,16 @@ type Genius struct { } `json:"response"` } +var AllowedTranslationLanguages = []string{"Turkish", + "English", + "Italian", + "Swedish", + "German", + "French", + "Russian", + "Spanish", +} + // Public Methods func (l *Lyric) GetLyric(artistName string, songName string) { @@ -141,9 +151,7 @@ func getFromSecondSource(l *Lyric, artistName string, songName string) { c.OnHTML("div.song_body-lyrics div.lyrics p", func(e *colly.HTMLElement) { lines := strings.SplitAfter(e.Text, "\n") for _, line := range lines { - if line == "\n" { - line = "" - } + line = strings.Trim(strings.TrimSpace(line), "\n") l.Lines = append(l.Lines, line) } }) @@ -190,13 +198,14 @@ func getFromFirstSource(l *Lyric, artistName string, songName string) { // Song lyric page. cc.OnHTML("div#song-body .ltf .par div, .emptyline", func(e *colly.HTMLElement) { - l.Lines = append(l.Lines, e.Text) + line := strings.Trim(strings.TrimSpace(e.Text), "\n") + l.Lines = append(l.Lines, line) }) // Song's language. cc.OnHTML(".langsmall-song span.langsmall-languages", func(e *colly.HTMLElement) { if strings.TrimSpace(e.Text) != "" { - l.Language = e.Text + l.Language = strings.Trim(strings.TrimSpace(e.Text), "\n") } }) @@ -215,32 +224,49 @@ func getFromFirstSource(l *Lyric, artistName string, songName string) { func getTranslations(l *Lyric, url string) { c := colly.NewCollector() - allowedTranslationLanguages := "Turkish English Italian Swedish German French" + addedTranslations := []string{} // Translation list for the song. - c.OnHTML("div.song-node-info li.song-node-info-translate a[href]", func(e *colly.HTMLElement) { + c.OnHTML("div.song-node div.masonry-grid div.song-list-translations-list a[href]", func(e *colly.HTMLElement) { cc := colly.NewCollector() // Lyric translations for the song. + cc.OnHTML("div.translate-node-text", func(e *colly.HTMLElement) { - translate := Translate{} - translate.Language = e.ChildText("div.langsmall-song span.mobile-only-inline") - translate.Author.Name = e.ChildText(".authorsubmitted a") - translate.Author.Href = e.ChildAttr(".authorsubmitted a[href]", "href") - if translate.Language != "" { - translate.Title = e.ChildText("h2.title-h2") - e.ForEach(".ltf .par div, .emptyline", func(_ int, e *colly.HTMLElement) { - translate.Lines = append(translate.Lines, e.Text) + language := e.ChildText("div.langsmall-song span.mobile-only-inline") + if !contains(addedTranslations, language) { + translate := Translate{} + translate.Language = language + e.ForEach("div.authorsubmitted a[href]", func(c int, e *colly.HTMLElement) { + if c == 0 { + translate.Author.Name = e.Text + translate.Author.Href = e.Attr("href") + } }) - l.Translates = append(l.Translates, translate) + if translate.Language != "" { + translate.Title = e.ChildText("h2.title-h2") + e.ForEach(".ltf .par div, .emptyline", func(_ int, e *colly.HTMLElement) { + line := strings.Trim(strings.TrimSpace(e.Text), "\n") + translate.Lines = append(translate.Lines, line) + }) + l.Translates = append(l.Translates, translate) + addedTranslations = append(addedTranslations, translate.Language) + } } }) - // TODO - // Fix more-then-one translate issue. - if strings.Contains(allowedTranslationLanguages, e.Text) { + if contains(AllowedTranslationLanguages, e.Text) { cc.Visit("https://lyricstranslate.com/" + e.Attr("href")) } }) c.Visit(url) } + +func contains(array []string, value string) bool { + for _, v := range array { + if v == value { + return true + } + } + return false +} diff --git a/views/songs.html b/views/songs.html index 3d461f0..9d6b2ca 100644 --- a/views/songs.html +++ b/views/songs.html @@ -35,7 +35,7 @@

{{ end }} {{ if .Song.Lyric.Source }}
-

These lyrics were taken from {{ .Song.Lyric.Source }}.

+

These lyrics were taken from {{ .Song.Lyric.Source }}

{{ end }} {{ else }}
@@ -53,9 +53,9 @@

{{ end }} From b90858a398cc42997527271dd53b29137e2325f3 Mon Sep 17 00:00:00 2001 From: boratanrikulu Date: Sat, 11 Jul 2020 13:42:05 +0300 Subject: [PATCH 2/2] Add database connection --- .gitignore | 3 +- controllers/lyric_controler.go | 2 +- env.sample | 12 ++++-- main.go | 46 ++++++++++++++++++++ models/lyric.go | 78 ++++++++++++++++++++++++++++++++-- 5 files changed, 133 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 3bf780b..6ed1f40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea -.env \ No newline at end of file +.env +database diff --git a/controllers/lyric_controler.go b/controllers/lyric_controler.go index baa276f..df880f2 100644 --- a/controllers/lyric_controler.go +++ b/controllers/lyric_controler.go @@ -24,7 +24,7 @@ func LyricGet(w http.ResponseWriter, r *http.Request) { albumImage := r.URL.Query().Get("albumImage") l := new(models.Lyric) - l.GetLyric(artistName, songName) + l.GetLyricByCheckingDatabase(artistName, songName) pageData := LyricPageData{ Artist: models.Artist{ diff --git a/env.sample b/env.sample index 6250d89..436d449 100644 --- a/env.sample +++ b/env.sample @@ -1,8 +1,14 @@ -# Spotify API - Application -# s-lyrics +# S-Lyrics, secret keys + +# Spotiy CLIENT_ID= CLIENT_SECRET= REDIRECT_URI= -GENIUS_ID + +# Genius +GENIUS_ID= GENIUS_SECRET= GENIUS_ACCESS= + +# Database +DATABASE_ADDRESS= diff --git a/main.go b/main.go index c08a6f9..0c11a0c 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,12 @@ package main import ( + "bytes" + "fmt" + "log" "net/http" "os" + "os/exec" "github.com/gorilla/mux" "github.com/joho/godotenv" @@ -12,11 +16,18 @@ import ( func main() { godotenv.Load() + + err := cloneOrPullDatabase(os.Getenv("DATABASE_ADDRESS")) + if err != nil { + log.Fatalln(err) + } + r := mux.NewRouter() r.HandleFunc("/spotify", controllers.SpotifyGet).Methods("GET") r.HandleFunc("/", controllers.WelcomeGet).Methods("GET") r.HandleFunc("/logout", controllers.LogoutGet).Methods("GET") r.PathPrefix("/assets").Handler(http.StripPrefix("/assets/", http.FileServer(http.Dir("./assets/")))) + serve(r, "3000") } @@ -27,3 +38,38 @@ func serve(r *mux.Router, defaultPort string) { } http.ListenAndServe(":"+port, r) } + +func cloneOrPullDatabase(databaseURL string) error { + var errOutput bytes.Buffer + + if folderExists("./database/.git") { + cmd := exec.Command("git", "pull", "origin", "master") + cmd.Dir = "./database" + cmd.Stderr = &errOutput + + err := cmd.Run() + if err != nil { + return fmt.Errorf("%s", errOutput) + } + + return nil + } + + cmd := exec.Command("git", "clone", databaseURL, "./database") + cmd.Stderr = &errOutput + + err := cmd.Run() + if err != nil { + return fmt.Errorf("%s", errOutput) + } + + return nil +} + +func folderExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return info.IsDir() +} diff --git a/models/lyric.go b/models/lyric.go index 7679cef..1e5a2d2 100644 --- a/models/lyric.go +++ b/models/lyric.go @@ -17,7 +17,7 @@ type Lyric struct { Lines []string IsAvaible bool Language string - Translates []Translate + Translates []Translate `json:"-"` Source string } @@ -59,16 +59,31 @@ var AllowedTranslationLanguages = []string{"Turkish", // Public Methods func (l *Lyric) GetLyric(artistName string, songName string) { + // Regex for the song name songName = songRegex(songName) // Get from lyricstranslates.com + // getFromFirstSource(l, artistName, songName) getFromFirstSource(l, artistName, songName) + // If there is no lyric on the first source, + // then get it from genius.com if !l.IsAvaible { - // If there is no lyric on the first source, - // then get it from genius.com getFromSecondSource(l, artistName, songName) } + +} + +func (l *Lyric) GetLyricByCheckingDatabase(artistName string, songName string) { + // Get from local storage source. + getFromDatabase(l, artistName, songName) + + if l.IsAvaible { + // Change the source as s-lyrics if it is gotten from database. + l.Source = "s-lyrics.com" + } else { + l.GetLyric(artistName, songName) + } } // Private Methods @@ -84,6 +99,7 @@ func songRegex(song string) string { `(?i)\[.*?cover.*?\]`, // Removes all [...cover...]s from song name. `(?i)\(.*?with.*?\)`, // Removes all (...with...)s from song name. `(?i)\[.*?with.*?\]`, // Removes all [...with...]s from song name. + // `[-(].+`, // Removes all thigns after '-'. } // Run regexs. @@ -98,6 +114,28 @@ func songRegex(song string) string { return song } +func getFromDatabase(l *Lyric, artistName string, songName string) { + fileName := getFileName(artistName, songName) + + f, err := os.Open(fileName) + if err != nil { + return + } + defer f.Close() + + b, err := ioutil.ReadAll(f) + if err != nil { + return + } + + err = json.Unmarshal(b, l) + if err != nil { + return + } + + getTranslationsFromDatabase(l, fileName) +} + func getFromSecondSource(l *Lyric, artistName string, songName string) { u, _ := url.Parse("https://api.genius.com/search") q, _ := url.ParseQuery(u.RawQuery) @@ -262,6 +300,40 @@ func getTranslations(l *Lyric, url string) { c.Visit(url) } +func getTranslationsFromDatabase(l *Lyric, fileName string) { + for _, translate := range AllowedTranslationLanguages { + fName := fileName + "_" + translate + + f, err := os.Open(fName) + if err != nil { + continue + } + defer f.Close() + + b, err := ioutil.ReadAll(f) + if err != nil { + continue + } + + t := Translate{} + err = json.Unmarshal(b, &t) + if err != nil { + continue + } + + l.Translates = append(l.Translates, t) + } +} + +func getFileName(artist string, songName string) string { + fileName := artist + "-" + songName + ".json" + fileName = strings.ReplaceAll(fileName, "/", "_") + fileName = strings.ReplaceAll(fileName, "\\", "_") + fileName = "./database/lyrics/" + fileName + + return fileName +} + func contains(array []string, value string) bool { for _, v := range array { if v == value {