From 956865ab8f4e2d49bdbfcb71347d6aba7b24506c Mon Sep 17 00:00:00 2001 From: Lukas Matt Date: Mon, 26 Feb 2018 21:09:41 +0100 Subject: [PATCH] Replace webhook build by agent construct This will enable us to restart tests in future and a void broken builds after restarting the server --- build.go | 261 ++++++++++++++++++++++++++++++++++++++++++++++++++++- server.go | 3 + travis.go | 208 ------------------------------------------ webhook.go | 18 +++- 4 files changed, 277 insertions(+), 213 deletions(-) delete mode 100644 travis.go diff --git a/build.go b/build.go index 8b480fd..bff5573 100644 --- a/build.go +++ b/build.go @@ -18,12 +18,33 @@ package main import ( + "fmt" + "net/http" + "strings" + "encoding/json" + "golang.org/x/oauth2" + "github.com/google/go-github/github" + "io/ioutil" + "context" + "time" "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" +) + +const ( + STATUS_ERROR = "error" + STATUS_FAIL = "failure" + STATUS_PENDING = "pending" + STATUS_SUCCESS = "success" + + BUILD_NOT_STARTED = 0 + BUILD_PENDING = 1 + BUILD_FINISHED = 2 ) type Build struct { gorm.Model - RepoID int + RepoID uint Matrix string TravisType string TravisRequestID int64 @@ -31,10 +52,248 @@ type Build struct { PRUser string PRRepo string PRSha string + Status int `gorm:"default:0"` Repo Repo } +type Builds []Build + +type TravisStatus struct { + State string `json:"state"` + Builds []struct { + ID int64 `json:"id"` + State string `json:"state"` + } `json:"builds"` +} + +type TravisRequest struct { + Type string `json:"@type"` + Request struct { + ID int64 `json:"id"` + Repository struct { + ID int64 `json:"id"` + } `json:"repository"` + } `json:"request"` +} + +var ( + travisTestEndpoint = "https://travis-ci.org/thefederationinfo/federation-tests/builds/%d" + travisTestDescription = "Continuous integration tests for the federation network" + travisTestContext = "Federation Suite" + travisEndpoint = "https://api.travis-ci.org/repo/" + travisSlug = "thefederationinfo%2Ffederation-tests" + travisRequests = travisEndpoint + travisSlug + "/requests" +) + func (build *Build) AfterFind(db *gorm.DB) error { return db.Model(build).Related(&build.Repo).Error } + +func BuildAgent() { + logger.Println("Started build agent") + db, err := gorm.Open(databaseDriver, databaseDSN) + if err != nil { + panic("failed to connect database") + } + defer db.Close() + + for { + var builds Builds + err := db.Find(&builds).Error + if err != nil { + logger.Printf("Cannot fetch new builds: %+v\n", err) + continue + } + for _, build := range builds { + if build.Status == BUILD_NOT_STARTED { + build.Status = BUILD_PENDING + err = db.Save(&build).Error + if err != nil { + logger.Printf("#%d: cannot update status: %+v\n", build.ID, err) + continue + } + logger.Printf("#%d: starting new build\n", build.ID) + go build.Run(false) + } + } + time.Sleep(10 * time.Second) + } + logger.Println("Build agent died :S\n") +} + +func (build *Build) Run(watch bool) { + db, err := gorm.Open(databaseDriver, databaseDSN) + if err != nil { + logger.Println(err) + return + } + defer db.Close() + + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: build.Repo.Token}, + ) + tc := oauth2.NewClient(context.Background(), ts) + client := github.NewClient(tc) + + if !watch { + status := build.TriggerTravis() + logger.Printf("#%d: travis build triggered\n", build.ID) + build.UpdateStatus(client, status) + if status == STATUS_ERROR { + (*build).Status = BUILD_FINISHED + err := db.Save(&build).Error + if err != nil { + logger.Printf("#%d: cannot update status: %+v\n", build.ID, err) + } + return + } + } + + var statusHref string + started := time.Now() + timeout := started.Add(-1 * time.Hour) + for { + status := build.FetchStatus() + logger.Printf("#%d: request status: %+v", build.ID, status) + if status.State == "finished" { + var failure bool + var passed int + for _, build := range status.Builds { + // canceled, passed, errored, started + switch build.State { + case "canceled": + fallthrough + case "errored": + fallthrough + case "failed": + failure = true + case "passed": + passed += 1 + } + } + if failure { + build.UpdateStatus(client, STATUS_FAIL, statusHref) + (*build).Status = BUILD_FINISHED + err := db.Save(&build).Error + if err != nil { + logger.Printf("#%d: cannot update status: %+v\n", build.ID, err) + } + break + } else if len(status.Builds) == passed { + build.UpdateStatus(client, STATUS_SUCCESS, statusHref) + (*build).Status = BUILD_FINISHED + err := db.Save(&build).Error + if err != nil { + logger.Printf("#%d: cannot update status: %+v\n", build.ID, err) + } + break + } + // update the status line in the PR once + if len(status.Builds) > 0 && statusHref == "" { + statusHref = fmt.Sprintf(travisTestEndpoint, status.Builds[0].ID) + build.UpdateStatus(client, STATUS_PENDING, statusHref) + } + } + + if time.Now().Before(timeout) { + build.UpdateStatus(client, STATUS_ERROR, statusHref) + logger.Printf("#%d: Timeout..\n", build.ID) + (*build).Status = BUILD_FINISHED + err := db.Save(&build).Error + if err != nil { + logger.Printf("#%d: cannot update status: %+v\n", build.ID, err) + } + break + } + time.Sleep(1 * time.Minute) + } + logger.Printf("#%d: Travis build finished\n", build.ID) +} + +func (build *Build) TriggerTravis() string { + var requestJson = `{"request":{"branch":"continuous_integration","config":{"env":{"matrix":[%s]}}}}` + resp, err := build.fetch("POST", travisRequests, + fmt.Sprintf(requestJson, build.Matrix)) + if err != nil { + fmt.Println("#%d: Cannot create request: %+v", build.ID, err) + return STATUS_ERROR + } + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("#%d: Cannot read status body: %+v", build.ID, err) + return STATUS_ERROR + } + + var request TravisRequest + err = json.Unmarshal(b, &request) + if err != nil { + fmt.Println("#%d: Cannot unmarshal body: %+v <> %s", build.ID, err, string(b)) + return STATUS_ERROR + } + + build.TravisType = request.Type + build.TravisRequestID = request.Request.ID + build.TravisRepositoryID = request.Request.Repository.ID + + return STATUS_PENDING +} + +func (build *Build) UpdateStatus(client *github.Client, params... string) { + if len(params) <= 0 { + panic("state is mandatory") + } + + repoStatus := github.RepoStatus{ + State: ¶ms[0], + Description: &travisTestDescription, + Context: &travisTestContext, + } + if len(params) >= 2 { + repoStatus.TargetURL = ¶ms[1] + } + if _, _, err := client.Repositories.CreateStatus(context.Background(), + build.PRUser, build.PRRepo, build.PRSha, &repoStatus); err != nil { + fmt.Println("#%d: Cannot update status: %+v", build.ID, err) + } +} + +func (build *Build) FetchStatus() (status TravisStatus) { + resp, err := build.fetch("GET", fmt.Sprintf( + "%s%d%s%d", travisEndpoint, build.TravisRepositoryID, + "/request/", build.TravisRequestID), "") + if err != nil { + logger.Printf("#%d: Cannot fetch build status: %+v\n", build.ID, err) + return + } + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger.Printf("#%d: Cannot read status body: %+v\n", build.ID, err) + return + } + + err = json.Unmarshal(b, &status) + if err != nil { + logger.Printf("#%d: Cannot unmarshal body: %+v <> %s\n", build.ID, err, string(b)) + return + } + return +} + +func (build *Build) fetch(method, url, body string) (*http.Response, error) { + req, err := http.NewRequest(method, url, strings.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("Travis-API-Version", "3") + req.Header.Set("Authorization", "token " + travisToken) + + client := &http.Client{} + return client.Do(req) +} diff --git a/server.go b/server.go index eb4b7a3..40fd5cb 100644 --- a/server.go +++ b/server.go @@ -85,6 +85,9 @@ func main() { return } + // start build agent + go BuildAgent() + http.HandleFunc("/", frontend) http.HandleFunc("/auth", authentication) http.HandleFunc("/hook", webhook) diff --git a/travis.go b/travis.go deleted file mode 100644 index a6b5c09..0000000 --- a/travis.go +++ /dev/null @@ -1,208 +0,0 @@ -// -// TheFederation Github Integration Server -// Copyright (C) 2018 Lukas Matt -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -package main - -import ( - "fmt" - "net/http" - "strings" - "encoding/json" - "golang.org/x/oauth2" - "github.com/google/go-github/github" - "io/ioutil" - "context" - "time" -) - -const ( - STATUS_ERROR = "error" - STATUS_FAIL = "failure" - STATUS_PENDING = "pending" - STATUS_SUCCESS = "success" -) - -var ( - travisTestEndpoint = "https://travis-ci.org/thefederationinfo/federation-tests/builds/%d" - travisTestDescription = "Continuous integration tests for the federation network" - travisTestContext = "Federation Suite" - travisEndpoint = "https://api.travis-ci.org/repo/" - travisSlug = "thefederationinfo%2Ffederation-tests" - travisRequests = travisEndpoint + travisSlug + "/requests" -) - -type TravisStatus struct { - State string `json:"state"` - Builds []struct { - ID int64 `json:"id"` - State string `json:"state"` - } `json:"builds"` -} - -type TravisRequest struct { - Type string `json:"@type"` - Request struct { - ID int64 `json:"id"` - Repository struct { - ID int64 `json:"id"` - } `json:"repository"` - } `json:"request"` -} - -func (request *TravisRequest) Run(token string, matrix []string, pr *github.PullRequest) { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(context.Background(), ts) - client := github.NewClient(tc) - - status := request.Build(matrix) - request.UpdateStatus(client, pr, status) - if status == STATUS_ERROR { - return - } - - var statusHref string - started := time.Now() - timeout := started.Add(-1 * time.Hour) - logger.Printf("#%d: Travis build triggered\n", request.Request.ID) - for { - status := request.Status() - logger.Printf("#%d: request status: %+v", request.Request.ID, status) - if status.State == "finished" { - var failure bool - var passed int - for _, build := range status.Builds { - // canceled, passed, errored, started - switch build.State { - case "canceled": - fallthrough - case "errored": - fallthrough - case "failed": - failure = true - case "passed": - passed += 1 - } - } - if failure { - request.UpdateStatus(client, pr, STATUS_FAIL, statusHref) - } else if len(status.Builds) == passed { - request.UpdateStatus(client, pr, STATUS_SUCCESS, statusHref) - break - } - // update the status lin in the PR once - if len(status.Builds) > 0 && statusHref == "" { - statusHref = fmt.Sprintf(travisTestEndpoint, status.Builds[0].ID) - request.UpdateStatus(client, pr, STATUS_PENDING, statusHref) - } - } - - if time.Now().Before(timeout) { - request.UpdateStatus(client, pr, STATUS_ERROR, statusHref) - logger.Printf("#%d: Timeout..\n", request.Request.ID) - break - } - time.Sleep(1 * time.Minute) - } - logger.Printf("#%d: Travis build finished\n", request.Request.ID) -} - -func (request *TravisRequest) UpdateStatus(client *github.Client, pr *github.PullRequest, params... string) { - if len(params) <= 0 { - panic("state is mandatory") - } - - repoStatus := github.RepoStatus{ - State: ¶ms[0], - Description: &travisTestDescription, - Context: &travisTestContext, - } - if len(params) >= 2 { - repoStatus.TargetURL = ¶ms[1] - } - if _, _, err := client.Repositories.CreateStatus( - context.Background(), *pr.Head.User.Login, *pr.Head.Repo.Name, - *pr.Head.SHA, &repoStatus); err != nil { - fmt.Println("#%d: Cannot update status: %+v", request.Request.ID, err) - } -} - -func (request *TravisRequest) Build(matrix []string) string { - var requestJson = `{"request":{"branch":"continuous_integration","config":{"env":{"matrix":[%s]}}}}` - resp, err := request.fetch("POST", travisRequests, - fmt.Sprintf(requestJson, strings.Join(matrix, ","))) - if err != nil { - fmt.Println("#%d: Cannot create request: %+v", request.Request.ID, err) - return STATUS_ERROR - } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - fmt.Println("#%d: Cannot read status body: %+v", request.Request.ID, err) - return STATUS_ERROR - } - - err = json.Unmarshal(b, &request) - if err != nil { - fmt.Println("#%d: Cannot unmarshal body: %+v <> %s", - request.Request.ID, err, string(b)) - return STATUS_ERROR - } - return STATUS_PENDING -} - -func (request *TravisRequest) Status() (status TravisStatus) { - resp, err := request.fetch("GET", fmt.Sprintf( - "%s%d%s%d", travisEndpoint, - request.Request.Repository.ID, - "/request/", request.Request.ID), "") - if err != nil { - logger.Printf("#%d: Cannot fetch build status: %+v\n", request.Request.ID, err) - return - } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - logger.Printf("#%d: Cannot read status body: %+v\n", request.Request.ID, err) - return - } - - err = json.Unmarshal(b, &status) - if err != nil { - logger.Printf("#%d: Cannot unmarshal body: %+v <> %s\n", - request.Request.ID, err, string(b)) - return - } - return -} - -func (request *TravisRequest) fetch(method, url, body string) (*http.Response, error) { - req, err := http.NewRequest(method, url, strings.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("Travis-API-Version", "3") - req.Header.Set("Authorization", "token " + travisToken) - - client := &http.Client{} - return client.Do(req) -} diff --git a/webhook.go b/webhook.go index d0d1f6c..70abe97 100644 --- a/webhook.go +++ b/webhook.go @@ -76,13 +76,23 @@ func webhook(w http.ResponseWriter, r *http.Request) { // return //} - var request TravisRequest - go request.Run(repo.Token, - []string{fmt.Sprintf( + build := Build{ + RepoID: repo.ID, + Matrix: fmt.Sprintf( `"PRREPO=%s PRSHA=%s"`, *pr.Head.Repo.CloneURL, *pr.Head.SHA, - )}, pr) + ), + PRUser: *pr.Head.User.Login, + PRRepo: *pr.Head.Repo.Name, + PRSha: *pr.Head.SHA, + } + err = db.Create(&build).Error + if err != nil { + logger.Println(err) + fmt.Fprintf(w, `{"error":"database error"}`) + return + } fmt.Fprintf(w, `{}`) }