Skip to content

Commit

Permalink
[CI-6246]: Add TI Support (#7)
Browse files Browse the repository at this point in the history 
* [CI-6246]: Add TI Support

* [CI-6246]: Added UTs
  • Loading branch information
@rutvijmehta-harness
rutvijmehta-harness authored Jan 28, 2023
1 parent 9818c13 commit 8f0193c
Show file tree
Hide file tree
Showing 17 changed files with 425 additions and 253 deletions.
2 changes: 2 additions & 0 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ type (
TargetBranch string `json:"target_branch,omitempty"`
CommitBranch string `json:"commit_branch,omitempty"`
CommitLink string `json:"commit_link,omitempty"`
// Local path where TI data will be stored
TmpDir string
}

TestReport struct {
Expand Down
26 changes: 23 additions & 3 deletions handler/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package handler

import (
"encoding/json"
"fmt"
"net/http"
"os"
"runtime"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/harness/harness-docker-runner/logger"
"github.com/harness/harness-docker-runner/pipeline"
prruntime "github.com/harness/harness-docker-runner/pipeline/runtime"
"github.com/harness/harness-docker-runner/ti"

"github.com/sirupsen/logrus"
)
Expand All @@ -31,7 +33,8 @@ var random = func() string {
return uniuri.NewLen(20)
}

// HandleExecuteStep returns an http.HandlerFunc that executes a step
// HandleSetup returns an http.HandlerFunc that does the initial setup
// for executing the step
func HandleSetup() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
st := time.Now()
Expand All @@ -46,6 +49,12 @@ func HandleSetup() http.HandlerFunc {

updateVolumes(s)

// Add ti volume where all the TI related data (CG, Agent logs, config) will be stored
// Add this dir to TIConfig for uploading the data
tiVolume := getTiVolume(s.ID)
s.Volumes = append(s.Volumes, tiVolume)
s.TIConfig.TmpDir = tiVolume.HostPath.Path

setProxyEnvs(s.Envs)
engine, err := engine.NewEnv(docker.Opts{})
if err != nil {
Expand All @@ -55,8 +64,6 @@ func HandleSetup() http.HandlerFunc {
}
stepExecutor := prruntime.NewStepExecutor(engine)
state := pipeline.NewState()
// s.LogConfig.IndirectUpload = true
// s.LogConfig.URL = "http://localhost:8079"
state.Set(s.Volumes, s.Secrets, s.LogConfig, s.TIConfig, s.SetupRequestConfig.Network.ID)

log := logrus.New()
Expand Down Expand Up @@ -161,6 +168,19 @@ func getSharedVolume() *spec.Volume {
}
}

// getTiVolume returns a volume (directory) which is used to store TI related data
func getTiVolume(setupID string) *spec.Volume {
tiDir := fmt.Sprintf("%s-%s", ti.VolumePath, sanitize(setupID))
return &spec.Volume{
HostPath: &spec.VolumeHostPath{
Name: ti.VolumeName,
Path: tiDir,
Create: true,
Remove: true,
},
}
}

func sanitize(r string) string {
return strings.ReplaceAll(r, "[-_]", "")
}
Expand Down
2 changes: 1 addition & 1 deletion handler/step.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func HandleStartStep() http.HandlerFunc {
ctx := r.Context()
logger.FromRequest(r).WithField("stage_id", s.StageRuntimeID).
WithField("step_id", s.ID).Traceln("starting step execution")
if err := stageData.StepExecutor.StartStep(ctx, &s, stageData.State.GetSecrets(), stageData.State.GetLogStreamClient(), stageData.State.GetTiClient()); err != nil {
if err := stageData.StepExecutor.StartStep(ctx, &s, stageData.State.GetSecrets(), stageData.State.GetLogStreamClient(), *stageData.State.GetTIConfig()); err != nil {
WriteError(w, err)
}

Expand Down
6 changes: 3 additions & 3 deletions pipeline/runtime/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,18 @@ import (
"fmt"
"io"
"os"
"time"

"github.com/drone/runner-go/pipeline/runtime"
"github.com/sirupsen/logrus"

"github.com/harness/harness-docker-runner/api"
"github.com/harness/harness-docker-runner/engine"
"github.com/harness/harness-docker-runner/pipeline"
ticlient "github.com/harness/harness-docker-runner/ti/client"
"github.com/harness/harness-docker-runner/ti/report"
)

func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, ticlient ticlient.Client) (
func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, tiConfig api.TIConfig) (
*runtime.State, map[string]string, error) {
step := toStep(r)
step.Command = r.Run.Command
Expand All @@ -43,7 +43,7 @@ func executeRunStep(ctx context.Context, engine *engine.Engine, r *api.StartStep

exited, err := engine.Run(ctx, step, out)
logrus.WithField("step_id", r.ID).WithField("stage_id", r.StageRuntimeID).Traceln("completed step run")
if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, ticlient); rerr != nil {
if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, time.Now(), tiConfig); rerr != nil {
logrus.WithError(rerr).WithField("step", step.Name).Errorln("failed to upload report")
}

Expand Down
9 changes: 4 additions & 5 deletions pipeline/runtime/runtest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,15 @@ import (
"github.com/harness/harness-docker-runner/engine"
"github.com/harness/harness-docker-runner/pipeline"
"github.com/harness/harness-docker-runner/ti/callgraph"
ticlient "github.com/harness/harness-docker-runner/ti/client"
"github.com/harness/harness-docker-runner/ti/instrumentation"
"github.com/harness/harness-docker-runner/ti/report"
"github.com/sirupsen/logrus"
)

func executeRunTestStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, ticlient ticlient.Client) (
func executeRunTestStep(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, tiConfig api.TIConfig) (
*runtime.State, map[string]string, error) {
start := time.Now()
cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, out)
cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, out, tiConfig)
if err != nil {
return nil, nil, err
}
Expand All @@ -46,11 +45,11 @@ func executeRunTestStep(ctx context.Context, engine *engine.Engine, r *api.Start
log.Out = out

exited, err := engine.Run(ctx, step, out)
if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, ticlient); rerr != nil {
if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, time.Now(), tiConfig); rerr != nil {
log.WithError(rerr).Errorln("failed to upload report")
}

if uerr := callgraph.Upload(ctx, step.Name, time.Since(start).Milliseconds(), out, ticlient); uerr != nil {
if uerr := callgraph.Upload(ctx, step.Name, time.Since(start).Milliseconds(), out, time.Now(), tiConfig); uerr != nil {
log.WithError(uerr).Errorln("unable to collect callgraph")
}

Expand Down
26 changes: 12 additions & 14 deletions pipeline/runtime/step_executor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,12 @@ import (
"sync"
"time"

"github.com/drone/runner-go/pipeline/runtime"
"github.com/harness/harness-docker-runner/api"
"github.com/harness/harness-docker-runner/engine"
"github.com/harness/harness-docker-runner/errors"
"github.com/harness/harness-docker-runner/livelog"
"github.com/harness/harness-docker-runner/logstream"
ticlient "github.com/harness/harness-docker-runner/ti/client"

"github.com/drone/runner-go/pipeline/runtime"

"github.com/hashicorp/go-multierror"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -57,7 +55,7 @@ func NewStepExecutor(engine *engine.Engine) *StepExecutor {
}
}

func (e *StepExecutor) StartStep(ctx context.Context, r *api.StartStepRequest, secrets []string, client logstream.Client, ticlient ticlient.Client) error {
func (e *StepExecutor) StartStep(ctx context.Context, r *api.StartStepRequest, secrets []string, client logstream.Client, tiConfig api.TIConfig) error {
if r.ID == "" {
return &errors.BadRequestError{Msg: "ID needs to be set"}
}
Expand All @@ -73,7 +71,7 @@ func (e *StepExecutor) StartStep(ctx context.Context, r *api.StartStepRequest, s
e.mu.Unlock()

go func() {
state, outputs, stepErr := e.executeStep(r, secrets, client, ticlient)
state, outputs, stepErr := e.executeStep(r, secrets, client, tiConfig)
status := StepStatus{Status: Complete, State: state, StepErr: stepErr, Outputs: outputs}
e.mu.Lock()
e.stepStatus[r.ID] = status
Expand Down Expand Up @@ -174,7 +172,7 @@ func (e *StepExecutor) StreamOutput(ctx context.Context, r *api.StreamOutputRequ
return //nolint:nakedret
}

func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest, ticlient ticlient.Client) (*runtime.State, error) {
func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest, tiConfig api.TIConfig) (*runtime.State, error) {
ctx := context.Background()
var cancel context.CancelFunc
if r.Timeout > 0 {
Expand All @@ -198,7 +196,7 @@ func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest, ticlient ticlie

r.Kind = api.Run // only this kind is supported

exited, _, err := e.run(ctx, e.engine, r, stepLog, ticlient)
exited, _, err := e.run(ctx, e.engine, r, stepLog, tiConfig)
if ctx.Err() == context.Canceled || ctx.Err() == context.DeadlineExceeded {
logr.WithError(err).Warnln("step execution canceled")
return nil, ctx.Err()
Expand Down Expand Up @@ -229,9 +227,9 @@ func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest, ticlient ticlie
return runStep()
}

func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, client logstream.Client, ticlient ticlient.Client) (*runtime.State, map[string]string, error) {
func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, client logstream.Client, tiConfig api.TIConfig) (*runtime.State, map[string]string, error) {
if r.LogDrone {
state, err := e.executeStepDrone(r, ticlient)
state, err := e.executeStepDrone(r, tiConfig)
return state, nil, err
}

Expand All @@ -252,7 +250,7 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(r.Timeout))
defer cancel()
}
e.run(ctx, e.engine, r, wr, ticlient) // nolint:errcheck
e.run(ctx, e.engine, r, wr, tiConfig) // nolint:errcheck
wc.Close()
}()
return &runtime.State{Exited: false}, nil, nil
Expand All @@ -267,7 +265,7 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
defer cancel()
}

exited, outputs, err := e.run(ctx, e.engine, r, wr, ticlient)
exited, outputs, err := e.run(ctx, e.engine, r, wr, tiConfig)
if err != nil {
result = multierror.Append(result, err)
}
Expand Down Expand Up @@ -301,12 +299,12 @@ func (e *StepExecutor) executeStep(r *api.StartStepRequest, secrets []string, cl
return exited, outputs, result
}

func (e *StepExecutor) run(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, ticlient ticlient.Client) (
func (e *StepExecutor) run(ctx context.Context, engine *engine.Engine, r *api.StartStepRequest, out io.Writer, tiConfig api.TIConfig) (
*runtime.State, map[string]string, error) {
if r.Kind == api.Run {
return executeRunStep(ctx, engine, r, out, ticlient)
return executeRunStep(ctx, engine, r, out, tiConfig)
}
return executeRunTestStep(ctx, engine, r, out, ticlient)
return executeRunTestStep(ctx, engine, r, out, tiConfig)
}

func convertStatus(status StepStatus) *api.PollStepResponse {
Expand Down
19 changes: 11 additions & 8 deletions ti/callgraph/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ import (
"fmt"
"io"
"path/filepath"
"time"

"github.com/harness/harness-docker-runner/api"
"github.com/harness/harness-docker-runner/internal/filesystem"
"github.com/harness/harness-docker-runner/pipeline"
"github.com/harness/harness-docker-runner/ti/avro"
"github.com/harness/harness-docker-runner/ti/client"
"github.com/mattn/go-zglob"
Expand All @@ -26,14 +26,11 @@ const (
)

// Upload method uploads the callgraph.
func Upload(ctx context.Context, stepID string, timeMs int64, out io.Writer, ticlient client.Client) error {
func Upload(ctx context.Context, stepID string, timeMs int64, out io.Writer, start time.Time, cfg api.TIConfig) error {
log := logrus.New()
log.Out = out

// TODO: Pass in the config here and use that, right now it will be empty
// cfg := pipeline.GetState().GetTIConfig()
cfg := &api.TIConfig{}
if cfg == nil || cfg.URL == "" {
if cfg.URL == "" {
return fmt.Errorf("TI config is not provided in setup")
}

Expand All @@ -57,12 +54,18 @@ func Upload(ctx context.Context, stepID string, timeMs int64, out io.Writer, tic
}
}

encCg, err := encodeCg(fmt.Sprintf(cgDir, pipeline.SharedVolPath), log)
encCg, err := encodeCg(fmt.Sprintf(cgDir, cfg.TmpDir), log)
if err != nil {
return errors.Wrap(err, "failed to get avro encoded callgraph")
}

return ticlient.UploadCg(ctx, stepID, source, target, timeMs, encCg)
c := client.NewHTTPClient(cfg.URL, cfg.Token, cfg.AccountID, cfg.OrgID, cfg.ProjectID,
cfg.PipelineID, cfg.BuildID, cfg.StageID, cfg.Repo, cfg.Sha, false)
if cgErr := c.UploadCg(ctx, stepID, source, target, timeMs, encCg); cgErr != nil {
return cgErr
}
log.Infoln(fmt.Sprintf("Successfully uploaded callgraph in %s time", time.Since(start)))
return nil
}

// encodeCg reads all files of specified format from datadir folder and returns byte array of avro encoded format
Expand Down
11 changes: 5 additions & 6 deletions ti/instrumentation/instrumentation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,21 @@ import (

"github.com/harness/harness-docker-runner/api"
"github.com/harness/harness-docker-runner/internal/filesystem"
"github.com/harness/harness-docker-runner/pipeline"
"github.com/harness/harness-docker-runner/ti"
"github.com/harness/harness-docker-runner/ti/instrumentation/csharp"
"github.com/harness/harness-docker-runner/ti/instrumentation/java"
)

func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace string, out io.Writer) (string, error) { // nolint:funlen, gocyclo
func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace string, out io.Writer, tiConfig api.TIConfig) (string, error) { // nolint:funlen, gocyclo
fs := filesystem.New()
tmpFilePath := pipeline.SharedVolPath
tmpFilePath := tiConfig.TmpDir
log := logrus.New()
log.Out = out

// Get the tests that need to be run if we are running selected tests
var selection ti.SelectTestsResp

isManual := isManualExecution()
isManual := isManualExecution(tiConfig)
files, err := getChangedFiles(ctx, workspace, log)
if err != nil {
log.WithError(err).Println("could not get changed files")
Expand All @@ -46,7 +45,7 @@ func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace st
log.Infoln("detected manual execution - for intelligence to be configured, a PR must be raised. Running all the tests.")
runOnlySelectedTests = false // run all the tests if it is a manual execution
}
selection, err = selectTests(ctx, workspace, files, runOnlySelectedTests, stepID, fs)
selection, err = selectTests(ctx, workspace, files, runOnlySelectedTests, stepID, fs, tiConfig)
if err != nil {
log.WithError(err).Errorln("there was some issue in trying to intelligently figure out tests to run. Running all the tests")
runOnlySelectedTests = false // run all the tests if an error was encountered
Expand Down Expand Up @@ -92,7 +91,7 @@ func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace st
}

// Install agent artifacts if not present
artifactDir, err := installAgents(ctx, tmpFilePath, config.Language, runtime.GOOS, runtime.GOARCH, config.BuildTool, fs, log)
artifactDir, err := installAgents(ctx, tmpFilePath, config.Language, runtime.GOOS, runtime.GOARCH, config.BuildTool, fs, log, tiConfig)
if err != nil {
return "", err
}
Expand Down
Loading

0 comments on commit 8f0193c

Please sign in to comment.