From 424a8cbd171fa44a2dec71779f3a099f77f65809 Mon Sep 17 00:00:00 2001 From: Jussi Maki Date: Fri, 25 Oct 2024 11:48:23 +0200 Subject: [PATCH] script: Add '*' prefix for retrying commands This adds a generic facility for retrying failing commands. A command that is prefixed with '*' and fails will cause the whole section (delimited by '#' comments) to be retried from the top. Retrying is repeated until the command succeeds or the context is cancelled. The retry interval can be set in (*Engine).RetryInterval. Example usage: # Verify table contents db show -o=out.table my-table * cmp out.table expected.table Signed-off-by: Jussi Maki --- script/engine.go | 65 ++++++++++++++++++++-------- script/scripttest/scripttest_test.go | 8 ++-- script/scripttest/testdata/basic.txt | 6 +++ script/state.go | 1 - 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/script/engine.go b/script/engine.go index 7b9b9bf..e45124c 100644 --- a/script/engine.go +++ b/script/engine.go @@ -73,13 +73,18 @@ type Engine struct { // If Quiet is true, Execute deletes log prints from the previous // section when starting a new section. Quiet bool + + // RetryInterval for retrying commands marked with '*'. If zero, then + // retries are disabled. + RetryInterval time.Duration } // NewEngine returns an Engine configured with a basic set of commands and conditions. func NewEngine() *Engine { return &Engine{ - Cmds: DefaultCmds(), - Conds: DefaultConds(), + Cmds: DefaultCmds(), + Conds: DefaultConds(), + RetryInterval: 100 * time.Millisecond, } } @@ -167,7 +172,10 @@ func (e *Engine) Execute(s *State, file string, script *bufio.Reader, log io.Wri defer func(prev io.Writer) { s.logOut = prev }(s.logOut) s.logOut = log - var sectionStart time.Time + var ( + sectionStart time.Time + sectionCmds []*command + ) // endSection flushes the logs for the current section from s.log to log. // ok indicates whether all commands in the section succeeded. endSection := func(ok bool) error { @@ -193,6 +201,7 @@ func (e *Engine) Execute(s *State, file string, script *bufio.Reader, log io.Wri } sectionStart = time.Time{} + sectionCmds = nil return err } @@ -257,6 +266,8 @@ func (e *Engine) Execute(s *State, file string, script *bufio.Reader, log io.Wri if cmd == nil && err == nil { continue // Ignore blank lines. } + sectionCmds = append(sectionCmds, cmd) + s.Logf("> %s\n", line) if err != nil { return lineErr(err) @@ -296,16 +307,34 @@ func (e *Engine) Execute(s *State, file string, script *bufio.Reader, log io.Wri // Run the command. err = e.runCommand(s, cmd, impl) if err != nil { - if stop := (stopError{}); errors.As(err, &stop) { - // Since the 'stop' command halts execution of the entire script, - // log its message separately from the section in which it appears. - err = endSection(true) - s.Logf("%v\n", stop) - if err == nil { - return nil + if cmd.want == successRetryOnFailure && e.RetryInterval > 0 { + // Command wants retries. Retry the whole section + for err != nil { + select { + case <-s.Context().Done(): + err = lineErr(s.Context().Err()) + break + case <-time.After(e.RetryInterval): + } + for _, cmd := range sectionCmds { + impl := e.Cmds[cmd.name] + if err = e.runCommand(s, cmd, impl); err != nil { + break + } + } } + } else { + if stop := (stopError{}); errors.As(err, &stop) { + // Since the 'stop' command halts execution of the entire script, + // log its message separately from the section in which it appears. + err = endSection(true) + s.Logf("%v\n", stop) + if err == nil { + return nil + } + } + return lineErr(err) } - return lineErr(err) } } @@ -394,9 +423,10 @@ type command struct { type expectedStatus string const ( - success expectedStatus = "" - failure expectedStatus = "!" - successOrFailure expectedStatus = "?" + success expectedStatus = "" + failure expectedStatus = "!" + successOrFailure expectedStatus = "?" + successRetryOnFailure expectedStatus = "*" ) type argFragment struct { @@ -437,10 +467,11 @@ func parse(filename string, lineno int, line string) (cmd *command, err error) { // Command prefix ! means negate the expectations about this command: // go command should fail, match should not be found, etc. // Prefix ? means allow either success or failure. + // Prefix * means to retry the command a few times. switch want := expectedStatus(arg); want { - case failure, successOrFailure: + case failure, successOrFailure, successRetryOnFailure: if cmd.want != "" { - return errors.New("duplicated '!' or '?' token") + return errors.New("duplicated '!', '?' or '*' token") } cmd.want = want return nil @@ -695,7 +726,7 @@ func checkStatus(cmd *command, err error) error { return cmdError(cmd, err) } - if cmd.want == success { + if cmd.want == success || cmd.want == successRetryOnFailure { return cmdError(cmd, err) } diff --git a/script/scripttest/scripttest_test.go b/script/scripttest/scripttest_test.go index 3178e8d..5cab3db 100644 --- a/script/scripttest/scripttest_test.go +++ b/script/scripttest/scripttest_test.go @@ -8,6 +8,7 @@ import ( "context" "os" "testing" + "time" "github.com/cilium/hive/script" "github.com/cilium/hive/script/scripttest" @@ -16,9 +17,10 @@ import ( func TestAll(t *testing.T) { ctx := context.Background() engine := &script.Engine{ - Conds: scripttest.DefaultConds(), - Cmds: scripttest.DefaultCmds(), - Quiet: !testing.Verbose(), + Conds: scripttest.DefaultConds(), + Cmds: scripttest.DefaultCmds(), + Quiet: !testing.Verbose(), + RetryInterval: 10 * time.Millisecond, } env := os.Environ() scripttest.Test(t, ctx, func(t testing.TB) *script.Engine { return engine }, env, "testdata/*.txt") diff --git a/script/scripttest/testdata/basic.txt b/script/scripttest/testdata/basic.txt index de5fc2b..6f05cdb 100644 --- a/script/scripttest/testdata/basic.txt +++ b/script/scripttest/testdata/basic.txt @@ -2,5 +2,11 @@ cat hello.txt stdout 'hello world' ! stderr 'hello world' +exec sh -c 'sleep 0.1 && echo > out.txt' & + +# Retry section test +echo hello +* exec cat out.txt + -- hello.txt -- hello world diff --git a/script/state.go b/script/state.go index 1226dcf..f6b2745 100644 --- a/script/state.go +++ b/script/state.go @@ -26,7 +26,6 @@ type State struct { ctx context.Context cancel context.CancelFunc - file string log bytes.Buffer logOut io.Writer