From 25cdf5e924389ecadf9360eb71095cfb38ef2f8f Mon Sep 17 00:00:00 2001 From: Ori Seri <6756779+oriser@users.noreply.github.com> Date: Sun, 11 Dec 2022 18:37:37 +0200 Subject: [PATCH] Retry wolt http (#7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add retry * tests and configuration for tetry * 🐶 --- go.mod | 8 ++++ go.sum | 11 ++++++ service/debt.go | 4 +- service/rate.go | 12 ++++-- service/service.go | 49 ++++++++++-------------- testing/integration_test.go | 30 +++++++++------ testing/woltserver/handlers.go | 3 -- testing/woltserver/woltserver.go | 20 +++++++++- wolt/group.go | 64 ++++++++++++++++++++------------ 9 files changed, 127 insertions(+), 74 deletions(-) diff --git a/go.mod b/go.mod index 99a871d..863295b 100644 --- a/go.mod +++ b/go.mod @@ -11,10 +11,12 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 + github.com/hashicorp/go-retryablehttp v0.7.1 github.com/jmoiron/sqlx v1.3.4 github.com/mattn/go-sqlite3 v1.14.10 github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3 github.com/paul-mannino/go-fuzzywuzzy v0.0.0-20200127021948-54652b135d0e + github.com/prometheus/common v0.10.0 github.com/slack-go/slack v0.10.3 github.com/stretchr/testify v1.7.0 golang.org/x/net v0.0.0-20211013171255-e13a2654a71e @@ -23,9 +25,12 @@ require ( require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-multierror v1.1.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -34,7 +39,10 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index e65c16f..d291c98 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -527,10 +529,15 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMW github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -789,6 +796,7 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -829,6 +837,7 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/slack-go/slack v0.10.3 h1:kKYwlKY73AfSrtAk9UHWCXXfitudkDztNI9GYBviLxw= github.com/slack-go/slack v0.10.3/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= @@ -1173,6 +1182,7 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c h1:taxlMj0D/1sOAuv/CbSD+MMDof2vbyPTqz5FNYKpXt8= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1407,6 +1417,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/service/debt.go b/service/debt.go index 648e1e6..134e2a6 100644 --- a/service/debt.go +++ b/service/debt.go @@ -74,7 +74,7 @@ func (h *Service) DebtWorker(ctx context.Context, orderID string) { return } - reminderInterval := time.NewTicker(h.debtReminderInterval) + reminderInterval := time.NewTicker(h.cfg.DebtReminderInterval) defer reminderInterval.Stop() for { @@ -178,7 +178,7 @@ func (h *Service) addDebts(usersMap map[string]*userDomain.User, initiatedTransp } //goland:noinspection ALL - ctx, _ := context.WithTimeout(context.Background(), h.debtMaximumDuration) // nolint + ctx, _ := context.WithTimeout(context.Background(), h.cfg.DebtMaximumDuration) // nolint go h.DebtWorker(ctx, orderID) return nil diff --git a/service/rate.go b/service/rate.go index 995e6f6..22cf44c 100644 --- a/service/rate.go +++ b/service/rate.go @@ -218,7 +218,7 @@ func (h *Service) shouldHandleOrder() bool { } func (h *Service) waitForGroupProgress(g *wolt.Group) error { - timeoutTime := time.Now().Add(h.timeoutForReady) + timeoutTime := time.Now().Add(h.cfg.TimeoutForReady) details, err := g.Details() if err != nil { @@ -233,7 +233,7 @@ func (h *Service) waitForGroupProgress(g *wolt.Group) error { if time.Now().After(timeoutTime) { return fmt.Errorf("timeout waiting for group to progress") } - time.Sleep(h.waitBetweenStatusCheck) + time.Sleep(h.cfg.WaitBetweenStatusCheck) details, err = g.Details() if err != nil { @@ -277,8 +277,12 @@ func (h *Service) calculateDeliveryRate(g *wolt.Group, details *wolt.OrderDetail func (h *Service) getRateForGroup(receiver, groupID, messageID string) (GroupRate, error) { g, err := wolt.NewGroupWithExistingID(wolt.WoltAddr{ - BaseAddr: h.woltBaseAddr, - APIBaseAddr: h.woltApiAddr, + BaseAddr: h.cfg.WoltBaseAddr, + APIBaseAddr: h.cfg.WoltApiBaseAddr, + }, wolt.RetryConfig{ + HTTPMaxRetries: h.cfg.WoltHTTPMaxRetryCount, + HTTPMinRetryDuration: h.cfg.WoltHTTPMinRetryDuration, + HTTPMaxRetryDuration: h.cfg.WoltHTTPMaxRetryDuration, }, groupID) if err != nil { return GroupRate{}, fmt.Errorf("new existing group: %w", err) diff --git a/service/service.go b/service/service.go index 587b2c0..9c85d95 100644 --- a/service/service.go +++ b/service/service.go @@ -15,32 +15,29 @@ type EventNotification interface { } type Config struct { - TimeoutForReady time.Duration `env:"ORDER_READY_TIMEOUT" envDefault:"40m"` - TimeoutForDeliveryRate time.Duration `env:"GET_DELIVERY_RATE_TIMEOUT" envDefault:"10m"` - WaitBetweenStatusCheck time.Duration `env:"WAIT_BETWEEN_STATUS_CHECK" envDefault:"20s"` - DebtReminderInterval time.Duration `env:"DEBT_REMINDER_INTERVAL" envDefault:"3h"` - DebtMaximumDuration time.Duration `env:"DEBT_MAXIMUM_DURATION" envDefault:"24h"` - DontJoinAfter string `env:"DONT_JOIN_AFTER"` - DontJoinAfterTZ string `env:"DONT_JOIN_AFTER_TZ"` - WoltBaseAddr string `env:"WOLT_BASE_ADDR" envDefault:"https://wolt.com"` - WoltApiBaseAddr string `env:"WOLT_API_BASE_ADDR" envDefault:"https://restaurant-api.wolt.com"` + TimeoutForReady time.Duration `env:"ORDER_READY_TIMEOUT" envDefault:"40m"` + TimeoutForDeliveryRate time.Duration `env:"GET_DELIVERY_RATE_TIMEOUT" envDefault:"10m"` + WaitBetweenStatusCheck time.Duration `env:"WAIT_BETWEEN_STATUS_CHECK" envDefault:"20s"` + DebtReminderInterval time.Duration `env:"DEBT_REMINDER_INTERVAL" envDefault:"3h"` + DebtMaximumDuration time.Duration `env:"DEBT_MAXIMUM_DURATION" envDefault:"24h"` + DontJoinAfter string `env:"DONT_JOIN_AFTER"` + DontJoinAfterTZ string `env:"DONT_JOIN_AFTER_TZ"` + WoltBaseAddr string `env:"WOLT_BASE_ADDR" envDefault:"https://wolt.com"` + WoltApiBaseAddr string `env:"WOLT_API_BASE_ADDR" envDefault:"https://restaurant-api.wolt.com"` + WoltHTTPMaxRetryCount int `env:"WOLT_HTTP_MAX_RETRY_COUNT" envDefault:"5"` + WoltHTTPMinRetryDuration time.Duration `env:"WOLT_HTTP_MIN_RETRY_DURATION" envDefault:"1s"` + WoltHTTPMaxRetryDuration time.Duration `env:"WOLT_HTTP_MAX_RETRY_DURATION" envDefault:"30s"` } type Service struct { - timeoutForReady time.Duration - timeoutDeliveryRate time.Duration - waitBetweenStatusCheck time.Duration + cfg Config eventNotification EventNotification currentlyWorkingOrders sync.Map userStore user.Store debtStore debt.Store - debtReminderInterval time.Duration - debtMaximumDuration time.Duration selfID string dontJoinAfter time.Time dontJoinAfterTZ *time.Location - woltBaseAddr string - woltApiAddr string } type ReactionAddRequest struct { @@ -79,18 +76,12 @@ func New(cfg Config, userStore user.Store, debtStore debt.Store, selfID string, } } return &Service{ - timeoutForReady: cfg.TimeoutForReady, - timeoutDeliveryRate: cfg.TimeoutForDeliveryRate, - waitBetweenStatusCheck: cfg.WaitBetweenStatusCheck, - eventNotification: eventNotification, - userStore: userStore, - debtStore: debtStore, - debtReminderInterval: cfg.DebtReminderInterval, - debtMaximumDuration: cfg.DebtMaximumDuration, - selfID: selfID, - dontJoinAfter: dontJoinAfter, - dontJoinAfterTZ: dontJoinAfterTZ, - woltBaseAddr: cfg.WoltBaseAddr, - woltApiAddr: cfg.WoltApiBaseAddr, + cfg: cfg, + eventNotification: eventNotification, + userStore: userStore, + debtStore: debtStore, + selfID: selfID, + dontJoinAfter: dontJoinAfter, + dontJoinAfterTZ: dontJoinAfterTZ, }, nil } diff --git a/testing/integration_test.go b/testing/integration_test.go index 5cd6ada..0c5a8df 100644 --- a/testing/integration_test.go +++ b/testing/integration_test.go @@ -11,6 +11,7 @@ import ( "os" "path" "sort" + "strconv" "strings" "testing" "time" @@ -39,11 +40,15 @@ var ( ) const ( + WaitForMessageTimeout = 20 * time.Second OrderReadyTimeout = 10 * time.Second WaitBetweenStatusCheck = 500 * time.Millisecond DebtReminderInterval = 3 * time.Second DebtMaximumDuration = 10 * time.Second AdminSlackUserID = "ABC123" + MaxHttpAttempts = 10000 // A lot of attempts to make sure the request will succeed at last (we return 502 randomly for tests) + MinHttpRetryWait = time.Millisecond + MaxHttpRetryWait = 5 * time.Millisecond DefaultNonBotUserID = "W012A3CDE" // From slack test package, it's not exposed, and it's constant MessageChannel = "some-channel" @@ -92,6 +97,9 @@ func initEnvs(t *testing.T, tdata testData) { require.NoError(t, os.Setenv("DEBT_MAXIMUM_DURATION", DebtMaximumDuration.String())) require.NoError(t, os.Setenv("WOLT_BASE_ADDR", "http://"+tdata.woltServer.Addr())) require.NoError(t, os.Setenv("WOLT_API_BASE_ADDR", "http://"+tdata.woltServer.Addr())) + require.NoError(t, os.Setenv("WOLT_HTTP_MAX_RETRY_COUNT", strconv.Itoa(MaxHttpAttempts))) + require.NoError(t, os.Setenv("WOLT_HTTP_MIN_RETRY_DURATION", MinHttpRetryWait.String())) + require.NoError(t, os.Setenv("WOLT_HTTP_MAX_RETRY_DURATION", MaxHttpRetryWait.String())) // main require.NoError(t, os.Setenv("DB_LOCATION", path.Join(tmpDir, "db.sqlite"))) @@ -99,7 +107,7 @@ func initEnvs(t *testing.T, tdata testData) { func initTest(t *testing.T) testData { t.Helper() - woltServer := woltserver.NewWoltServer() + woltServer := woltserver.NewWoltServer(t) t.Log("Starting test wolt server") woltServer.Start() @@ -359,7 +367,7 @@ func validateDebts(t *testing.T, willRemainDebts = append(willRemainDebts, participant) continue } - _, err := WaitForOutboundSlackMessage(1*time.Second, tdata.slackServer, + _, err := WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("Reminder, you should pay %.2f nis to <@%s> for Wolt order ID %s.\n", ratesMap[participant], participantIDsMapping[host], orderID), participantIDsMapping[participant], "", ContainsMatch) @@ -372,12 +380,12 @@ func validateDebts(t *testing.T, assert.Equal(t, 200, resp.StatusCode) // Checking messages sent to user and host - _, err = WaitForOutboundSlackMessage(1*time.Second, tdata.slackServer, + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("OK! I removed your debt for order %s", orderID), participantIDsMapping[participant], "", EqualMatch) require.NoError(t, err) - _, err = WaitForOutboundSlackMessage(1*time.Second, tdata.slackServer, + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("<@%s> marked himself as paid for order ID %s", participantIDsMapping[participant], orderID), participantIDsMapping[host], "", EqualMatch) require.NoError(t, err) @@ -391,7 +399,7 @@ func validateDebts(t *testing.T, t.Log("Waiting until debt timeout will reach") time.Sleep(DebtMaximumDuration - 2*DebtReminderInterval) if len(willRemainDebts) > 0 { - _, err := WaitForOutboundSlackMessage(1*time.Second, tdata.slackServer, + _, err := WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("I removed all debts for order ID %s because timeout has been reached", orderID), participantIDsMapping[host], "", EqualMatch) require.NoError(t, err) @@ -409,7 +417,7 @@ func cancelDebts(t *testing.T, require.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) - _, err = WaitForOutboundSlackMessage(1*time.Second, tdata.slackServer, + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("I removed all debts for order ID %s because the host requested to cancel debts tracking", orderID), hostUser, "", EqualMatch) require.NoError(t, err) @@ -633,7 +641,7 @@ func TestSlackPurchaseGroup(t *testing.T) { assert.Equal(t, 200, resp.StatusCode) // Verifying joined message - _, err = WaitForOutboundSlackMessage(2*time.Second, tdata.slackServer, fmt.Sprintf(HelloPattern, orderShortID), + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf(HelloPattern, orderShortID), MessageChannel, timestamp, EqualMatch) require.NoError(t, err) @@ -669,7 +677,7 @@ func TestSlackPurchaseGroup(t *testing.T) { // Validating the rates message rates, ratesMessage := buildRatesMessage(t, order, expectedDelivery, participantIDsMapping) - msg, err := WaitForOutboundSlackMessage(3*time.Second, tdata.slackServer, + msg, err := WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("Rates for Wolt order ID %s", orderShortID), MessageChannel, timestamp, ContainsMatch) require.NoError(t, err) @@ -685,14 +693,14 @@ func TestSlackPurchaseGroup(t *testing.T) { if !tc.addHostToSlack { // No debts mode - _, err = WaitForOutboundSlackMessage(2*time.Second, tdata.slackServer, + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("I didn't find the user of the host (%s), I won't track debts for order %s", order.Host, orderShortID), MessageChannel, timestamp, EqualMatch) assert.NoError(t, err) } else { // Debt mode // First, verifying debts message - _, err = WaitForOutboundSlackMessage(2*time.Second, tdata.slackServer, + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("<@%s>, as the host, you can react with :x: to the rates message to cancel debts tracking for Wolt order ID %s", participantIDsMapping[order.Host], orderShortID), MessageChannel, timestamp, ContainsMatch) @@ -703,7 +711,7 @@ func TestSlackPurchaseGroup(t *testing.T) { // participant should exist and no "not found" message should be sent continue } - _, err = WaitForOutboundSlackMessage(2*time.Second, tdata.slackServer, + _, err = WaitForOutboundSlackMessage(WaitForMessageTimeout, tdata.slackServer, fmt.Sprintf("I won't track %q payment because I can't find his user.", participant), MessageChannel, timestamp, EqualMatch) assert.NoError(t, err) diff --git a/testing/woltserver/handlers.go b/testing/woltserver/handlers.go index b8a6b64..103afeb 100644 --- a/testing/woltserver/handlers.go +++ b/testing/woltserver/handlers.go @@ -55,7 +55,6 @@ func (ws *WoltServer) joinByShortIDHandler(res http.ResponseWriter, req *http.Re ws.writeError(res, http.StatusInternalServerError, err) return } - res.WriteHeader(http.StatusOK) } func (ws *WoltServer) joinByIDHandler(res http.ResponseWriter, req *http.Request) { @@ -91,7 +90,6 @@ func (ws *WoltServer) orderDetailsHandler(res http.ResponseWriter, req *http.Req ws.writeError(res, http.StatusInternalServerError, err) return } - res.WriteHeader(http.StatusOK) } func (ws *WoltServer) getVenueHandler(res http.ResponseWriter, req *http.Request) { @@ -112,5 +110,4 @@ func (ws *WoltServer) getVenueHandler(res http.ResponseWriter, req *http.Request ws.writeError(res, http.StatusInternalServerError, err) return } - res.WriteHeader(http.StatusOK) } diff --git a/testing/woltserver/woltserver.go b/testing/woltserver/woltserver.go index b9b5f6f..ce886d4 100644 --- a/testing/woltserver/woltserver.go +++ b/testing/woltserver/woltserver.go @@ -2,9 +2,12 @@ package woltserver import ( "fmt" + "math/rand" "net/http" "net/http/httptest" "sync" + "testing" + "time" "github.com/gorilla/mux" ) @@ -20,9 +23,12 @@ type WoltServer struct { orders map[string]*Order // ID to order shortIDOrder map[string]string // Order short ID to ID venues map[string]*Venue + t *testing.T } -func NewWoltServer() *WoltServer { +func NewWoltServer(t *testing.T) *WoltServer { + rand.Seed(time.Now().UnixNano()) // For random 50x http errors + router := mux.NewRouter() server := httptest.NewUnstartedServer(router) @@ -32,6 +38,7 @@ func NewWoltServer() *WoltServer { orders: make(map[string]*Order), shortIDOrder: make(map[string]string), venues: make(map[string]*Venue), + t: t, } ws.registerDefaults() return ws @@ -62,7 +69,16 @@ func (ws *WoltServer) Stop() { } func (ws *WoltServer) RegisterEndpoint(pattern string, handler http.HandlerFunc) { - ws.router.HandleFunc(pattern, handler) + ws.router.HandleFunc(pattern, func(writer http.ResponseWriter, request *http.Request) { + if 0 == rand.Intn(7) { + // Randomly return some 502 errors to simulate wolt server errors + ws.t.Log("Returning 502 error") + ws.writeError(writer, http.StatusBadGateway, fmt.Errorf("random error")) + return + } + + handler.ServeHTTP(writer, request) + }) } func (ws *WoltServer) GetOrder(orderID string) (*Order, error) { diff --git a/wolt/group.go b/wolt/group.go index 9484c53..faf838d 100644 --- a/wolt/group.go +++ b/wolt/group.go @@ -10,8 +10,11 @@ import ( "net/url" "path" "strings" + "time" "github.com/Jeffail/gabs/v2" + retryablehttp "github.com/hashicorp/go-retryablehttp" + "github.com/prometheus/common/log" "golang.org/x/net/html" ) @@ -23,6 +26,12 @@ type WoltAddr struct { apiAddrParsed *url.URL } +type RetryConfig struct { + HTTPMaxRetries int + HTTPMinRetryDuration time.Duration + HTTPMaxRetryDuration time.Duration +} + func (w *WoltAddr) parse() error { u, err := url.Parse(w.BaseAddr) if err != nil { @@ -47,14 +56,22 @@ type Group struct { headers map[string]string } -func newGroup(woltAddrs WoltAddr, id string) (*Group, error) { +func newGroup(woltAddrs WoltAddr, retryConfig RetryConfig, id string) (*Group, error) { jar, err := cookiejar.New(nil) if err != nil { return nil, fmt.Errorf("cookiejar: %w", err) } - client := &http.Client{ - Jar: jar, + client := retryablehttp.NewClient() + client.HTTPClient.Jar = jar + client.RetryWaitMax = retryConfig.HTTPMaxRetryDuration + client.RetryWaitMin = retryConfig.HTTPMinRetryDuration + client.RetryMax = retryConfig.HTTPMaxRetries + client.Logger = nil + client.RequestLogHook = func(logger retryablehttp.Logger, request *http.Request, i int) { + if i != 0 { + log.Errorf("Retrying request for %s (attempt %d)", request.URL.String(), i) + } } if err = woltAddrs.parse(); err != nil { @@ -64,7 +81,7 @@ func newGroup(woltAddrs WoltAddr, id string) (*Group, error) { return &Group{ woltAddrs: woltAddrs, prettyID: id, - client: client, + client: client.StandardClient(), headers: map[string]string{ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", "Origin": woltAddrs.BaseAddr, @@ -72,8 +89,8 @@ func newGroup(woltAddrs WoltAddr, id string) (*Group, error) { }}, nil } -func NewGroupWithExistingID(woltAddrs WoltAddr, id string) (*Group, error) { - return newGroup(woltAddrs, id) +func NewGroupWithExistingID(woltAddrs WoltAddr, retryConfig RetryConfig, id string) (*Group, error) { + return newGroup(woltAddrs, retryConfig, id) } func isIDMatch(n *html.Node, id string) bool { @@ -163,6 +180,19 @@ func (g *Group) prepareReq(method, url string, body io.Reader, extraHeaders map[ return req, nil } +func (g *Group) sendReq(req *http.Request) (*http.Response, error) { + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("sending https req: %w", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("got non 200 response: %d", resp.StatusCode) + } + + return resp, nil +} + func (g *Group) joinByRealID() error { body := bytes.NewBuffer([]byte(`{"first_name":"Wolt Bot"}`)) @@ -180,15 +210,11 @@ func (g *Group) joinByRealID() error { return fmt.Errorf("new request: %w", err) } - resp, err := g.client.Do(req) + _, err = g.sendReq(req) if err != nil { return fmt.Errorf("join request http res: %w", err) } - if resp.StatusCode != 200 { - return fmt.Errorf("got non 200 response: %d", resp.StatusCode) - } - return nil } @@ -198,7 +224,7 @@ func (g *Group) Join() error { return fmt.Errorf("new request: %w", err) } - resp, err := g.client.Do(req) + resp, err := g.sendReq(req) if err != nil { return fmt.Errorf("getting http response: %w", err) } @@ -222,7 +248,7 @@ func (g *Group) Details() (*OrderDetails, error) { return nil, fmt.Errorf("new request: %w", err) } - resp, err := g.client.Do(req) + resp, err := g.sendReq(req) if err != nil { return nil, fmt.Errorf("details http res: %w", err) } @@ -256,15 +282,11 @@ func (g *Group) VenueDetails() (*VenueDetails, error) { return nil, fmt.Errorf("prepare venue request: %w", err) } - resp, err := g.client.Do(req) + resp, err := g.sendReq(req) if err != nil { return nil, fmt.Errorf("send venue details request: %w", err) } - if resp.StatusCode != 200 { - return nil, fmt.Errorf("status code is not 200. Response: %+v", resp) - } - output, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading output: %w", err) @@ -290,14 +312,10 @@ func (g *Group) MarkAsReady() error { return fmt.Errorf("new request: %w", err) } - resp, err := g.client.Do(req) + _, err = g.sendReq(req) if err != nil { return fmt.Errorf("mark as ready http res: %w", err) } - if resp.StatusCode != 200 { - return fmt.Errorf("status is not 200. Response: %+v", resp) - } - return nil }