Skip to content

Commit

Permalink
Retry docker-pulls
Browse files Browse the repository at this point in the history
Exceptions occur here semi-frequently, for rate limits or transient
network issues it seems. This adds exponential backoff starting with a
base of 1s and retrying up to 5 times.

Example:

    Run Seconds Timestamp
    0   0.000   00:00:02
    1   1.000   00:00:03
    2   3.000   00:00:05
    3   7.000   00:00:09
    4   15.000  00:00:17
  • Loading branch information
pbrisbin committed Dec 16, 2024
1 parent 23f243c commit 0412309
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 1 deletion.
1 change: 1 addition & 0 deletions package.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ library:
- opt-env-conf >= 0.6.0.2
- path
- relude
- retry
- ronn
- ronn-opt-env-conf
- safe-coloured-text
Expand Down
1 change: 1 addition & 0 deletions restyler.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ library
, opt-env-conf >=0.6.0.2
, path
, relude
, retry
, ronn
, ronn-opt-env-conf
, safe-coloured-text
Expand Down
30 changes: 29 additions & 1 deletion src/Restyler/Monad/Docker.hs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ import Restyler.Prelude
import Data.Text qualified as T
import Restyler.AnnotatedException
import System.Process.Typed
import UnliftIO.Retry
( RetryPolicyM
, RetryStatus (..)
, exponentialBackoff
, limitRetries
, recovering
, skipAsyncExceptions
)

class Monad m => MonadDocker m where
dockerPull :: HasCallStack => String -> m ExitCode
Expand All @@ -46,11 +54,31 @@ instance
(MonadUnliftIO m, MonadLogger m, MonadReader env m, HasLogger env)
=> MonadDocker (ActualDocker m)
where
dockerPull image = runDocker ["pull", "--quiet", image]
dockerPull image =
recovering dockerPullRetryPolicy skipAsyncExceptions $ \status -> do
when (rsIterNumber status > 0) $ do
logWarn
$ "Retrying docker-pull"
:# [ "attempt" .= rsIterNumber status
, "limit" .= dockerPullRetryLimit
]

runDocker ["pull", "--quiet", image]
dockerRun args = runDocker $ ["run", "--rm"] <> args
dockerRunStdout args = runDockerStdout $ ["run", "--rm"] <> args
dockerImageRm image = runDocker_ ["image", "rm", "--force", image]

dockerPullRetryPolicy :: Monad m => RetryPolicyM m
dockerPullRetryPolicy =
exponentialBackoff dockerPullRetryBaseBackoff
<> limitRetries dockerPullRetryLimit

dockerPullRetryBaseBackoff :: Int
dockerPullRetryBaseBackoff = 1 * 1000000

dockerPullRetryLimit :: Int
dockerPullRetryLimit = 5

runDocker
:: (MonadUnliftIO m, MonadLogger m, MonadReader env m, HasLogger env, HasCallStack)
=> [String]
Expand Down

0 comments on commit 0412309

Please sign in to comment.