Skip to content
This repository has been archived by the owner on Jun 13, 2023. It is now read-only.

Use a dedicated timeout for each package #64

Merged
merged 2 commits into from
Jun 21, 2018
Merged

Use a dedicated timeout for each package #64

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c60d5c7
timeout: use a dedicated timeout for each package
JulienBalestra Jun 21, 2018
72caee6
cmd: lowercase, rephrase timeouts
JulienBalestra Jun 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions cmd/cli/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func NewCommand() (*cobra.Command, *int) {
exitCode = 1
return
}
r, err := run.NewRunner(env)
r, err := run.NewRunner(env, config.ViperConfig.GetDuration("run-timeout"), config.ViperConfig.GetDuration("gc"))
if err != nil {
exitCode = 2
return
Expand Down Expand Up @@ -210,7 +210,7 @@ func NewCommand() (*cobra.Command, *int) {
),
Run: func(cmd *cobra.Command, args []string) {
for i := 0; i < len(args); i++ {
err := api.ResetNamespace(config.ViperConfig.GetString("api-address"), args[i])
err := api.ResetNamespace(config.ViperConfig.GetDuration("client-timeout"), config.ViperConfig.GetString("api-address"), args[i])
if err != nil {
exitCode = 2
return
Expand All @@ -219,7 +219,7 @@ func NewCommand() (*cobra.Command, *int) {
if !config.ViperConfig.GetBool("apply") {
return
}
err := api.Apply(config.ViperConfig.GetString("api-address"))
err := api.Apply(config.ViperConfig.GetDuration("client-timeout"), config.ViperConfig.GetString("api-address"))
if err != nil {
exitCode = 2
return
Expand Down Expand Up @@ -250,7 +250,7 @@ func NewCommand() (*cobra.Command, *int) {
exitCode = 1
return
}
err := wait.NewWaiter(unitToWatch, config.ViperConfig.GetDuration("timeout"), config.ViperConfig.GetDuration("logging-since")).Wait()
err := wait.NewWaiter(unitToWatch, config.ViperConfig.GetDuration("wait-timeout"), config.ViperConfig.GetDuration("logging-since")).Wait()
if err != nil {
exitCode = 2
return
Expand Down Expand Up @@ -300,8 +300,8 @@ func NewCommand() (*cobra.Command, *int) {
runCommand.PersistentFlags().StringP("drain", "d", config.ViperConfig.GetString("drain"), fmt.Sprintf("drain options after %s: %s", runCommand.Name(), options.GetOptionNames(options.Drain{})))
config.ViperConfig.BindPFlag("drain", runCommand.PersistentFlags().Lookup("drain"))

runCommand.PersistentFlags().Duration("timeout", config.ViperConfig.GetDuration("timeout"), fmt.Sprintf("timeout for %s", runCommand.Name()))
config.ViperConfig.BindPFlag("timeout", runCommand.PersistentFlags().Lookup("timeout"))
runCommand.PersistentFlags().Duration("run-timeout", config.ViperConfig.GetDuration("run-timeout"), fmt.Sprintf("timeout for %s", runCommand.Name()))
config.ViperConfig.BindPFlag("run-timeout", runCommand.PersistentFlags().Lookup("run-timeout"))

runCommand.PersistentFlags().Duration("gc", config.ViperConfig.GetDuration("gc"), fmt.Sprintf("grace period for the kubelet GC trigger when draining %s, no-op if not draining", runCommand.Name()))
config.ViperConfig.BindPFlag("gc", runCommand.PersistentFlags().Lookup("gc"))
Expand All @@ -323,11 +323,14 @@ func NewCommand() (*cobra.Command, *int) {
resetCommand.PersistentFlags().BoolP("apply", "a", config.ViperConfig.GetBool("apply"), "Apply manifests-api after reset, useful when resetting kube-system namespace")
config.ViperConfig.BindPFlag("apply", resetCommand.PersistentFlags().Lookup("apply"))

resetCommand.PersistentFlags().Duration("client-timeout", config.ViperConfig.GetDuration("client-timeout"), fmt.Sprintf("timeout for %s", resetCommand.Name()))
config.ViperConfig.BindPFlag("client-timeout", resetCommand.PersistentFlags().Lookup("client-timeout"))

// Wait
rootCommand.AddCommand(waitCommand)

waitCommand.PersistentFlags().Duration("timeout", time.Minute*15, fmt.Sprintf("Timeout for %s", waitCommand.Name()))
config.ViperConfig.BindPFlag("timeout", waitCommand.PersistentFlags().Lookup("timeout"))
waitCommand.PersistentFlags().Duration("wait-timeout", time.Minute*15, fmt.Sprintf("Timeout for %s", waitCommand.Name()))
config.ViperConfig.BindPFlag("wait-timeout", waitCommand.PersistentFlags().Lookup("wait-timeout"))

waitCommand.PersistentFlags().Duration("logging-since", config.ViperConfig.GetDuration("logging-since"), "Display the logs of the unit since")
config.ViperConfig.BindPFlag("logging-since", waitCommand.PersistentFlags().Lookup("logging-since"))
Expand Down
2 changes: 1 addition & 1 deletion docs/pupernetes_daemon_run.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ pupernetes daemon run state/ --job-type systemd
--gc duration grace period for the kubelet GC trigger when draining run, no-op if not draining (default 1m0s)
-h, --help help for run
--job-type string type of job: fg or systemd (default "fg")
--run-timeout duration timeout for run (default 7h0m0s)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--run-timeout duration Maximum time to run pupernetes for until self shutdown

--systemd-job-name string unit name used when running as systemd service (default "pupernetes")
--timeout duration timeout for run (default 6h0m0s)
```

### Options inherited from parent commands
Expand Down
7 changes: 4 additions & 3 deletions docs/pupernetes_reset.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@ pupernetes reset default $(kubectl get ns -o name) --apply
### Options

```
--api-address string Address for the pupernetes API ip:port (default "127.0.0.1:8989")
-a, --apply Apply manifests-api after reset, useful when resetting kube-system namespace
-h, --help help for reset
--api-address string Address for the pupernetes API ip:port (default "127.0.0.1:8989")
-a, --apply Apply manifests-api after reset, useful when resetting kube-system namespace
--client-timeout duration timeout for reset (default 1m0s)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--client-timeout duration Maximum time waited for a pupernetes command to be executed

-h, --help help for reset
```

### Options inherited from parent commands
Expand Down
2 changes: 1 addition & 1 deletion docs/pupernetes_wait.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ pupernetes wait -u p8s-kubelet
```
-h, --help help for wait
--logging-since duration Display the logs of the unit since (default 5m0s)
--timeout duration Timeout for wait (default 15m0s)
-u, --unit-to-watch string Systemd unit name to watch (default "pupernetes.service")
--wait-timeout duration Timeout for wait (default 15m0s)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--wait-timeout duration Maximum time to download the required images and set up pupernetes

```

### Options inherited from parent commands
Expand Down
12 changes: 6 additions & 6 deletions pkg/api/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const (

// ResetNamespace executes an API call to the pupernetes API to reset
// the namespace in parameter. The namespace can be like ns/default or just default
func ResetNamespace(apiAddress, namespace string) error {
func ResetNamespace(timeout time.Duration, apiAddress, namespace string) error {
if strings.HasPrefix(namespace, namespacePrefix) {
glog.V(4).Infof("Stripping namespace %q", namespace)
namespace = namespace[len(namespacePrefix):]
Expand All @@ -27,19 +27,19 @@ func ResetNamespace(apiAddress, namespace string) error {
return err
}
glog.Infof("Resetting namespace %q ...", namespace)
return doPOST(apiAddress, fmt.Sprintf("%s/%s", resetRoute, namespace))
return doPOST(timeout, apiAddress, fmt.Sprintf("%s/%s", resetRoute, namespace))
}

// Apply executes an API call to the pupernetes API to force an apply of the "manifest-api" directory
func Apply(apiAddress string) error {
func Apply(timeout time.Duration, apiAddress string) error {
glog.Infof("Applying ...")
return doPOST(apiAddress, applyRoute)
return doPOST(timeout, apiAddress, applyRoute)
}

func doPOST(apiAddress, apiRoute string) error {
func doPOST(timeout time.Duration, apiAddress, apiRoute string) error {
glog.Infof("Calling POST %s ...", apiRoute)
c := &http.Client{}
c.Timeout = time.Second * 5
c.Timeout = timeout

u, err := url.Parse(fmt.Sprintf("http://%s%s", apiAddress, apiRoute))
if err != nil {
Expand Down
4 changes: 3 additions & 1 deletion pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func init() {

ViperConfig.SetDefault("clean", "etcd,kubelet,mounts,iptables")
ViperConfig.SetDefault("drain", "all")
ViperConfig.SetDefault("timeout", time.Hour*6)
ViperConfig.SetDefault("run-timeout", time.Hour*7)
ViperConfig.SetDefault("gc", time.Second*60)

// The supported job-type are "fg" and "systemd"
Expand All @@ -56,4 +56,6 @@ func init() {

ViperConfig.SetDefault("logging-since", time.Minute*5)
ViperConfig.SetDefault("unit-to-watch", "pupernetes.service")
ViperConfig.SetDefault("wait-timeout", time.Minute*15)
ViperConfig.SetDefault("client-timeout", time.Minute*1)
}
17 changes: 8 additions & 9 deletions pkg/run/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,22 @@ package run

import (
"fmt"
"io/ioutil"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"

"github.com/golang/glog"
"k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/DataDog/pupernetes/pkg/api"
"github.com/DataDog/pupernetes/pkg/config"
"github.com/DataDog/pupernetes/pkg/logging"
"github.com/DataDog/pupernetes/pkg/run/state"
"github.com/DataDog/pupernetes/pkg/setup"
"github.com/DataDog/pupernetes/pkg/util"
"io/ioutil"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"sync"
)

const (
Expand Down Expand Up @@ -51,7 +50,7 @@ type Runtime struct {
}

// NewRunner instantiate a new Runtimer with the given Environment
func NewRunner(env *setup.Environment) (*Runtime, error) {
func NewRunner(env *setup.Environment, runTimeout, waitKubeletGC time.Duration) (*Runtime, error) {
var zero int64

s, err := state.NewState()
Expand All @@ -67,8 +66,8 @@ func NewRunner(env *setup.Environment) (*Runtime, error) {
httpClient: &http.Client{
Timeout: time.Millisecond * 500,
},
runTimeout: config.ViperConfig.GetDuration("timeout"),
waitKubeletGC: config.ViperConfig.GetDuration("gc"),
runTimeout: runTimeout,
waitKubeletGC: waitKubeletGC,
kubeDeleteOption: &v1.DeleteOptions{
GracePeriodSeconds: &zero,
},
Expand Down Expand Up @@ -138,8 +137,8 @@ func (r *Runtime) Run() error {
if failures >= appProbeThreshold {
glog.Warningf("Probing failed, stopping ...")
// display some helpers to investigate:
glog.Infof("Investigate the kubelet logs with: journalctl -u %skubelet.service -o cat -e --no-pager", config.ViperConfig.GetString("systemd-unit-prefix"))
glog.Infof("Investigate the kubelet status with: systemctl status %skubelet.service -l --no-pager", config.ViperConfig.GetString("systemd-unit-prefix"))
glog.Infof("Investigate the kubelet logs with: journalctl -u %skubelet.service -o cat -e --no-pager", r.env.GetSystemdUnitPrefix())
glog.Infof("Investigate the kubelet status with: systemctl status %skubelet.service -l --no-pager", r.env.GetSystemdUnitPrefix())
// Propagate a stop
return r.Stop(fmt.Errorf("failure threshold reached %d/%d", failures, appProbeThreshold))
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/setup/getter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,3 +122,8 @@ func (e *Environment) GetPublicIP() string {
func (e *Environment) GetSystemdUnits() []string {
return e.systemdUnitNames
}

// GetSystemdUnitPrefix returns the prefix used with systemd units
func (e *Environment) GetSystemdUnitPrefix() string {
return e.systemdUnitPrefix
}