Use a dedicated timeout for each package (#64)

* timeout: use a dedicated timeout for each package

cmd/cli/cmd.go

@@ -138,7 +138,7 @@ func NewCommand() (*cobra.Command, *int) {
 				exitCode = 1
 				return
 			}
-			r, err := run.NewRunner(env)
+			r, err := run.NewRunner(env, config.ViperConfig.GetDuration("run-timeout"), config.ViperConfig.GetDuration("gc"))
 			if err != nil {
 				exitCode = 2
 				return
@@ -210,7 +210,7 @@ func NewCommand() (*cobra.Command, *int) {
 		),
 		Run: func(cmd *cobra.Command, args []string) {
 			for i := 0; i < len(args); i++ {
-				err := api.ResetNamespace(config.ViperConfig.GetString("api-address"), args[i])
+				err := api.ResetNamespace(config.ViperConfig.GetDuration("client-timeout"), config.ViperConfig.GetString("api-address"), args[i])
 				if err != nil {
 					exitCode = 2
 					return
@@ -219,7 +219,7 @@ func NewCommand() (*cobra.Command, *int) {
 			if !config.ViperConfig.GetBool("apply") {
 				return
 			}
-			err := api.Apply(config.ViperConfig.GetString("api-address"))
+			err := api.Apply(config.ViperConfig.GetDuration("client-timeout"), config.ViperConfig.GetString("api-address"))
 			if err != nil {
 				exitCode = 2
 				return
@@ -250,7 +250,7 @@ func NewCommand() (*cobra.Command, *int) {
 				exitCode = 1
 				return
 			}
-			err := wait.NewWaiter(unitToWatch, config.ViperConfig.GetDuration("timeout"), config.ViperConfig.GetDuration("logging-since")).Wait()
+			err := wait.NewWaiter(unitToWatch, config.ViperConfig.GetDuration("wait-timeout"), config.ViperConfig.GetDuration("logging-since")).Wait()
 			if err != nil {
 				exitCode = 2
 				return
@@ -282,7 +282,7 @@ func NewCommand() (*cobra.Command, *int) {
 	daemonCommand.PersistentFlags().String("systemd-unit-prefix", config.ViperConfig.GetString("systemd-unit-prefix"), "prefix for systemd unit name")
 	config.ViperConfig.BindPFlag("systemd-unit-prefix", daemonCommand.PersistentFlags().Lookup("systemd-unit-prefix"))
 
-	daemonCommand.PersistentFlags().String("kubectl-link", config.ViperConfig.GetString("kubectl-link"), "Path to create a kubectl link")
+	daemonCommand.PersistentFlags().String("kubectl-link", config.ViperConfig.GetString("kubectl-link"), "path to create a kubectl link")
 	config.ViperConfig.BindPFlag("kubectl-link", daemonCommand.PersistentFlags().Lookup("kubectl-link"))
 
 	daemonCommand.PersistentFlags().StringP("clean", "c", config.ViperConfig.GetString("clean"), fmt.Sprintf("clean options before %s: %s", setupCommand.Name(), options.GetOptionNames(options.Clean{})))
@@ -300,13 +300,13 @@ func NewCommand() (*cobra.Command, *int) {
 	runCommand.PersistentFlags().StringP("drain", "d", config.ViperConfig.GetString("drain"), fmt.Sprintf("drain options after %s: %s", runCommand.Name(), options.GetOptionNames(options.Drain{})))
 	config.ViperConfig.BindPFlag("drain", runCommand.PersistentFlags().Lookup("drain"))
 
-	runCommand.PersistentFlags().Duration("timeout", config.ViperConfig.GetDuration("timeout"), fmt.Sprintf("timeout for %s", runCommand.Name()))
-	config.ViperConfig.BindPFlag("timeout", runCommand.PersistentFlags().Lookup("timeout"))
+	runCommand.PersistentFlags().Duration("run-timeout", config.ViperConfig.GetDuration("run-timeout"), fmt.Sprintf("maximum time to run %s for until self shutdown", programName))
+	config.ViperConfig.BindPFlag("run-timeout", runCommand.PersistentFlags().Lookup("run-timeout"))
 
 	runCommand.PersistentFlags().Duration("gc", config.ViperConfig.GetDuration("gc"), fmt.Sprintf("grace period for the kubelet GC trigger when draining %s, no-op if not draining", runCommand.Name()))
 	config.ViperConfig.BindPFlag("gc", runCommand.PersistentFlags().Lookup("gc"))
 
-	runCommand.PersistentFlags().String("bind-address", config.ViperConfig.GetString("bind-address"), fmt.Sprintf("Bind address for %s API ip:port", programName))
+	runCommand.PersistentFlags().String("bind-address", config.ViperConfig.GetString("bind-address"), fmt.Sprintf("bind address for %s API ip:port", programName))
 	config.ViperConfig.BindPFlag("bind-address", runCommand.PersistentFlags().Lookup("bind-address"))
 
 	runCommand.PersistentFlags().String("systemd-job-name", config.ViperConfig.GetString("systemd-job-name"), "unit name used when running as systemd service")
@@ -317,22 +317,25 @@ func NewCommand() (*cobra.Command, *int) {
 
 	// Reset
 	rootCommand.AddCommand(resetCommand)
-	resetCommand.PersistentFlags().String("api-address", config.ViperConfig.GetString("api-address"), fmt.Sprintf("Address for the %s API ip:port", programName))
+	resetCommand.PersistentFlags().String("api-address", config.ViperConfig.GetString("api-address"), fmt.Sprintf("address for the %s API ip:port", programName))
 	config.ViperConfig.BindPFlag("api-address", resetCommand.PersistentFlags().Lookup("api-address"))
 
-	resetCommand.PersistentFlags().BoolP("apply", "a", config.ViperConfig.GetBool("apply"), "Apply manifests-api after reset, useful when resetting kube-system namespace")
+	resetCommand.PersistentFlags().BoolP("apply", "a", config.ViperConfig.GetBool("apply"), "apply manifests-api after reset, useful when resetting kube-system namespace")
 	config.ViperConfig.BindPFlag("apply", resetCommand.PersistentFlags().Lookup("apply"))
 
+	resetCommand.PersistentFlags().Duration("client-timeout", config.ViperConfig.GetDuration("client-timeout"), fmt.Sprintf("maximum time waited for a %s command to be executed", programName))
+	config.ViperConfig.BindPFlag("client-timeout", resetCommand.PersistentFlags().Lookup("client-timeout"))
+
 	// Wait
 	rootCommand.AddCommand(waitCommand)
 
-	waitCommand.PersistentFlags().Duration("timeout", time.Minute*15, fmt.Sprintf("Timeout for %s", waitCommand.Name()))
-	config.ViperConfig.BindPFlag("timeout", waitCommand.PersistentFlags().Lookup("timeout"))
+	waitCommand.PersistentFlags().Duration("wait-timeout", time.Minute*15, fmt.Sprintf("maximum time to download the required binaries, images and set up %s", programName))
+	config.ViperConfig.BindPFlag("wait-timeout", waitCommand.PersistentFlags().Lookup("wait-timeout"))
 
-	waitCommand.PersistentFlags().Duration("logging-since", config.ViperConfig.GetDuration("logging-since"), "Display the logs of the unit since")
+	waitCommand.PersistentFlags().Duration("logging-since", config.ViperConfig.GetDuration("logging-since"), "display the logs of the unit since")
 	config.ViperConfig.BindPFlag("logging-since", waitCommand.PersistentFlags().Lookup("logging-since"))
 
-	waitCommand.PersistentFlags().StringP("unit-to-watch", "u", config.ViperConfig.GetString("unit-to-watch"), "Systemd unit name to watch")
+	waitCommand.PersistentFlags().StringP("unit-to-watch", "u", config.ViperConfig.GetString("unit-to-watch"), "systemd unit name to watch")
 	config.ViperConfig.BindPFlag("unit-to-watch", waitCommand.PersistentFlags().Lookup("unit-to-watch"))
 
 	return rootCommand, &exitCode

docs/pupernetes_daemon.md

@@ -14,7 +14,7 @@ Use this command to clean setup and run a Kubernetes local environment
       --etcd-version string          etcd version (default "3.1.11")
   -h, --help                         help for daemon
       --hyperkube-version string     hyperkube version (default "1.10.3")
-      --kubectl-link string          Path to create a kubectl link
+      --kubectl-link string          path to create a kubectl link
       --kubelet-root-dir string      directory path for managing kubelet files (default "/var/lib/p8s-kubelet")
       --systemd-unit-prefix string   prefix for systemd unit name (default "p8s-")
       --vault-version string         vault version (default "0.9.5")

docs/pupernetes_daemon_clean.md

@@ -38,7 +38,7 @@ pupernetes daemon clean state/ -c etcd,network,secrets
       --cni-version string           container network interface (cni) version (default "0.7.0")
       --etcd-version string          etcd version (default "3.1.11")
       --hyperkube-version string     hyperkube version (default "1.10.3")
-      --kubectl-link string          Path to create a kubectl link
+      --kubectl-link string          path to create a kubectl link
       --kubelet-root-dir string      directory path for managing kubelet files (default "/var/lib/p8s-kubelet")
       --systemd-unit-prefix string   prefix for systemd unit name (default "p8s-")
       --vault-version string         vault version (default "0.9.5")

docs/pupernetes_daemon_run.md

@@ -39,13 +39,13 @@ pupernetes daemon run state/ --job-type systemd
 ### Options
 
 ```
-      --bind-address string       Bind address for pupernetes API ip:port (default "127.0.0.1:8989")
+      --bind-address string       bind address for pupernetes API ip:port (default "127.0.0.1:8989")
   -d, --drain string              drain options after run: iptables,kubeletgc,pods,all,none (default "all")
       --gc duration               grace period for the kubelet GC trigger when draining run, no-op if not draining (default 1m0s)
   -h, --help                      help for run
       --job-type string           type of job: fg or systemd (default "fg")
+      --run-timeout duration      maximum time to run pupernetes for until self shutdown (default 7h0m0s)
       --systemd-job-name string   unit name used when running as systemd service (default "pupernetes")
-      --timeout duration          timeout for run (default 6h0m0s)
 ```
 
 ### Options inherited from parent commands
@@ -55,7 +55,7 @@ pupernetes daemon run state/ --job-type systemd
       --cni-version string           container network interface (cni) version (default "0.7.0")
       --etcd-version string          etcd version (default "3.1.11")
       --hyperkube-version string     hyperkube version (default "1.10.3")
-      --kubectl-link string          Path to create a kubectl link
+      --kubectl-link string          path to create a kubectl link
       --kubelet-root-dir string      directory path for managing kubelet files (default "/var/lib/p8s-kubelet")
       --systemd-unit-prefix string   prefix for systemd unit name (default "p8s-")
       --vault-version string         vault version (default "0.9.5")

docs/pupernetes_daemon_setup.md

@@ -29,7 +29,7 @@ pupernetes daemon setup state/
       --cni-version string           container network interface (cni) version (default "0.7.0")
       --etcd-version string          etcd version (default "3.1.11")
       --hyperkube-version string     hyperkube version (default "1.10.3")
-      --kubectl-link string          Path to create a kubectl link
+      --kubectl-link string          path to create a kubectl link
       --kubelet-root-dir string      directory path for managing kubelet files (default "/var/lib/p8s-kubelet")
       --systemd-unit-prefix string   prefix for systemd unit name (default "p8s-")
       --vault-version string         vault version (default "0.9.5")

docs/pupernetes_reset.md

@@ -31,9 +31,10 @@ pupernetes reset default $(kubectl get ns -o name) --apply
 ### Options
 
 ```
-      --api-address string   Address for the pupernetes API ip:port (default "127.0.0.1:8989")
-  -a, --apply                Apply manifests-api after reset, useful when resetting kube-system namespace
-  -h, --help                 help for reset
+      --api-address string        address for the pupernetes API ip:port (default "127.0.0.1:8989")
+  -a, --apply                     apply manifests-api after reset, useful when resetting kube-system namespace
+      --client-timeout duration   maximum time waited for a pupernetes command to be executed (default 1m0s)
+  -h, --help                      help for reset
 ```
 
 ### Options inherited from parent commands

docs/pupernetes_wait.md

@@ -26,9 +26,9 @@ pupernetes wait -u p8s-kubelet
 
 ```
   -h, --help                     help for wait
-      --logging-since duration   Display the logs of the unit since (default 5m0s)
-      --timeout duration         Timeout for wait (default 15m0s)
-  -u, --unit-to-watch string     Systemd unit name to watch (default "pupernetes.service")
+      --logging-since duration   display the logs of the unit since (default 5m0s)
+  -u, --unit-to-watch string     systemd unit name to watch (default "pupernetes.service")
+      --wait-timeout duration    maximum time to download the required binaries, images and set up pupernetes (default 15m0s)
 ```
 
 ### Options inherited from parent commands

pkg/api/client.go

@@ -15,7 +15,7 @@ const (
 
 // ResetNamespace executes an API call to the pupernetes API to reset
 // the namespace in parameter. The namespace can be like ns/default or just default
-func ResetNamespace(apiAddress, namespace string) error {
+func ResetNamespace(timeout time.Duration, apiAddress, namespace string) error {
 	if strings.HasPrefix(namespace, namespacePrefix) {
 		glog.V(4).Infof("Stripping namespace %q", namespace)
 		namespace = namespace[len(namespacePrefix):]
@@ -27,19 +27,19 @@ func ResetNamespace(apiAddress, namespace string) error {
 		return err
 	}
 	glog.Infof("Resetting namespace %q ...", namespace)
-	return doPOST(apiAddress, fmt.Sprintf("%s/%s", resetRoute, namespace))
+	return doPOST(timeout, apiAddress, fmt.Sprintf("%s/%s", resetRoute, namespace))
 }
 
 // Apply executes an API call to the pupernetes API to force an apply of the "manifest-api" directory
-func Apply(apiAddress string) error {
+func Apply(timeout time.Duration, apiAddress string) error {
 	glog.Infof("Applying ...")
-	return doPOST(apiAddress, applyRoute)
+	return doPOST(timeout, apiAddress, applyRoute)
 }
 
-func doPOST(apiAddress, apiRoute string) error {
+func doPOST(timeout time.Duration, apiAddress, apiRoute string) error {
 	glog.Infof("Calling POST %s ...", apiRoute)
 	c := &http.Client{}
-	c.Timeout = time.Second * 5
+	c.Timeout = timeout
 
 	u, err := url.Parse(fmt.Sprintf("http://%s%s", apiAddress, apiRoute))
 	if err != nil {

pkg/config/config.go

@@ -44,7 +44,7 @@ func init() {
 
 	ViperConfig.SetDefault("clean", "etcd,kubelet,mounts,iptables")
 	ViperConfig.SetDefault("drain", "all")
-	ViperConfig.SetDefault("timeout", time.Hour*6)
+	ViperConfig.SetDefault("run-timeout", time.Hour*7)
 	ViperConfig.SetDefault("gc", time.Second*60)
 
 	// The supported job-type are "fg" and "systemd"
@@ -56,4 +56,6 @@ func init() {
 
 	ViperConfig.SetDefault("logging-since", time.Minute*5)
 	ViperConfig.SetDefault("unit-to-watch", "pupernetes.service")
+	ViperConfig.SetDefault("wait-timeout", time.Minute*15)
+	ViperConfig.SetDefault("client-timeout", time.Minute*1)
 }

pkg/run/run.go

@@ -7,23 +7,22 @@ package run
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 	"time"
 
 	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/DataDog/pupernetes/pkg/api"
-	"github.com/DataDog/pupernetes/pkg/config"
 	"github.com/DataDog/pupernetes/pkg/logging"
 	"github.com/DataDog/pupernetes/pkg/run/state"
 	"github.com/DataDog/pupernetes/pkg/setup"
 	"github.com/DataDog/pupernetes/pkg/util"
-	"io/ioutil"
-	"k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sync"
 )
 
 const (
@@ -51,7 +50,7 @@ type Runtime struct {
 }
 
 // NewRunner instantiate a new Runtimer with the given Environment
-func NewRunner(env *setup.Environment) (*Runtime, error) {
+func NewRunner(env *setup.Environment, runTimeout, waitKubeletGC time.Duration) (*Runtime, error) {
 	var zero int64
 
 	s, err := state.NewState()
@@ -67,8 +66,8 @@ func NewRunner(env *setup.Environment) (*Runtime, error) {
 		httpClient: &http.Client{
 			Timeout: time.Millisecond * 500,
 		},
-		runTimeout:    config.ViperConfig.GetDuration("timeout"),
-		waitKubeletGC: config.ViperConfig.GetDuration("gc"),
+		runTimeout:    runTimeout,
+		waitKubeletGC: waitKubeletGC,
 		kubeDeleteOption: &v1.DeleteOptions{
 			GracePeriodSeconds: &zero,
 		},
@@ -138,8 +137,8 @@ func (r *Runtime) Run() error {
 			if failures >= appProbeThreshold {
 				glog.Warningf("Probing failed, stopping ...")
 				// display some helpers to investigate:
-				glog.Infof("Investigate the kubelet logs with: journalctl -u %skubelet.service -o cat -e --no-pager", config.ViperConfig.GetString("systemd-unit-prefix"))
-				glog.Infof("Investigate the kubelet status with: systemctl status %skubelet.service -l --no-pager", config.ViperConfig.GetString("systemd-unit-prefix"))
+				glog.Infof("Investigate the kubelet logs with: journalctl -u %skubelet.service -o cat -e --no-pager", r.env.GetSystemdUnitPrefix())
+				glog.Infof("Investigate the kubelet status with: systemctl status %skubelet.service -l --no-pager", r.env.GetSystemdUnitPrefix())
 				// Propagate a stop
 				return r.Stop(fmt.Errorf("failure threshold reached %d/%d", failures, appProbeThreshold))
 			}

pkg/setup/getter.go

@@ -122,3 +122,8 @@ func (e *Environment) GetPublicIP() string {
 func (e *Environment) GetSystemdUnits() []string {
 	return e.systemdUnitNames
 }
+
+// GetSystemdUnitPrefix returns the prefix used with systemd units
+func (e *Environment) GetSystemdUnitPrefix() string {
+	return e.systemdUnitPrefix
+}