diff --git a/cmd/root.go b/cmd/root.go index 77cfe2a52..c415236da 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,6 +9,7 @@ import ( "runtime/pprof" "sync" "syscall" + "time" "github.com/spf13/cobra" @@ -99,25 +100,46 @@ func rootMain(cmd *cobra.Command, args []string) { } logger.Info("started all services") - wg.Wait() - // Collect memory profile. if cfg.Metrics.MemProfile != "" { - logger.Info("Writing memory profile", "dest", cfg.Metrics.MemProfile) - memF, err := os.Create(cfg.Metrics.MemProfile) - if err != nil { - logger.Error("failed to create memory profile", "err", err) - os.Exit(1) - } - defer memF.Close() - // runtime.GC() - if err := pprof.WriteHeapProfile(memF); err != nil { - logger.Error("failed to write memory profile", "err", err) - os.Exit(1) + go collectMemProfiles(cfg.Metrics.MemProfile, logger) + } + wg.Wait() +} + +func collectMemProfiles(dest string, logger *log.Logger) { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(signalChan) // Stop catching Ctrl+C signals. + + for { + select { + case <-time.After(5 * time.Minute): + logMemProfile(dest, logger) + case <-signalChan: + logger.Info("received interrupt, shutting down") + // Let the default handler handle ctrl+C so people can kill the process in a hurry. + signal.Stop(signalChan) + return } } } +func logMemProfile(dest string, logger *log.Logger) { + logger.Info("Writing memory profile", "dest", dest) + memF, err := os.Create(dest) + if err != nil { + logger.Error("failed to create memory profile", "err", err) + os.Exit(1) + } + defer memF.Close() + // runtime.GC() + if err := pprof.WriteHeapProfile(memF); err != nil { + logger.Error("failed to write memory profile", "err", err) + os.Exit(1) + } +} + // Execute spawns the main entry point after handing the config file. func Execute() { // Debug hook. If we receive SIGUSR1, dump all goroutines.