diff --git a/pluginmgr/core/plugins.go b/pluginmgr/core/plugins.go index d604ddf2d3e18ea60d79284599696891c3b95001..03191820233abc9204e3984a79342ad8c5787065 100644 --- a/pluginmgr/core/plugins.go +++ b/pluginmgr/core/plugins.go @@ -29,6 +29,7 @@ var PLUGINS = node.Plugins( config.Plugin(), logger.Plugin(), cli.Plugin(), + gracefulshutdown.Plugin(), portcheck.Plugin(), profiling.Plugin(), database.Plugin(), @@ -39,7 +40,6 @@ var PLUGINS = node.Plugins( issuer.Plugin(), bootstrap.Plugin(), sync.Plugin(), - gracefulshutdown.Plugin(), metrics.Plugin(), drng.Plugin(), faucet.App(), diff --git a/plugins/gracefulshutdown/parameters.go b/plugins/gracefulshutdown/parameters.go new file mode 100644 index 0000000000000000000000000000000000000000..fbdca5fe4f05e120fd4370133284404e778bf69c --- /dev/null +++ b/plugins/gracefulshutdown/parameters.go @@ -0,0 +1,14 @@ +package gracefulshutdown + +import ( + flag "github.com/spf13/pflag" +) + +const ( + // CfgWaitToKillTimeInSeconds the maximum amount of time to wait for background processes to terminate. + CfgWaitToKillTimeInSeconds = "gracefulshutdown.waitToKillTime" +) + +func init() { + flag.Int(CfgWaitToKillTimeInSeconds, 60, "the maximum amount of time to wait for background processes to terminate, in seconds") +} diff --git a/plugins/gracefulshutdown/plugin.go b/plugins/gracefulshutdown/plugin.go index efc50f743957b8bf6308429096e4c2eb232ee49c..b191f7e3f4ff6e0bc32386119a6f1d9ceb4fd5e1 100644 --- a/plugins/gracefulshutdown/plugin.go +++ b/plugins/gracefulshutdown/plugin.go @@ -9,6 +9,7 @@ import ( "syscall" "time" + "github.com/iotaledger/goshimmer/plugins/config" "github.com/iotaledger/hive.go/daemon" "github.com/iotaledger/hive.go/logger" "github.com/iotaledger/hive.go/node" @@ -17,59 +18,60 @@ import ( // PluginName is the name of the graceful shutdown plugin. const PluginName = "Graceful Shutdown" -// WaitToKillTimeInSeconds is the maximum amount of time to wait for background processes to terminate. -// After that the process is killed. -const WaitToKillTimeInSeconds = 60 - var ( // plugin is the plugin instance of the graceful shutdown plugin. - plugin *node.Plugin - once sync.Once - log *logger.Logger - gracefulStop chan os.Signal + plugin *node.Plugin + once sync.Once + log *logger.Logger + gracefulStop chan os.Signal + waitToKillTimeInSeconds int ) -// Plugin gets the plugin instance. -func Plugin() *node.Plugin { - once.Do(func() { - plugin = node.NewPlugin(PluginName, node.Enabled, func(plugin *node.Plugin) { - log = logger.NewLogger(PluginName) - gracefulStop = make(chan os.Signal) +func configure(*node.Plugin) { + waitToKillTimeInSeconds = config.Node().GetInt(CfgWaitToKillTimeInSeconds) + + log = logger.NewLogger(PluginName) + gracefulStop = make(chan os.Signal) - signal.Notify(gracefulStop, syscall.SIGTERM) - signal.Notify(gracefulStop, syscall.SIGINT) + signal.Notify(gracefulStop, syscall.SIGTERM) + signal.Notify(gracefulStop, syscall.SIGINT) - go func() { - <-gracefulStop + go func() { + <-gracefulStop - log.Warnf("Received shutdown request - waiting (max %d) to finish processing ...", WaitToKillTimeInSeconds) + log.Warnf("Received shutdown request - waiting (max %d) to finish processing ...", waitToKillTimeInSeconds) - go func() { - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() - start := time.Now() - for x := range ticker.C { - secondsSinceStart := x.Sub(start).Seconds() + start := time.Now() + for x := range ticker.C { + secondsSinceStart := x.Sub(start).Seconds() - if secondsSinceStart <= WaitToKillTimeInSeconds { - processList := "" - runningBackgroundWorkers := daemon.GetRunningBackgroundWorkers() - if len(runningBackgroundWorkers) >= 1 { - sort.Strings(runningBackgroundWorkers) - processList = "(" + strings.Join(runningBackgroundWorkers, ", ") + ") " - } - log.Warnf("Received shutdown request - waiting (max %d seconds) to finish processing %s...", WaitToKillTimeInSeconds-int(secondsSinceStart), processList) - } else { - log.Error("Background processes did not terminate in time! Forcing shutdown ...") - os.Exit(1) - } + if secondsSinceStart <= float64(waitToKillTimeInSeconds) { + processList := "" + runningBackgroundWorkers := daemon.GetRunningBackgroundWorkers() + if len(runningBackgroundWorkers) >= 1 { + sort.Strings(runningBackgroundWorkers) + processList = "(" + strings.Join(runningBackgroundWorkers, ", ") + ") " } - }() + log.Warnf("Received shutdown request - waiting (max %d seconds) to finish processing %s...", waitToKillTimeInSeconds-int(secondsSinceStart), processList) + } else { + log.Error("Background processes did not terminate in time! Forcing shutdown ...") + os.Exit(1) + } + } + }() - daemon.Shutdown() - }() - }) + daemon.Shutdown() + }() +} + +// Plugin gets the plugin instance. +func Plugin() *node.Plugin { + once.Do(func() { + plugin = node.NewPlugin(PluginName, node.Enabled, configure) }) return plugin } diff --git a/tools/integration-tests/tester/framework/docker.go b/tools/integration-tests/tester/framework/docker.go index 198296965999cb4c7a1e82b85b4d151e2611e9fc..31b9bfd5f26ceb32e26d3ff28c0c2d7caf4a112d 100644 --- a/tools/integration-tests/tester/framework/docker.go +++ b/tools/integration-tests/tester/framework/docker.go @@ -86,6 +86,7 @@ func (d *DockerContainer) CreateGoShimmerPeer(config GoShimmerConfig) error { fmt.Sprintf("--autopeering.outboundUpdateIntervalMs=%d", ParaOutboundUpdateIntervalMs), fmt.Sprintf("--node.disablePlugins=%s", config.DisabledPlugins), fmt.Sprintf("--pow.difficulty=%d", ParaPoWDifficulty), + fmt.Sprintf("--gracefulshutdown.waitToKillTime=%d", ParaWaitToKill), fmt.Sprintf("--node.enablePlugins=%s", func() string { var plugins []string if config.Bootstrap { diff --git a/tools/integration-tests/tester/framework/parameters.go b/tools/integration-tests/tester/framework/parameters.go index 6d33d3632f62aed22be6184d95de4aed2a700add..9fa13fe839cefc05874ce95917f054d1e8d1f40d 100644 --- a/tools/integration-tests/tester/framework/parameters.go +++ b/tools/integration-tests/tester/framework/parameters.go @@ -35,6 +35,8 @@ var ( ParaFaucetTokensPerRequest int64 = 1337 // ParaPoWDifficulty defines the PoW difficulty. ParaPoWDifficulty = 2 + // ParaWaitToKill defines the time to wait before killing the node. + ParaWaitToKill = 60 ) var ( diff --git a/tools/integration-tests/tester/tests/consensus/consensus_conflicts_test.go b/tools/integration-tests/tester/tests/consensus/consensus_conflicts_test.go index 46fbc4cacde20637fd9ef384912a6dd7fd0d115c..4aec007c066a5feaeab1cb9fe530631228da5f4f 100644 --- a/tools/integration-tests/tester/tests/consensus/consensus_conflicts_test.go +++ b/tools/integration-tests/tester/tests/consensus/consensus_conflicts_test.go @@ -27,13 +27,16 @@ func TestConsensusFiftyFiftyOpinionSplit(t *testing.T) { // override avg. network delay to accustom integration test slowness backupFCoBAvgNetworkDelay := framework.ParaFCoBAverageNetworkDelay backupBootstrapOnEveryNode := framework.ParaBootstrapOnEveryNode + backupParaWaitToKill := framework.ParaWaitToKill framework.ParaFCoBAverageNetworkDelay = 90 framework.ParaBootstrapOnEveryNode = true + framework.ParaWaitToKill = 2 * framework.ParaFCoBAverageNetworkDelay // reset framework paras defer func() { framework.ParaFCoBAverageNetworkDelay = backupFCoBAvgNetworkDelay framework.ParaBootstrapOnEveryNode = backupBootstrapOnEveryNode + framework.ParaWaitToKill = backupParaWaitToKill }() // create two partitions with their own peers