Removes periodic badgerdb gc & adds dirty/health flag to db (#526)

* removes periodic badgerdb gc, adds dirty/health flag to db, do badgerdb gc at startup/shutdown

* removes error shadowing

* instruct the user to delete the db folder if the db can't be opened

* addresses review comments

packages/database/prefix/prefix.go

@@ -7,5 +7,5 @@ const (
 	DBPrefixTransactionMetadata
 	DBPrefixAddressTransactions
 	DBPrefixAutoPeering
-	DBPrefixDatabaseVersion
+	DBPrefixHealth
 )

packages/shutdown/order.go

@@ -16,5 +16,4 @@ const (
 	PrioritySynchronization
 	PriorityBootstrap
 	PrioritySpammer
-	PriorityBadgerGarbageCollection
 )

plugins/database/health.go

+package database
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/iotaledger/goshimmer/packages/database/prefix"
+	"github.com/iotaledger/hive.go/kvstore"
+)
+
+var (
+	healthStore kvstore.KVStore
+	healthKey   = []byte("db_health")
+)
+
+func configureHealthStore(store kvstore.KVStore) {
+	healthStore = store.WithRealm([]byte{prefix.DBPrefixHealth})
+}
+
+// MarkDatabaseUnhealthy marks the database as not healthy, meaning
+// that it wasn't shutdown properly.
+func MarkDatabaseUnhealthy() {
+	if err := healthStore.Set(healthKey, []byte{}); err != nil {
+		panic(fmt.Errorf("failed to set database health state: %w", err))
+	}
+}
+
+// MarkDatabaseHealthy marks the database as healthy, respectively correctly closed.
+func MarkDatabaseHealthy() {
+	if err := healthStore.Delete(healthKey); err != nil && !errors.Is(err, kvstore.ErrKeyNotFound) {
+		panic(fmt.Errorf("failed to set database health state: %w", err))
+	}
+}
+
+// IsDatabaseUnhealthy tells whether the database is unhealthy, meaning not shutdown properly.
+func IsDatabaseUnhealthy() bool {
+	contains, err := healthStore.Has(healthKey)
+	if err != nil {
+		panic(fmt.Errorf("failed to set database health state: %w", err))
+	}
+	return contains
+}

plugins/database/plugin.go

@@ -7,14 +7,12 @@ import (
 	"time"
 
 	"github.com/iotaledger/goshimmer/packages/database"
-	"github.com/iotaledger/goshimmer/packages/database/prefix"
 	"github.com/iotaledger/goshimmer/packages/shutdown"
 	"github.com/iotaledger/goshimmer/plugins/config"
 	"github.com/iotaledger/hive.go/daemon"
 	"github.com/iotaledger/hive.go/kvstore"
 	"github.com/iotaledger/hive.go/logger"
 	"github.com/iotaledger/hive.go/node"
-	"github.com/iotaledger/hive.go/timeutil"
 )
 
 // PluginName is the name of the database plugin.
@@ -22,7 +20,7 @@ const PluginName = "Database"
 
 var (
 	// Plugin is the plugin instance of the database plugin.
-	Plugin = node.NewPlugin(PluginName, node.Enabled, configure, run)
+	Plugin = node.NewPlugin(PluginName, node.Enabled, configure)
 	log    *logger.Logger
 
 	db        database.DB
@@ -52,7 +50,7 @@ func createStore() {
 		db, err = database.NewDB(dbDir)
 	}
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal("Unable to open the database, please delete the database folder. Error: %s", err)
 	}
 
 	store = db.NewStore()
@@ -61,30 +59,38 @@ func createStore() {
 func configure(_ *node.Plugin) {
 	// assure that the store is initialized
 	store := Store()
+	configureHealthStore(store)
 
-	err := checkDatabaseVersion(store.WithRealm([]byte{prefix.DBPrefixDatabaseVersion}))
-	if errors.Is(err, ErrDBVersionIncompatible) {
-		log.Panicf("The database scheme was updated. Please delete the database folder.\n%s", err)
+	if err := checkDatabaseVersion(healthStore); err != nil {
+		if errors.Is(err, ErrDBVersionIncompatible) {
+			log.Fatalf("The database scheme was updated. Please delete the database folder. %s", err)
+		}
+		log.Fatalf("Failed to check database version: %s", err)
 	}
-	if err != nil {
-		log.Panicf("Failed to check database version: %s", err)
+
+	if IsDatabaseUnhealthy() {
+		log.Fatal("The database is marked as not properly shutdown/corrupted, please delete the database folder and restart.")
 	}
 
 	// we open the database in the configure, so we must also make sure it's closed here
-	err = daemon.BackgroundWorker(PluginName, closeDB, shutdown.PriorityDatabase)
-	if err != nil {
-		log.Panicf("Failed to start as daemon: %s", err)
+	if err := daemon.BackgroundWorker(PluginName, manageDBLifetime, shutdown.PriorityDatabase); err != nil {
+		log.Fatalf("Failed to start as daemon: %s", err)
 	}
-}
 
-func run(_ *node.Plugin) {
-	if err := daemon.BackgroundWorker(PluginName+"[GC]", runGC, shutdown.PriorityBadgerGarbageCollection); err != nil {
-		log.Panicf("Failed to start as daemon: %s", err)
-	}
+	// run GC up on startup
+	runDatabaseGC()
 }
 
-func closeDB(shutdownSignal <-chan struct{}) {
+// manageDBLifetime takes care of managing the lifetime of the database. It marks the database as dirty up on
+// startup and unmarks it up on shutdown. Up on shutdown it will run the db GC and then close the database.
+func manageDBLifetime(shutdownSignal <-chan struct{}) {
+	// we mark the database only as corrupted from within a background worker, which means
+	// that we only mark it as dirty, if the node actually started up properly (meaning no termination
+	// signal was received before all plugins loaded).
+	MarkDatabaseUnhealthy()
 	<-shutdownSignal
+	runDatabaseGC()
+	MarkDatabaseHealthy()
 	log.Infof("Syncing database to disk...")
 	if err := db.Close(); err != nil {
 		log.Errorf("Failed to flush the database: %s", err)
@@ -92,14 +98,15 @@ func closeDB(shutdownSignal <-chan struct{}) {
 	log.Infof("Syncing database to disk... done")
 }
 
-func runGC(shutdownSignal <-chan struct{}) {
+func runDatabaseGC() {
 	if !db.RequiresGC() {
 		return
 	}
-	// run the garbage collection with the given interval
-	timeutil.Ticker(func() {
-		if err := db.GC(); err != nil {
-			log.Warnf("Garbage collection failed: %s", err)
-		}
-	}, 5*time.Minute, shutdownSignal)
+	log.Info("Running database garbage collection...")
+	s := time.Now()
+	if err := db.GC(); err != nil {
+		log.Warnf("Database garbage collection failed: %s", err)
+		return
+	}
+	log.Infof("Database garbage collection done, took %v...", time.Since(s))
 }