Cleanup metrics and prometheus plugin

plugins/metrics/message.go

 package metrics
 
 import (
+	"time"
+
 	"github.com/iotaledger/goshimmer/packages/binary/messagelayer/payload"
 	"github.com/iotaledger/goshimmer/packages/metrics"
 	"github.com/iotaledger/goshimmer/plugins/messagelayer"
@@ -12,26 +14,26 @@ var (
 	// Total number of processed messages since start of the node.
 	messageTotalCount atomic.Uint64
 
-	// current number of solid messages in the node's database
-	messageSolidCountDBIter atomic.Uint64
+	// number of messages in the database at startup
+	initialMessageTotalCountDB atomic.Uint64
 
-	// current number of solid messages in the node's database
-	messageSolidCountDBInc atomic.Uint64
+	// current number of messages in the node's database
+	messageTotalCountDB atomic.Uint64
 
 	// number of solid messages in the database at startup
 	initialMessageSolidCountDB atomic.Uint64
 
-	// current number of messages in the node's database
-	messageTotalCountDBIter atomic.Uint64
-
-	// current number of messages in the node's database
-	messageTotalCountDBInc atomic.Uint64
+	// current number of solid messages in the node's database
+	messageSolidCountDBInc atomic.Uint64
 
-	// number of messages in the database at startup
-	initialMessageTotalCountDB atomic.Uint64
+	// initial average solidification time
+	initialAvgSolidificationTime atomic.Float64
+	// helper variable that is only calculated at init phase. unit is milliseconds!
+	initialSumSolidificationTime float64
 
-	// average time it takes to solidify a message
-	avgSolidificationTime atomic.Float64
+	// sum of solidification time (since start of the node)
+	sumSolidificationTime time.Duration
+	solidTimeMutex        syncutils.RWMutex
 
 	// current number of message tips.
 	messageTips atomic.Uint64
@@ -83,29 +85,19 @@ func MessageRequestQueueSize() int64 {
 	return requestQueueSize.Load()
 }
 
-// MessageSolidCountIter returns the number of messages that are solid.
-func MessageSolidCountIter() uint64 {
-	return messageSolidCountDBIter.Load()
-}
-
-// MessageTotalCountDBIter returns the number of messages that are stored in the DB.
-func MessageTotalCountDBIter() uint64 {
-	return messageTotalCountDBIter.Load()
-}
-
-// MessageSolidCountInc returns the number of messages that are solid.
-func MessageSolidCountInc() uint64 {
+// MessageSolidCountDB returns the number of messages that are solid in the DB.
+func MessageSolidCountDB() uint64 {
 	return initialMessageSolidCountDB.Load() + messageSolidCountDBInc.Load()
 }
 
-// MessageTotalCountDBInc returns the number of messages that are stored in the DB.
-func MessageTotalCountDBInc() uint64 {
-	return initialMessageTotalCountDB.Load() + messageTotalCountDBInc.Load()
+// MessageTotalCountDB returns the number of messages that are stored in the DB.
+func MessageTotalCountDB() uint64 {
+	return initialMessageTotalCountDB.Load() + messageTotalCountDB.Load()
 }
 
-// AvgSolidificationTime returns the average time it takes for a message to become solid.
+// AvgSolidificationTime returns the average time it takes for a message to become solid. [milliseconds]
 func AvgSolidificationTime() float64 {
-	return avgSolidificationTime.Load()
+	return (initialSumSolidificationTime + float64(sumSolidificationTime.Milliseconds())) / float64(MessageSolidCountDB())
 }
 
 // ReceivedMessagesPerSecond retrieves the current messages per second number.
@@ -153,16 +145,10 @@ func measureRequestQueueSize() {
 	requestQueueSize.Store(size)
 }
 
-func measureDBStats() {
-	solid, total, avgSolidTime := messagelayer.Tangle().DBStats()
-	messageSolidCountDBIter.Store(uint64(solid))
-	messageTotalCountDBIter.Store(uint64(total))
-	avgSolidificationTime.Store(avgSolidTime)
-}
-
 func measureInitialDBStats() {
 	solid, total, avgSolidTime := messagelayer.Tangle().DBStats()
 	initialMessageSolidCountDB.Store(uint64(solid))
 	initialMessageTotalCountDB.Store(uint64(total))
-	avgSolidificationTime.Store(avgSolidTime)
+	initialAvgSolidificationTime.Store(avgSolidTime)
+	initialSumSolidificationTime = avgSolidTime * float64(total)
 }

plugins/metrics/plugin.go

@@ -21,6 +21,7 @@ import (
 	"github.com/iotaledger/hive.go/events"
 	"github.com/iotaledger/hive.go/logger"
 	"github.com/iotaledger/hive.go/node"
+	"github.com/iotaledger/hive.go/objectstorage"
 	"github.com/iotaledger/hive.go/timeutil"
 )
 
@@ -49,7 +50,7 @@ func configure(_ *node.Plugin) {
 func run(_ *node.Plugin) {
 	log.Infof("Starting %s ...", PluginName)
 	if config.Node().GetBool(CfgMetricsLocal) {
-		// initial measurement
+		// initial measurement, since we have to know how many messages are there in the db
 		measureInitialDBStats()
 		registerLocalMetrics()
 	}
@@ -85,19 +86,6 @@ func run(_ *node.Plugin) {
 	}, shutdown.PriorityMetrics); err != nil {
 		log.Panicf("Failed to start as daemon: %s", err)
 	}
-
-	if config.Node().GetBool(CfgMetricsLocalDB) {
-		// create a background worker that updates the db metrics every 20 second
-		if err := daemon.BackgroundWorker("Metrics Updater[DB]", func(shutdownSignal <-chan struct{}) {
-			defer log.Infof("Stopping Metrics Updater[DB] ... done")
-			timeutil.Ticker(func() {
-				measureDBStats()
-			}, 20*time.Second, shutdownSignal)
-			log.Infof("Stopping Metrics Updater[DB] ...")
-		}, shutdown.PriorityMetrics); err != nil {
-			log.Panicf("Failed to start as daemon: %s", err)
-		}
-	}
 }
 
 func registerLocalMetrics() {
@@ -111,19 +99,26 @@ func registerLocalMetrics() {
 		increaseReceivedMPSCounter()
 		increasePerPayloadCounter(_payloadType)
 		// MessageAttached is triggered in storeMessageWorker that saves the msg to database
-		messageTotalCountDBInc.Inc()
+		messageTotalCountDB.Inc()
 
 	}))
 
 	messagelayer.Tangle().Events.MessageRemoved.Attach(events.NewClosure(func(messageId message.Id) {
-		// MessageRemoved triggered when the message get removed from database.
-		messageTotalCountDBInc.Dec()
+		// MessageRemoved triggered when the message gets removed from database.
+		messageTotalCountDB.Dec()
 	}))
 
+	// messages can only become solid once, then they stay like that, hence no .Dec() part
 	messagelayer.Tangle().Events.MessageSolid.Attach(events.NewClosure(func(cachedMessage *message.CachedMessage, cachedMessageMetadata *tangle.CachedMessageMetadata) {
-		cachedMessage.Release()
-		cachedMessageMetadata.Release()
+		defer cachedMessage.Release()
+		defer cachedMessageMetadata.Release()
 		messageSolidCountDBInc.Inc()
+		solidTimeMutex.Lock()
+		defer solidTimeMutex.Unlock()
+		cachedMessageMetadata.Consume(func(object objectstorage.StorableObject) {
+			msgMetaData := object.(*tangle.MessageMetadata)
+			sumSolidificationTime += msgMetaData.SolidificationTime().Sub(msgMetaData.ReceivedTime())
+		})
 	}))
 
 	// Value payload attached

plugins/prometheus/tangle.go

@@ -7,15 +7,13 @@ import (
 )
 
 var (
-	messageTips             prometheus.Gauge
-	messagePerTypeCount     *prometheus.GaugeVec
-	messageTotalCount       prometheus.Gauge
-	messageTotalCountDBIter prometheus.Gauge
-	messageTotalCountDBInc  prometheus.Gauge
-	messageSolidCountDBIter prometheus.Gauge
-	messageSolidCountDBInc  prometheus.Gauge
-	avgSolidificationTime   prometheus.Gauge
-	messageRequestCount     prometheus.Gauge
+	messageTips           prometheus.Gauge
+	messagePerTypeCount   *prometheus.GaugeVec
+	messageTotalCount     prometheus.Gauge
+	messageTotalCountDB   prometheus.Gauge
+	messageSolidCountDB   prometheus.Gauge
+	avgSolidificationTime prometheus.Gauge
+	messageRequestCount   prometheus.Gauge
 
 	transactionCounter prometheus.Gauge
 	valueTips          prometheus.Gauge
@@ -40,23 +38,13 @@ func registerTangleMetrics() {
 		Help: "total number of messages seen since the start of the node",
 	})
 
-	messageTotalCountDBIter = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "tangle_message_total_count_db_iter",
+	messageTotalCountDB = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "tangle_message_total_count_db",
 		Help: "total number of messages in the node's database",
 	})
 
-	messageTotalCountDBInc = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "tangle_message_total_count_db_inc",
-		Help: "total number of messages in the node's database",
-	})
-
-	messageSolidCountDBIter = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "tangle_message_solid_count_int",
-		Help: "number of solid messages on the node's database",
-	})
-
-	messageSolidCountDBInc = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "tangle_message_solid_count_inc",
+	messageSolidCountDB = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "tangle_message_solid_count",
 		Help: "number of solid messages on the node's database",
 	})
 
@@ -83,10 +71,8 @@ func registerTangleMetrics() {
 	registry.MustRegister(messageTips)
 	registry.MustRegister(messagePerTypeCount)
 	registry.MustRegister(messageTotalCount)
-	registry.MustRegister(messageTotalCountDBIter)
-	registry.MustRegister(messageSolidCountDBIter)
-	registry.MustRegister(messageTotalCountDBInc)
-	registry.MustRegister(messageSolidCountDBInc)
+	registry.MustRegister(messageTotalCountDB)
+	registry.MustRegister(messageSolidCountDB)
 	registry.MustRegister(avgSolidificationTime)
 	registry.MustRegister(messageRequestCount)
 	registry.MustRegister(transactionCounter)
@@ -102,10 +88,8 @@ func collectTangleMetrics() {
 		messagePerTypeCount.WithLabelValues(payload.Name(payloadType)).Set(float64(count))
 	}
 	messageTotalCount.Set(float64(metrics.MessageTotalCountSinceStart()))
-	messageTotalCountDBIter.Set(float64(metrics.MessageTotalCountDBIter()))
-	messageSolidCountDBIter.Set(float64(metrics.MessageSolidCountIter()))
-	messageTotalCountDBInc.Set(float64(metrics.MessageTotalCountDBInc()))
-	messageSolidCountDBInc.Set(float64(metrics.MessageSolidCountInc()))
+	messageTotalCountDB.Set(float64(metrics.MessageTotalCountDB()))
+	messageSolidCountDB.Set(float64(metrics.MessageSolidCountDB()))
 	avgSolidificationTime.Set(metrics.AvgSolidificationTime())
 	messageRequestCount.Set(float64(metrics.MessageRequestQueueSize()))
 	transactionCounter.Set(float64(metrics.ValueTransactionCounter()))