From f7c2985792f4174ac12af9f8be29af3e27d75a26 Mon Sep 17 00:00:00 2001
From: Levente Pap <levente.pap@iota.org>
Date: Wed, 22 Jul 2020 16:15:32 +0200
Subject: [PATCH] =?UTF-8?q?=F0=9F=93=88=20Introduce=20missing=20message=20?=
 =?UTF-8?q?metrics=20(#666)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- measure number missing messages in DB at startup
- listen to events of the messagelayer tangle to determine current
  number of missing messages
- export missing message metrics through prometheus
- add missing messages chart to grafana local dashboard
---
 packages/binary/messagelayer/tangle/tangle.go |  12 +-
 plugins/metrics/message.go                    |  14 +-
 plugins/metrics/plugin.go                     |  12 +
 plugins/prometheus/tangle.go                  |   8 +
 .../grafana/dashboards/local_dashboard.json   | 239 ++++++++++++------
 .../grafana/dashboards/local_dashboard.json   | 239 ++++++++++++------
 6 files changed, 377 insertions(+), 147 deletions(-)

diff --git a/packages/binary/messagelayer/tangle/tangle.go b/packages/binary/messagelayer/tangle/tangle.go
index 4f81ddb0..32cb2d43 100644
--- a/packages/binary/messagelayer/tangle/tangle.go
+++ b/packages/binary/messagelayer/tangle/tangle.go
@@ -135,8 +135,10 @@ func (tangle *Tangle) Prune() error {
 	return nil
 }
 
-// DBStats returns the number of solid messages and total number of messages in the database, furthermore the average time it takes to solidify messages.
-func (tangle *Tangle) DBStats() (solidCount int, messageCount int, avgSolidificationTime float64) {
+// DBStats returns the number of solid messages and total number of messages in the database (messageMetadataStorage,
+// that should contain the messages as messageStorage), the number of messages in missingMessageStorage, furthermore
+// the average time it takes to solidify messages.
+func (tangle *Tangle) DBStats() (solidCount int, messageCount int, avgSolidificationTime float64, missingMessageCount int) {
 	var sumSolidificationTime time.Duration
 	tangle.messageMetadataStorage.ForEach(func(key []byte, cachedObject objectstorage.CachedObject) bool {
 		cachedObject.Consume(func(object objectstorage.StorableObject) {
@@ -153,6 +155,12 @@ func (tangle *Tangle) DBStats() (solidCount int, messageCount int, avgSolidifica
 	if solidCount > 0 {
 		avgSolidificationTime = float64(sumSolidificationTime.Milliseconds()) / float64(solidCount)
 	}
+	tangle.messageMetadataStorage.ForEach(func(key []byte, cachedObject objectstorage.CachedObject) bool {
+		cachedObject.Consume(func(object objectstorage.StorableObject) {
+			missingMessageCount++
+		})
+		return true
+	})
 	return
 }
 
diff --git a/plugins/metrics/message.go b/plugins/metrics/message.go
index 59bc98f4..e9041728 100644
--- a/plugins/metrics/message.go
+++ b/plugins/metrics/message.go
@@ -33,6 +33,12 @@ var (
 	sumSolidificationTime time.Duration
 	solidTimeMutex        syncutils.RWMutex
 
+	// initial number of missing messages in missingMessageStorage (at startup)
+	initialMissingMessageCountDB uint64
+
+	// current number of missing messages in missingMessageStorage
+	missingMessageCountDB atomic.Uint64
+
 	// current number of message tips.
 	messageTips atomic.Uint64
 
@@ -104,6 +110,11 @@ func AvgSolidificationTime() (result float64) {
 	return
 }
 
+// MessageMissingCountDB returns the number of messages in missingMessageStore.
+func MessageMissingCountDB() uint64 {
+	return initialMissingMessageCountDB + missingMessageCountDB.Load()
+}
+
 // ReceivedMessagesPerSecond retrieves the current messages per second number.
 func ReceivedMessagesPerSecond() uint64 {
 	return measuredReceivedMPS.Load()
@@ -150,8 +161,9 @@ func measureRequestQueueSize() {
 }
 
 func measureInitialDBStats() {
-	solid, total, avgSolidTime := messagelayer.Tangle().DBStats()
+	solid, total, avgSolidTime, missing := messagelayer.Tangle().DBStats()
 	initialMessageSolidCountDB = uint64(solid)
 	initialMessageTotalCountDB = uint64(total)
 	initialSumSolidificationTime = avgSolidTime * float64(solid)
+	initialMissingMessageCountDB = uint64(missing)
 }
diff --git a/plugins/metrics/plugin.go b/plugins/metrics/plugin.go
index 500409ed..3166edc6 100644
--- a/plugins/metrics/plugin.go
+++ b/plugins/metrics/plugin.go
@@ -123,6 +123,18 @@ func registerLocalMetrics() {
 		})
 	}))
 
+	// fired when a message gets added to missing message storage
+	messagelayer.Tangle().Events.MessageMissing.Attach(events.NewClosure(func(messageId message.Id) {
+		missingMessageCountDB.Inc()
+	}))
+
+	// fired when a missing message was received and removed from missing message storage
+	messagelayer.Tangle().Events.MissingMessageReceived.Attach(events.NewClosure(func(cachedMessage *message.CachedMessage, cachedMessageMetadata *tangle.CachedMessageMetadata) {
+		cachedMessage.Release()
+		cachedMessageMetadata.Release()
+		missingMessageCountDB.Dec()
+	}))
+
 	// Value payload attached
 	valuetransfers.Tangle().Events.PayloadAttached.Attach(events.NewClosure(func(cachedPayload *payload.CachedPayload, cachedPayloadMetadata *valuetangle.CachedPayloadMetadata) {
 		cachedPayload.Release()
diff --git a/plugins/prometheus/tangle.go b/plugins/prometheus/tangle.go
index 10e7ef4f..71061179 100644
--- a/plugins/prometheus/tangle.go
+++ b/plugins/prometheus/tangle.go
@@ -13,6 +13,7 @@ var (
 	messageTotalCountDB   prometheus.Gauge
 	messageSolidCountDB   prometheus.Gauge
 	avgSolidificationTime prometheus.Gauge
+	messageMissingCountDB prometheus.Gauge
 	messageRequestCount   prometheus.Gauge
 
 	transactionCounter prometheus.Gauge
@@ -53,6 +54,11 @@ func registerTangleMetrics() {
 		Help: "average time it takes for a message to become solid",
 	})
 
+	messageMissingCountDB = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "tangle_message_missing_count_db",
+		Help: "number of missing messages in the node's database",
+	})
+
 	transactionCounter = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "tangle_value_transaction_counter",
 		Help: "number of value transactions (value payloads) seen",
@@ -74,6 +80,7 @@ func registerTangleMetrics() {
 	registry.MustRegister(messageTotalCountDB)
 	registry.MustRegister(messageSolidCountDB)
 	registry.MustRegister(avgSolidificationTime)
+	registry.MustRegister(messageMissingCountDB)
 	registry.MustRegister(messageRequestCount)
 	registry.MustRegister(transactionCounter)
 	registry.MustRegister(valueTips)
@@ -91,6 +98,7 @@ func collectTangleMetrics() {
 	messageTotalCountDB.Set(float64(metrics.MessageTotalCountDB()))
 	messageSolidCountDB.Set(float64(metrics.MessageSolidCountDB()))
 	avgSolidificationTime.Set(metrics.AvgSolidificationTime())
+	messageMissingCountDB.Set(float64(metrics.MessageMissingCountDB()))
 	messageRequestCount.Set(float64(metrics.MessageRequestQueueSize()))
 	transactionCounter.Set(float64(metrics.ValueTransactionCounter()))
 	valueTips.Set(float64(metrics.ValueTips()))
diff --git a/tools/docker-network/grafana/dashboards/local_dashboard.json b/tools/docker-network/grafana/dashboards/local_dashboard.json
index e6af8e79..b7e6e835 100644
--- a/tools/docker-network/grafana/dashboards/local_dashboard.json
+++ b/tools/docker-network/grafana/dashboards/local_dashboard.json
@@ -16,7 +16,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 2,
+  "id": 1,
   "links": [],
   "panels": [
     {
@@ -74,7 +74,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "iota_info_app",
@@ -153,7 +153,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "sync",
@@ -212,7 +212,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "irate(tangle_message_total_count[5m])",
@@ -270,7 +270,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "process_cpu_usage",
@@ -324,7 +324,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "process_mem_usage_bytes/1024/1024",
@@ -382,7 +382,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "db_size_bytes/1024/1024",
@@ -693,7 +693,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_neighbor_connections_count - autopeering_neighbor_drop_count",
@@ -747,7 +747,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_avg_neighbor_connection_lifetime",
@@ -804,7 +804,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_neighbor_connections_count",
@@ -861,7 +861,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_neighbor_drop_count",
@@ -1170,7 +1170,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "description": "Average time it takes to solidify messages.",
+      "description": "Number of messages in missingMessageStore. These are the message the node knows it doesn't have and tries to requests them.",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -1186,7 +1186,7 @@
         "y": 25
       },
       "hiddenSeries": false,
-      "id": 75,
+      "id": 77,
       "legend": {
         "avg": false,
         "current": false,
@@ -1212,9 +1212,9 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "tangle_message_avg_solidification_time",
+          "expr": "tangle_message_missing_count_db",
           "interval": "",
-          "legendFormat": "Avg Solidification Time",
+          "legendFormat": "Missing Messages",
           "refId": "A"
         }
       ],
@@ -1222,7 +1222,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Average Solidification Time",
+      "title": "Missing Messages",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1238,7 +1238,7 @@
       },
       "yaxes": [
         {
-          "format": "ms",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1593,6 +1593,101 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Average time it takes to solidify messages.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 10,
+        "w": 11,
+        "x": 11,
+        "y": 41
+      },
+      "hiddenSeries": false,
+      "id": 75,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "tangle_message_avg_solidification_time",
+          "interval": "",
+          "legendFormat": "Avg Solidification Time",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Average Solidification Time",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "ms",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "content": "\n# Total Network Traffic\n\n\nSince the start of the node.\n",
       "datasource": null,
@@ -1604,10 +1699,10 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 3,
-        "w": 6,
-        "x": 11,
-        "y": 41
+        "h": 4,
+        "w": 3,
+        "x": 0,
+        "y": 51
       },
       "id": 61,
       "mode": "markdown",
@@ -1638,8 +1733,8 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 11,
-        "y": 44
+        "x": 3,
+        "y": 51
       },
       "id": 67,
       "options": {
@@ -1655,7 +1750,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "traffic_gossip_outbound_bytes",
@@ -1691,8 +1786,8 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 13,
-        "y": 44
+        "x": 5,
+        "y": 51
       },
       "id": 66,
       "options": {
@@ -1708,7 +1803,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "traffic_gossip_inbound_bytes",
@@ -1744,10 +1839,10 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 15,
-        "y": 44
+        "x": 7,
+        "y": 51
       },
-      "id": 59,
+      "id": 63,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1761,10 +1856,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_analysis_outbound_bytes",
+          "expr": "traffic_autopeering_outbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1772,7 +1867,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Analysis",
+      "title": "Autopeering TX",
       "type": "stat"
     },
     {
@@ -1796,11 +1891,11 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 3,
-        "x": 11,
-        "y": 46
+        "w": 2,
+        "x": 9,
+        "y": 51
       },
-      "id": 63,
+      "id": 62,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1814,10 +1909,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_autopeering_outbound_bytes",
+          "expr": "traffic_autopeering_inbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1825,7 +1920,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Autopeering TX",
+      "title": "Autopeering RX",
       "type": "stat"
     },
     {
@@ -1849,11 +1944,11 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 3,
-        "x": 14,
-        "y": 46
+        "w": 2,
+        "x": 3,
+        "y": 53
       },
-      "id": 62,
+      "id": 64,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1867,10 +1962,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_autopeering_inbound_bytes",
+          "expr": "traffic_fpc_inbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1878,7 +1973,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Autopeering RX",
+      "title": "FPC RX",
       "type": "stat"
     },
     {
@@ -1903,10 +1998,10 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 11,
-        "y": 48
+        "x": 5,
+        "y": 53
       },
-      "id": 64,
+      "id": 65,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1920,10 +2015,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_fpc_inbound_bytes",
+          "expr": "traffic_fpc_outbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1931,7 +2026,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "FPC RX",
+      "title": "FPC TX",
       "type": "stat"
     },
     {
@@ -1956,10 +2051,10 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 13,
-        "y": 48
+        "x": 7,
+        "y": 53
       },
-      "id": 65,
+      "id": 59,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1973,10 +2068,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_fpc_outbound_bytes",
+          "expr": "traffic_analysis_outbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1984,7 +2079,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "FPC TX",
+      "title": "Analysis",
       "type": "stat"
     },
     {
@@ -1994,7 +2089,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 51
+        "y": 55
       },
       "id": 34,
       "panels": [],
@@ -2019,7 +2114,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 52
+        "y": 56
       },
       "hiddenSeries": false,
       "id": 36,
@@ -2122,7 +2217,7 @@
         "h": 8,
         "w": 4,
         "x": 12,
-        "y": 52
+        "y": 56
       },
       "id": 38,
       "options": {
@@ -2138,7 +2233,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "fpc_avg_rounds_to_finalize",
@@ -2179,7 +2274,7 @@
         "h": 8,
         "w": 3,
         "x": 16,
-        "y": 52
+        "y": 56
       },
       "id": 42,
       "options": {
@@ -2195,7 +2290,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "fpc_finalized_conflicts",
@@ -2236,7 +2331,7 @@
         "h": 8,
         "w": 3,
         "x": 19,
-        "y": 52
+        "y": 56
       },
       "id": 40,
       "options": {
@@ -2252,7 +2347,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "fpc_failed_conflicts",
@@ -2285,7 +2380,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 60
+        "y": 64
       },
       "hiddenSeries": false,
       "id": 44,
@@ -2379,7 +2474,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 60
+        "y": 64
       },
       "hiddenSeries": false,
       "id": 50,
@@ -2474,7 +2569,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 68
+        "y": 72
       },
       "hiddenSeries": false,
       "id": 49,
@@ -2568,7 +2663,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 68
+        "y": 72
       },
       "hiddenSeries": false,
       "id": 46,
@@ -2675,5 +2770,5 @@
   "timezone": "",
   "title": "GoShimmer Local Metrics",
   "uid": "kjOQZ2ZMk",
-  "version": 11
+  "version": 6
 }
diff --git a/tools/monitoring/grafana/dashboards/local_dashboard.json b/tools/monitoring/grafana/dashboards/local_dashboard.json
index e6af8e79..b7e6e835 100755
--- a/tools/monitoring/grafana/dashboards/local_dashboard.json
+++ b/tools/monitoring/grafana/dashboards/local_dashboard.json
@@ -16,7 +16,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 2,
+  "id": 1,
   "links": [],
   "panels": [
     {
@@ -74,7 +74,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "iota_info_app",
@@ -153,7 +153,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "sync",
@@ -212,7 +212,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "irate(tangle_message_total_count[5m])",
@@ -270,7 +270,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "process_cpu_usage",
@@ -324,7 +324,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "process_mem_usage_bytes/1024/1024",
@@ -382,7 +382,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "db_size_bytes/1024/1024",
@@ -693,7 +693,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_neighbor_connections_count - autopeering_neighbor_drop_count",
@@ -747,7 +747,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_avg_neighbor_connection_lifetime",
@@ -804,7 +804,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_neighbor_connections_count",
@@ -861,7 +861,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "autopeering_neighbor_drop_count",
@@ -1170,7 +1170,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "Prometheus",
-      "description": "Average time it takes to solidify messages.",
+      "description": "Number of messages in missingMessageStore. These are the message the node knows it doesn't have and tries to requests them.",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -1186,7 +1186,7 @@
         "y": 25
       },
       "hiddenSeries": false,
-      "id": 75,
+      "id": 77,
       "legend": {
         "avg": false,
         "current": false,
@@ -1212,9 +1212,9 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "tangle_message_avg_solidification_time",
+          "expr": "tangle_message_missing_count_db",
           "interval": "",
-          "legendFormat": "Avg Solidification Time",
+          "legendFormat": "Missing Messages",
           "refId": "A"
         }
       ],
@@ -1222,7 +1222,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Average Solidification Time",
+      "title": "Missing Messages",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1238,7 +1238,7 @@
       },
       "yaxes": [
         {
-          "format": "ms",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1593,6 +1593,101 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Average time it takes to solidify messages.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 10,
+        "w": 11,
+        "x": 11,
+        "y": 41
+      },
+      "hiddenSeries": false,
+      "id": 75,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "tangle_message_avg_solidification_time",
+          "interval": "",
+          "legendFormat": "Avg Solidification Time",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Average Solidification Time",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "ms",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "content": "\n# Total Network Traffic\n\n\nSince the start of the node.\n",
       "datasource": null,
@@ -1604,10 +1699,10 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 3,
-        "w": 6,
-        "x": 11,
-        "y": 41
+        "h": 4,
+        "w": 3,
+        "x": 0,
+        "y": 51
       },
       "id": 61,
       "mode": "markdown",
@@ -1638,8 +1733,8 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 11,
-        "y": 44
+        "x": 3,
+        "y": 51
       },
       "id": 67,
       "options": {
@@ -1655,7 +1750,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "traffic_gossip_outbound_bytes",
@@ -1691,8 +1786,8 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 13,
-        "y": 44
+        "x": 5,
+        "y": 51
       },
       "id": 66,
       "options": {
@@ -1708,7 +1803,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "traffic_gossip_inbound_bytes",
@@ -1744,10 +1839,10 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 15,
-        "y": 44
+        "x": 7,
+        "y": 51
       },
-      "id": 59,
+      "id": 63,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1761,10 +1856,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_analysis_outbound_bytes",
+          "expr": "traffic_autopeering_outbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1772,7 +1867,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Analysis",
+      "title": "Autopeering TX",
       "type": "stat"
     },
     {
@@ -1796,11 +1891,11 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 3,
-        "x": 11,
-        "y": 46
+        "w": 2,
+        "x": 9,
+        "y": 51
       },
-      "id": 63,
+      "id": 62,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1814,10 +1909,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_autopeering_outbound_bytes",
+          "expr": "traffic_autopeering_inbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1825,7 +1920,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Autopeering TX",
+      "title": "Autopeering RX",
       "type": "stat"
     },
     {
@@ -1849,11 +1944,11 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 3,
-        "x": 14,
-        "y": 46
+        "w": 2,
+        "x": 3,
+        "y": 53
       },
-      "id": 62,
+      "id": 64,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1867,10 +1962,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_autopeering_inbound_bytes",
+          "expr": "traffic_fpc_inbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1878,7 +1973,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Autopeering RX",
+      "title": "FPC RX",
       "type": "stat"
     },
     {
@@ -1903,10 +1998,10 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 11,
-        "y": 48
+        "x": 5,
+        "y": 53
       },
-      "id": 64,
+      "id": 65,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1920,10 +2015,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_fpc_inbound_bytes",
+          "expr": "traffic_fpc_outbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1931,7 +2026,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "FPC RX",
+      "title": "FPC TX",
       "type": "stat"
     },
     {
@@ -1956,10 +2051,10 @@
       "gridPos": {
         "h": 2,
         "w": 2,
-        "x": 13,
-        "y": 48
+        "x": 7,
+        "y": 53
       },
-      "id": 65,
+      "id": 59,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
@@ -1973,10 +2068,10 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
-          "expr": "traffic_fpc_outbound_bytes",
+          "expr": "traffic_analysis_outbound_bytes",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1984,7 +2079,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "FPC TX",
+      "title": "Analysis",
       "type": "stat"
     },
     {
@@ -1994,7 +2089,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 51
+        "y": 55
       },
       "id": 34,
       "panels": [],
@@ -2019,7 +2114,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 52
+        "y": 56
       },
       "hiddenSeries": false,
       "id": 36,
@@ -2122,7 +2217,7 @@
         "h": 8,
         "w": 4,
         "x": 12,
-        "y": 52
+        "y": 56
       },
       "id": 38,
       "options": {
@@ -2138,7 +2233,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "fpc_avg_rounds_to_finalize",
@@ -2179,7 +2274,7 @@
         "h": 8,
         "w": 3,
         "x": 16,
-        "y": 52
+        "y": 56
       },
       "id": 42,
       "options": {
@@ -2195,7 +2290,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "fpc_finalized_conflicts",
@@ -2236,7 +2331,7 @@
         "h": 8,
         "w": 3,
         "x": 19,
-        "y": 52
+        "y": 56
       },
       "id": 40,
       "options": {
@@ -2252,7 +2347,7 @@
           "values": false
         }
       },
-      "pluginVersion": "7.0.4",
+      "pluginVersion": "7.0.3",
       "targets": [
         {
           "expr": "fpc_failed_conflicts",
@@ -2285,7 +2380,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 60
+        "y": 64
       },
       "hiddenSeries": false,
       "id": 44,
@@ -2379,7 +2474,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 60
+        "y": 64
       },
       "hiddenSeries": false,
       "id": 50,
@@ -2474,7 +2569,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 68
+        "y": 72
       },
       "hiddenSeries": false,
       "id": 49,
@@ -2568,7 +2663,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 68
+        "y": 72
       },
       "hiddenSeries": false,
       "id": 46,
@@ -2675,5 +2770,5 @@
   "timezone": "",
   "title": "GoShimmer Local Metrics",
   "uid": "kjOQZ2ZMk",
-  "version": 11
+  "version": 6
 }
-- 
GitLab