Adds msg approval analysis api (#924)

*  Add data structure and table description

*  Add comments

*  Add FutureCone iterator and FirstApprovalAnalysis

*  Fix return value

*  Simplify CSV conversion

*  Improve code readability

*  Add comments

*  Add approval API endpoint

*  Fix missing fields

*  Use filePath

*  Fix linter warning

*  Add tooling

 - script to conveniently trigger approval checks in
   docker-network and collect the results.
 - works for all network sizes, extracts data
   from peer_master plus all replicas.

* 

 Update docker-network README

* Refactor approversInfo initialization

Co-authored-by: Levente Pap <levente.pap@iota.org>

Co-authored-by: Levente Pap <levente.pap@iota.org>

packages/tangle/analysis.go

+package tangle
+
+import (
+	"container/list"
+	"encoding/csv"
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/iotaledger/hive.go/identity"
+	"github.com/iotaledger/hive.go/objectstorage"
+	"github.com/iotaledger/hive.go/types"
+)
+
+// TableDescription holds the description of the First Approval analysis table.
+var TableDescription = []string{
+	"nodeID",
+	"MsgID",
+	"MsgIssuerID",
+	"MsgIssuanceTime",
+	"MsgArrivalTime",
+	"MsgSolidTime",
+	"ByIssuanceMsgID",
+	"ByIssuanceMsgIssuerID",
+	"ByIssuanceMsgIssuanceTime",
+	"ByIssuanceMsgArrivalTime",
+	"ByIssuanceMsgSolidTime",
+	"ByArrivalMsgID",
+	"ByArrivalMsgIssuerID",
+	"ByArrivalMsgIssuanceTime",
+	"ByArrivalMsgArrivalTime",
+	"ByArrivalMsgSolidTime",
+	"BySolidMsgID",
+	"BySolidMsgIssuerID",
+	"BySolidMsgIssuanceTime",
+	"BySolidMsgArrivalTime",
+	"BySolidMsgSolidTime",
+}
+
+// MsgInfo holds the information of a message.
+type MsgInfo struct {
+	MsgID                string
+	MsgIssuerID          string
+	MsgIssuanceTimestamp time.Time
+	MsgArrivalTime       time.Time
+	MsgSolidTime         time.Time
+}
+
+// MsgApproval holds the information of the first approval by issucane, arrival and solid time.
+type MsgApproval struct {
+	NodeID                  string
+	Msg                     MsgInfo
+	FirstApproverByIssuance MsgInfo
+	FirstApproverByArrival  MsgInfo
+	FirstApproverBySolid    MsgInfo
+}
+
+type approverType uint8
+
+const (
+	byIssuance approverType = iota
+	byArrival
+	bySolid
+)
+
+// ByIssuance defines a slice of MsgInfo sortable by timestamp issuance.
+type ByIssuance []MsgInfo
+
+func (a ByIssuance) Len() int { return len(a) }
+func (a ByIssuance) Less(i, j int) bool {
+	return a[i].MsgIssuanceTimestamp.Before(a[j].MsgIssuanceTimestamp)
+}
+func (a ByIssuance) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+// ByArrival defines a slice of MsgInfo sortable by arrival time.
+type ByArrival []MsgInfo
+
+func (a ByArrival) Len() int { return len(a) }
+func (a ByArrival) Less(i, j int) bool {
+	return a[i].MsgArrivalTime.Before(a[j].MsgArrivalTime)
+}
+func (a ByArrival) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+// BySolid defines a slice of MsgInfo sortable by solid time.
+type BySolid []MsgInfo
+
+func (a BySolid) Len() int { return len(a) }
+func (a BySolid) Less(i, j int) bool {
+	return a[i].MsgSolidTime.Before(a[j].MsgSolidTime)
+}
+func (a BySolid) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+// FirstApprovalAnalysis performs the first approval analysis and write the result into a csv.
+// This function is very heavy to compute especially when starting from the Genesis.
+// A better alternative for the future would be to keep this analysis updated as the Tangle grows.
+func (t *Tangle) FirstApprovalAnalysis(nodeID string, filePath string) error {
+	// If the file doesn't exist, create it, or truncate the file
+	f, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	w := csv.NewWriter(f)
+
+	// write TableDescription
+	if err := w.Write(TableDescription); err != nil {
+		return err
+	}
+
+	return t.FutureCone(EmptyMessageID, func(msgID MessageID) error {
+		approverInfo, err := t.firstApprovers(msgID)
+		// firstApprovers returns an error when the msgID is a tip, thus
+		// we want to stop the computation but continue with the future cone iteration.
+		if err != nil {
+			return nil
+		}
+
+		msgApproval := MsgApproval{
+			NodeID:                  nodeID,
+			Msg:                     t.info(msgID),
+			FirstApproverByIssuance: approverInfo[byIssuance],
+			FirstApproverByArrival:  approverInfo[byArrival],
+			FirstApproverBySolid:    approverInfo[bySolid],
+		}
+
+		// write msgApproval to file
+		if err := w.Write(msgApproval.toCSV()); err != nil {
+			return err
+		}
+		w.Flush()
+		if err := w.Error(); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+// FutureCone iterates over the future cone of the given messageID and computes the given function.
+func (t *Tangle) FutureCone(messageID MessageID, compute func(ID MessageID) error) error {
+	futureConeStack := list.New()
+	futureConeStack.PushBack(messageID)
+
+	processedMessages := make(map[MessageID]types.Empty)
+	processedMessages[messageID] = types.Void
+
+	for futureConeStack.Len() >= 1 {
+		currentStackEntry := futureConeStack.Front()
+		currentMessageID := currentStackEntry.Value.(MessageID)
+		futureConeStack.Remove(currentStackEntry)
+
+		if err := compute(currentMessageID); err != nil {
+			return err
+		}
+
+		t.Approvers(currentMessageID).Consume(func(approver *Approver) {
+			approverID := approver.ApproverMessageID()
+			if _, messageProcessed := processedMessages[approverID]; !messageProcessed {
+				futureConeStack.PushBack(approverID)
+				processedMessages[approverID] = types.Void
+			}
+		})
+	}
+
+	return nil
+}
+
+func (t *Tangle) firstApprovers(msgID MessageID) ([]MsgInfo, error) {
+	approversInfo := make([]MsgInfo, 0)
+
+	t.Approvers(msgID).Consume(func(approver *Approver) {
+		approversInfo = append(approversInfo, t.info(approver.ApproverMessageID()))
+	})
+
+	if len(approversInfo) == 0 {
+		return nil, fmt.Errorf("message: %v is a tip", msgID)
+	}
+
+	result := make([]MsgInfo, 3)
+
+	sort.Sort(ByIssuance(approversInfo))
+	result[byIssuance] = approversInfo[0]
+
+	sort.Sort(ByArrival(approversInfo))
+	result[byArrival] = approversInfo[0]
+
+	sort.Sort(BySolid(approversInfo))
+	result[bySolid] = approversInfo[0]
+
+	return result, nil
+}
+
+func (t *Tangle) info(msgID MessageID) MsgInfo {
+	msgInfo := MsgInfo{
+		MsgID: msgID.String(),
+	}
+
+	t.Message(msgID).Consume(func(msg *Message) {
+		msgInfo.MsgIssuanceTimestamp = msg.IssuingTime()
+		msgInfo.MsgIssuerID = identity.NewID(msg.IssuerPublicKey()).String()
+	})
+
+	t.MessageMetadata(msgID).Consume(func(object objectstorage.StorableObject) {
+		msgMetadata := object.(*MessageMetadata)
+		msgInfo.MsgArrivalTime = msgMetadata.ReceivedTime()
+		msgInfo.MsgSolidTime = msgMetadata.SolidificationTime()
+	}, false)
+
+	return msgInfo
+}
+
+func (m MsgApproval) toCSV() (row []string) {
+	row = append(row, m.NodeID)
+	row = append(row, m.Msg.toCSV()...)
+	row = append(row, m.FirstApproverByIssuance.toCSV()...)
+	row = append(row, m.FirstApproverByArrival.toCSV()...)
+	row = append(row, m.FirstApproverBySolid.toCSV()...)
+	return
+}
+
+func (m MsgInfo) toCSV() (row []string) {
+	row = append(row, []string{
+		m.MsgID,
+		m.MsgIssuerID,
+		fmt.Sprint(m.MsgIssuanceTimestamp.UnixNano()),
+		fmt.Sprint(m.MsgArrivalTime.UnixNano()),
+		fmt.Sprint(m.MsgSolidTime.UnixNano())}...)
+
+	return
+}

plugins/webapi/tools/message/approval.go

+package message
+
+import (
+	"net/http"
+
+	"github.com/iotaledger/goshimmer/plugins/autopeering/local"
+	"github.com/iotaledger/goshimmer/plugins/config"
+	"github.com/iotaledger/goshimmer/plugins/messagelayer"
+	"github.com/labstack/echo"
+)
+
+var fileName = "approval-analysis.csv"
+
+// ApprovalHandler runs the approval analysis.
+func ApprovalHandler(c echo.Context) error {
+	path := config.Node().String(CfgExportPath)
+	res := &ApprovalResponse{}
+	res.Err = messagelayer.Tangle().FirstApprovalAnalysis(local.GetInstance().Identity.ID().String(), path+fileName)
+	if res.Err != nil {
+		c.JSON(http.StatusInternalServerError, res)
+	}
+	return c.JSON(http.StatusOK, res)
+}
+
+// ApprovalResponse is the HTTP response.
+type ApprovalResponse struct {
+	Err error `json:"error,omitempty"`
+}

plugins/webapi/tools/message/parameters.go

+package message
+
+import flag "github.com/spf13/pflag"
+
+const (
+	// CfgExportPath the directory where exported files sit.
+	CfgExportPath = "webapi.exportPath"
+)
+
+func init() {
+	flag.String(CfgExportPath, ".", "default export path")
+}

plugins/webapi/tools/plugin.go

@@ -32,6 +32,7 @@ func configure(_ *node.Plugin) {
 	log = logger.NewLogger(PluginName)
 	webapi.Server().GET("tools/message/pastcone", message.PastconeHandler)
 	webapi.Server().GET("tools/message/missing", message.MissingHandler)
+	webapi.Server().GET("tools/message/approval", message.ApprovalHandler)
 	webapi.Server().GET("tools/value/tips", value.TipsHandler)
 	webapi.Server().GET("tools/value/objects", value.ObjectsHandler)
 }

tools/docker-network/README.md

@@ -28,3 +28,30 @@ via `docker logs --follow CONTAINER` or all of them combined when running via:
 ```
 ./run.sh 5
 ```
+
+## How to use message approval check tool
+
+`get_approval_csv.sh` script helps you conveniently trigger the message approval checks on all nodes in the docker
+network, and gather their results in the `csv` folder.
+
+Once the network is up and running, execute the script:
+```
+$ ./get_approval_csv.sh
+```
+Example output:
+```
+Triggering approval analysis on peer_master and 20 replicas...
+Triggering approval analysis on peer_master and 20 replicas... DONE
+Copying csv files from peer_master and 20 replicas...
+Copying csv files from peer_master and 20 replicas... DONE
+Copied files are located at /home/{user}/repo/goshimmer/tools/docker-network/csv
+```
+The exported csv files are timestamped to the date of request.
+```
+csv
+├── 210120_16_34_14-docker-network_peer_replica_10.csv
+├── 210120_16_34_14-docker-network_peer_replica_11.csv
+├── 210120_16_34_14-docker-network_peer_replica_12.csv
+...
+```
+Note, that the record length of the files might differ, since the approval check execution time of the nodes might differ.

tools/docker-network/docker-compose.yml

@@ -41,13 +41,14 @@ services:
       --config=/tmp/config.json
       --database.directory=/tmp/mainnetdb
       --autopeering.seed=base58:8q491c3YWjbPwLmF2WD95YmCgh61j2kenCKHfGfByoWi
-      --node.enablePlugins=prometheus,spammer,faucet,syncbeacon
+      --node.enablePlugins=prometheus,spammer,faucet,syncbeacon,"webapi tools endpoint"
       --node.disablePlugins=syncbeaconfollower,clock
       --faucet.seed=7R1itJx5hVuo9w9hjg5cwKFmek4HMSoBDgJZN8hKGxih
       --valueLayer.snapshot.file=/tmp/assets/7R1itJx5hVuo9w9hjg5cwKFmek4HMSoBDgJZN8hKGxih.bin
       --syncbeacon.broadcastInterval=5
       --syncbeacon.startSynced=true
       --statement.writeStatement=true
+      --webapi.exportPath=/tmp/
     volumes:
       - ./config.docker.json:/tmp/config.json:ro
       - goshimmer-cache:/go
@@ -66,10 +67,11 @@ services:
     command: >
       --config=/tmp/config.json
       --database.directory=/tmp/mainnetdb
-      --node.enablePlugins=bootstrap
+      --node.enablePlugins=bootstrap,"webapi tools endpoint"
       --valueLayer.snapshot.file=/tmp/assets/7R1itJx5hVuo9w9hjg5cwKFmek4HMSoBDgJZN8hKGxih.bin
       --node.disablePlugins=portcheck,clock
       --syncbeaconfollower.followNodes=EYsaGXnUVA9aTYL9FwYEvoQ8d1HCJveQVL7vogu6pqCP
+      --webapi.exportPath=/tmp/
     volumes:
       - ./config.docker.json:/tmp/config.json:ro
       - goshimmer-cache:/go

tools/docker-network/get_approval_csv.sh

+#!/bin/bash
+
+## Get the number of replicas
+NUM_REPLICAS=`docker ps | grep peer_replica | wc -l`
+
+echo "Triggering approval analysis on peer_master and $NUM_REPLICAS replicas..."
+
+curl -s -i -H \"Accept: application/json\" -H \"Content-Type: application/json\"\
+ -X GET http://localhost:8080/tools/message/approval > /dev/null &
+
+for (( c=1; c<=NUM_REPLICAS; c++ ))
+do
+  docker exec --interactive peer_master /bin/bash -c\
+  "curl -s -i -o /dev/null -H \"Accept: application/json\"\
+  -H \"Content-Type: application/json\"\
+  -X GET http://docker-network_peer_replica_$c:8080/tools/message/approval" &
+done
+wait
+echo "Triggering approval analysis on peer_master and $NUM_REPLICAS replicas... DONE"
+
+now=$(date +"%y%m%d_%H_%M_%S")
+mkdir -p csv
+
+echo "Copying csv files from peer_master and $NUM_REPLICAS replicas..."
+docker cp peer_master:/tmp/approval-analysis.csv ./csv/$now-peer_master.csv
+for (( i=1; i<=NUM_REPLICAS; i++ ))
+do
+  docker cp docker-network_peer_replica_$i:/tmp/approval-analysis.csv ./csv/$now-docker-network_peer_replica_$i.csv
+done
+
+echo "Copying csv files from peer_master and $NUM_REPLICAS replicas... DONE"
+echo "Copied files are located at `pwd`/csv"
\ No newline at end of file