Merge branch 'master' into feat/iota.go

plugins/autopeering/saltmanager/saltmanager.go

@@ -68,7 +68,7 @@ func scheduleUpdateForSalt(saltToUpdate *salt.Salt, settingsKey []byte, lifeSpan
 	if saltToUpdate.ExpirationTime.Before(now) {
 		updatePublicSalt(saltToUpdate, settingsKey, lifeSpan, callback)
 	} else {
-		daemon.BackgroundWorker(func() {
+		daemon.BackgroundWorker("Salt Updater", func() {
 			select {
 			case <-time.After(saltToUpdate.ExpirationTime.Sub(now)):
 				updatePublicSalt(saltToUpdate, settingsKey, lifeSpan, callback)

plugins/autopeering/server/tcp/server.go

@@ -37,7 +37,7 @@ func ConfigureServer(plugin *node.Plugin) {
 }
 
 func RunServer(plugin *node.Plugin) {
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Autopeering TCP Server", func() {
 		if *parameters.ADDRESS.Value == "0.0.0.0" {
 			plugin.LogInfo("Starting TCP Server (port " + strconv.Itoa(*parameters.PORT.Value) + ") ...")
 		} else {

plugins/autopeering/server/udp/server.go

@@ -41,7 +41,7 @@ func ConfigureServer(plugin *node.Plugin) {
 }
 
 func RunServer(plugin *node.Plugin) {
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Autopeering UDP Server", func() {
 		if *parameters.ADDRESS.Value == "0.0.0.0" {
 			plugin.LogInfo("Starting UDP Server (port " + strconv.Itoa(*parameters.PORT.Value) + ") ...")
 		} else {

plugins/bundleprocessor/bundleprocessor.go

+package bundleprocessor
+
+import (
+	"github.com/iotaledger/goshimmer/packages/errors"
+	"github.com/iotaledger/goshimmer/packages/model/bundle"
+	"github.com/iotaledger/goshimmer/packages/model/transactionmetadata"
+	"github.com/iotaledger/goshimmer/packages/model/value_transaction"
+	"github.com/iotaledger/goshimmer/packages/ternary"
+	"github.com/iotaledger/goshimmer/plugins/tangle"
+)
+
+func ProcessSolidBundleHead(headTransaction *value_transaction.ValueTransaction) (*bundle.Bundle, errors.IdentifiableError) {
+	// only process the bundle if we didn't process it, yet
+	return tangle.GetBundle(headTransaction.GetHash(), func(headTransactionHash ternary.Trytes) (*bundle.Bundle, errors.IdentifiableError) {
+		// abort if bundle syntax is wrong
+		if !headTransaction.IsHead() {
+			return nil, ErrProcessBundleFailed.Derive(errors.New("invalid parameter"), "transaction needs to be head of bundle")
+		}
+
+		// initialize event variables
+		newBundle := bundle.New(headTransactionHash)
+		bundleTransactions := make([]*value_transaction.ValueTransaction, 0)
+
+		// iterate through trunk transactions until we reach the tail
+		currentTransaction := headTransaction
+		for {
+			// abort if we reached a previous head
+			if currentTransaction.IsHead() && currentTransaction != headTransaction {
+				newBundle.SetTransactionHashes(mapTransactionsToTransactionHashes(bundleTransactions))
+
+				Events.InvalidBundle.Trigger(newBundle, bundleTransactions)
+
+				return nil, ErrProcessBundleFailed.Derive(errors.New("invalid bundle found"), "missing bundle tail")
+			}
+
+			// update bundle transactions
+			bundleTransactions = append(bundleTransactions, currentTransaction)
+
+			// retrieve & update metadata
+			currentTransactionMetadata, dbErr := tangle.GetTransactionMetadata(currentTransaction.GetHash(), transactionmetadata.New)
+			if dbErr != nil {
+				return nil, ErrProcessBundleFailed.Derive(dbErr, "failed to retrieve transaction metadata")
+			}
+			currentTransactionMetadata.SetBundleHeadHash(headTransactionHash)
+
+			// update value bundle flag
+			if !newBundle.IsValueBundle() && currentTransaction.GetValue() != 0 {
+				newBundle.SetValueBundle(true)
+			}
+
+			// if we are done -> trigger events
+			if currentTransaction.IsTail() {
+				newBundle.SetTransactionHashes(mapTransactionsToTransactionHashes(bundleTransactions))
+
+				Events.BundleSolid.Trigger(newBundle, bundleTransactions)
+
+				return newBundle, nil
+			}
+
+			// try to iterate to next turn
+			if nextTransaction, err := tangle.GetTransaction(currentTransaction.GetTrunkTransactionHash()); err != nil {
+				return nil, ErrProcessBundleFailed.Derive(err, "failed to retrieve trunk while processing bundle")
+			} else {
+				currentTransaction = nextTransaction
+			}
+		}
+	})
+}
+
+func mapTransactionsToTransactionHashes(transactions []*value_transaction.ValueTransaction) (result []ternary.Trytes) {
+	result = make([]ternary.Trytes, len(transactions))
+	for k, v := range transactions {
+		result[k] = v.GetHash()
+	}
+
+	return
+}

plugins/bundleprocessor/bundleprocessor_test.go

+package bundleprocessor
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/iotaledger/goshimmer/packages/events"
+	"github.com/iotaledger/goshimmer/packages/model/bundle"
+	"github.com/iotaledger/goshimmer/packages/model/value_transaction"
+	"github.com/iotaledger/goshimmer/packages/ternary"
+	"github.com/iotaledger/goshimmer/plugins/tangle"
+	"github.com/magiconair/properties/assert"
+)
+
+func TestProcessSolidBundleHead(t *testing.T) {
+	tangle.PLUGIN.InitTest()
+
+	tx := value_transaction.New()
+	tx.SetTail(true)
+	tx.SetValue(3)
+
+	tx1 := value_transaction.New()
+	tx1.SetTrunkTransactionHash(tx.GetHash())
+	tx1.SetHead(true)
+
+	tangle.StoreTransaction(tx)
+	tangle.StoreTransaction(tx1)
+
+	Events.BundleSolid.Attach(events.NewClosure(func(bundle *bundle.Bundle, transactions []*value_transaction.ValueTransaction) {
+		fmt.Println("IT HAPPENED")
+		fmt.Println(bundle.GetHash())
+	}))
+
+	result, err := ProcessSolidBundleHead(tx1)
+	if err != nil {
+		t.Error(err)
+	} else {
+		assert.Equal(t, result.GetHash(), ternary.Trytes("UFWJYEWKMEQDNSQUCUWBGOFRHVBGHVVYEZCLCGRDTRQSMAFALTIPMJEEYFDPMQCNJWLXUWFMBZGHQRO99"), "invalid bundle hash")
+		assert.Equal(t, result.IsValueBundle(), true, "invalid value bundle status")
+	}
+}

plugins/bundleprocessor/events.go

@@ -7,15 +7,13 @@ import (
 )
 
 var Events = pluginEvents{
-	DataBundleReceived:    events.NewEvent(bundleEventCaller),
-	ValueBundleReceived:   events.NewEvent(bundleEventCaller),
-	InvalidBundleReceived: events.NewEvent(bundleEventCaller),
+	BundleSolid:   events.NewEvent(bundleEventCaller),
+	InvalidBundle: events.NewEvent(bundleEventCaller),
 }
 
 type pluginEvents struct {
-	DataBundleReceived    *events.Event
-	ValueBundleReceived   *events.Event
-	InvalidBundleReceived *events.Event
+	BundleSolid   *events.Event
+	InvalidBundle *events.Event
 }
 
 func bundleEventCaller(handler interface{}, params ...interface{}) {

plugins/bundleprocessor/plugin.go

 package bundleprocessor
 
 import (
-	"github.com/iotaledger/goshimmer/packages/errors"
+	"github.com/iotaledger/goshimmer/packages/daemon"
 	"github.com/iotaledger/goshimmer/packages/events"
-	"github.com/iotaledger/goshimmer/packages/model/bundle"
-	"github.com/iotaledger/goshimmer/packages/model/transactionmetadata"
 	"github.com/iotaledger/goshimmer/packages/model/value_transaction"
 	"github.com/iotaledger/goshimmer/packages/node"
+	"github.com/iotaledger/goshimmer/packages/workerpool"
 	"github.com/iotaledger/goshimmer/plugins/tangle"
 	"github.com/iotaledger/iota.go/trinary"
 )
 
-var PLUGIN = node.NewPlugin("Bundle Processor", configure)
+var PLUGIN = node.NewPlugin("Bundle Processor", configure, run)
 
 func configure(plugin *node.Plugin) {
+	workerPool = workerpool.New(func(task workerpool.Task) {
+		if _, err := ProcessSolidBundleHead(task.Param(0).(*value_transaction.ValueTransaction)); err != nil {
+			plugin.LogFailure(err.Error())
+		}
+	}, workerpool.WorkerCount(WORKER_COUNT), workerpool.QueueSize(2*WORKER_COUNT))
+
 	tangle.Events.TransactionSolid.Attach(events.NewClosure(func(tx *value_transaction.ValueTransaction) {
 		if tx.IsHead() {
-			if _, err := ProcessSolidBundleHead(tx); err != nil {
-				plugin.LogFailure(err.Error())
-			}
+			workerPool.Submit(tx)
 		}
 	}))
-}
-
-func ProcessSolidBundleHead(headTransaction *value_transaction.ValueTransaction) (*bundle.Bundle, errors.IdentifiableError) {
-	// only process the bundle if we didn't process it, yet
-	return tangle.GetBundle(headTransaction.GetHash(), func(headTransactionHash trinary.Trytes) (*bundle.Bundle, errors.IdentifiableError) {
-		// abort if bundle syntax is wrong
-		if !headTransaction.IsHead() {
-			return nil, ErrProcessBundleFailed.Derive(errors.New("invalid parameter"), "transaction needs to be head of bundle")
-		}
-
-		// initialize event variables
-		newBundle := bundle.New(headTransactionHash)
-		bundleTransactions := make([]*value_transaction.ValueTransaction, 0)
-
-		// iterate through trunk transactions until we reach the tail
-		currentTransaction := headTransaction
-		for {
-			// abort if we reached a previous head
-			if currentTransaction.IsHead() && currentTransaction != headTransaction {
-				newBundle.SetTransactionHashes(mapTransactionsToTransactionHashes(bundleTransactions))
 
-				Events.InvalidBundleReceived.Trigger(newBundle, bundleTransactions)
+	daemon.Events.Shutdown.Attach(events.NewClosure(func() {
+		plugin.LogInfo("Stopping Bundle Processor ...")
 
-				return nil, ErrProcessBundleFailed.Derive(errors.New("invalid bundle found"), "missing bundle tail")
-			}
-
-			// update bundle transactions
-			bundleTransactions = append(bundleTransactions, currentTransaction)
-
-			// retrieve & update metadata
-			currentTransactionMetadata, dbErr := tangle.GetTransactionMetadata(currentTransaction.GetHash(), transactionmetadata.New)
-			if dbErr != nil {
-				return nil, ErrProcessBundleFailed.Derive(dbErr, "failed to retrieve transaction metadata")
-			}
-			currentTransactionMetadata.SetBundleHeadHash(headTransactionHash)
-
-			// update value bundle flag
-			if !newBundle.IsValueBundle() && currentTransaction.GetValue() != 0 {
-				newBundle.SetValueBundle(true)
-			}
+		workerPool.Stop()
+	}))
+}
 
-			// if we are done -> trigger events
-			if currentTransaction.IsTail() {
-				newBundle.SetTransactionHashes(mapTransactionsToTransactionHashes(bundleTransactions))
+func run(plugin *node.Plugin) {
+	plugin.LogInfo("Starting Bundle Processor ...")
 
-				if newBundle.IsValueBundle() {
-					Events.ValueBundleReceived.Trigger(newBundle, bundleTransactions)
-				} else {
-					Events.DataBundleReceived.Trigger(newBundle, bundleTransactions)
-				}
+	daemon.BackgroundWorker("Bundle Processor", func() {
+		plugin.LogSuccess("Starting Bundle Processor ... done")
 
-				return newBundle, nil
-			}
+		workerPool.Run()
 
-			// try to iterate to next turn
-			if nextTransaction, err := tangle.GetTransaction(currentTransaction.GetTrunkTransactionHash()); err != nil {
-				return nil, ErrProcessBundleFailed.Derive(err, "failed to retrieve trunk while processing bundle")
-			} else {
-				currentTransaction = nextTransaction
-			}
-		}
+		plugin.LogSuccess("Stopping Bundle Processor ... done")
 	})
 }
 
-func mapTransactionsToTransactionHashes(transactions []*value_transaction.ValueTransaction) (result []trinary.Trytes) {
-	result = make([]trinary.Trytes, len(transactions))
-	for k, v := range transactions {
-		result[k] = v.GetHash()
-	}
+var workerPool *workerpool.WorkerPool
 
-	return
-}
+const WORKER_COUNT = 10000

plugins/gossip/neighbors.go

@@ -47,7 +47,7 @@ func runNeighbors(plugin *node.Plugin) {
 }
 
 func manageConnection(plugin *node.Plugin, neighbor *Neighbor) {
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Connection Manager ("+neighbor.Identity.StringIdentifier+")", func() {
 		failedConnectionAttempts := 0
 
 		for _, exists := GetNeighbor(neighbor.Identity.StringIdentifier); exists && failedConnectionAttempts < CONNECTION_MAX_ATTEMPTS; {

plugins/gossip/send_queue.go

@@ -26,7 +26,7 @@ func configureSendQueue(plugin *node.Plugin) {
 func runSendQueue(plugin *node.Plugin) {
 	plugin.LogInfo("Starting Send Queue Dispatcher ...")
 
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Gossip Send Queue Dispatcher", func() {
 		plugin.LogSuccess("Starting Send Queue Dispatcher ... done")
 
 		for {
@@ -54,7 +54,7 @@ func runSendQueue(plugin *node.Plugin) {
 
 	connectedNeighborsMutex.Lock()
 	for _, neighborQueue := range neighborQueues {
-		startNeighborSendQueue(neighborQueue)
+		startNeighborSendQueue(neighborQueue.protocol.Neighbor, neighborQueue)
 	}
 	connectedNeighborsMutex.Unlock()
 }
@@ -104,13 +104,13 @@ func setupEventHandlers(neighbor *Neighbor) {
 		}))
 
 		if daemon.IsRunning() {
-			startNeighborSendQueue(queue)
+			startNeighborSendQueue(neighbor, queue)
 		}
 	}))
 }
 
-func startNeighborSendQueue(neighborQueue *neighborQueue) {
-	daemon.BackgroundWorker(func() {
+func startNeighborSendQueue(neighbor *Neighbor, neighborQueue *neighborQueue) {
+	daemon.BackgroundWorker("Gossip Send Queue ("+neighbor.Identity.StringIdentifier+")", func() {
 		for {
 			select {
 			case <-daemon.ShutdownSignal:

plugins/gossip/server.go

@@ -73,7 +73,7 @@ func configureServer(plugin *node.Plugin) {
 func runServer(plugin *node.Plugin) {
 	plugin.LogInfo("Starting TCP Server (port " + strconv.Itoa(*PORT.Value) + ") ...")
 
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Gossip TCP Server", func() {
 		plugin.LogSuccess("Starting TCP Server (port " + strconv.Itoa(*PORT.Value) + ") ... done")
 
 		TCPServer.Listen(*PORT.Value)

plugins/gracefulshutdown/plugin.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
@@ -31,7 +32,13 @@ var PLUGIN = node.NewPlugin("Graceful Shutdown", func(plugin *node.Plugin) {
 				secondsSinceStart := x.Sub(start).Seconds()
 
 				if secondsSinceStart <= WAIT_TO_KILL_TIME_IN_SECONDS {
-					plugin.LogWarning("Received shutdown request - waiting (max " + strconv.Itoa(WAIT_TO_KILL_TIME_IN_SECONDS-int(secondsSinceStart)) + " seconds) to finish processing ...")
+					processList := ""
+					runningBackgroundWorkers := daemon.GetRunningBackgroundWorkers()
+					if len(runningBackgroundWorkers) >= 1 {
+						processList = "(" + strings.Join(runningBackgroundWorkers, ", ") + ") "
+					}
+
+					plugin.LogWarning("Received shutdown request - waiting (max " + strconv.Itoa(WAIT_TO_KILL_TIME_IN_SECONDS-int(secondsSinceStart)) + " seconds) to finish processing " + processList + "...")
 				} else {
 					plugin.LogFailure("Background processes did not terminate in time! Forcing shutdown ...")
 

plugins/statusscreen-tps/plugin.go

@@ -36,7 +36,7 @@ var PLUGIN = node.NewPlugin("Statusscreen TPS", func(plugin *node.Plugin) {
 		return "TPS", strconv.FormatUint(atomic.LoadUint64(&receivedTps), 10) + " received / " + strconv.FormatUint(atomic.LoadUint64(&solidTps), 10) + " new"
 	})
 }, func(plugin *node.Plugin) {
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Statusscreen TPS Tracker", func() {
 		ticker := time.NewTicker(time.Second)
 
 		for {

plugins/statusscreen/statusscreen.go

@@ -100,7 +100,7 @@ func run(plugin *node.Plugin) {
 		return false
 	})
 
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Statusscreen Refresher", func() {
 		for {
 			select {
 			case <-daemon.ShutdownSignal:
@@ -111,7 +111,7 @@ func run(plugin *node.Plugin) {
 		}
 	})
 
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("Statusscreen App", func() {
 		if err := app.SetRoot(frame, true).SetFocus(frame).Run(); err != nil {
 			panic(err)
 		}

plugins/tangle/plugin.go

@@ -12,11 +12,12 @@ func configure(plugin *node.Plugin) {
 	configureTransactionDatabase(plugin)
 	configureTransactionMetaDataDatabase(plugin)
 	configureApproversDatabase(plugin)
+	configureBundleDatabase(plugin)
 	configureSolidifier(plugin)
 }
 
 func run(plugin *node.Plugin) {
-	// this plugin has no background workers
+	runSolidifier(plugin)
 }
 
 // endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////

plugins/tangle/solidifier.go

@@ -9,37 +9,44 @@ import (
 	"github.com/iotaledger/goshimmer/packages/model/transactionmetadata"
 	"github.com/iotaledger/goshimmer/packages/model/value_transaction"
 	"github.com/iotaledger/goshimmer/packages/node"
+	"github.com/iotaledger/goshimmer/packages/ternary"
+	"github.com/iotaledger/goshimmer/packages/workerpool"
 	"github.com/iotaledger/goshimmer/plugins/gossip"
 	"github.com/iotaledger/iota.go/trinary"
 )
 
 // region plugin module setup //////////////////////////////////////////////////////////////////////////////////////////
 
-//var solidifierChan = make(chan *value_transaction.ValueTransaction, 1000)
-
-const NUMBER_OF_WORKERS = 3000
-
-var tasksChan = make(chan *meta_transaction.MetaTransaction, NUMBER_OF_WORKERS)
+var workerPool *workerpool.WorkerPool
 
 func configureSolidifier(plugin *node.Plugin) {
-	for i := 0; i < NUMBER_OF_WORKERS; i++ {
-		go func() {
-			for {
-				select {
-				case <-daemon.ShutdownSignal:
-					return
-				case rawTransaction := <-tasksChan:
-					processMetaTransaction(plugin, rawTransaction)
-				}
-			}
-		}()
-	}
+	workerPool = workerpool.New(func(task workerpool.Task) {
+		processMetaTransaction(plugin, task.Param(0).(*meta_transaction.MetaTransaction))
+	}, workerpool.WorkerCount(WORKER_COUNT), workerpool.QueueSize(2*WORKER_COUNT))
 
 	gossip.Events.ReceiveTransaction.Attach(events.NewClosure(func(rawTransaction *meta_transaction.MetaTransaction) {
-		tasksChan <- rawTransaction
+		workerPool.Submit(rawTransaction)
+	}))
+
+	daemon.Events.Shutdown.Attach(events.NewClosure(func() {
+		plugin.LogInfo("Stopping Solidifier ...")
+
+		workerPool.Stop()
 	}))
 }
 
+func runSolidifier(plugin *node.Plugin) {
+	plugin.LogInfo("Starting Solidifier ...")
+
+	daemon.BackgroundWorker("Tangle Solidifier", func() {
+		plugin.LogSuccess("Starting Solidifier ... done")
+
+		workerPool.Run()
+
+		plugin.LogSuccess("Stopping Solidifier ... done")
+	})
+}
+
 // endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 // Checks and updates the solid flag of a single transaction.
@@ -115,10 +122,10 @@ func IsSolid(transaction *value_transaction.ValueTransaction) (bool, errors.Iden
 }
 
 func propagateSolidity(transactionHash trinary.Trytes) errors.IdentifiableError {
-	if approvers, err := GetApprovers(transactionHash, approvers.New); err != nil {
+	if transactionApprovers, err := GetApprovers(transactionHash, approvers.New); err != nil {
 		return err
 	} else {
-		for _, approverHash := range approvers.GetHashes() {
+		for _, approverHash := range transactionApprovers.GetHashes() {
 			if approver, err := GetTransaction(approverHash); err != nil {
 				return err
 			} else if approver != nil {
@@ -178,5 +185,6 @@ func processTransaction(plugin *node.Plugin, transaction *value_transaction.Valu
 
 		return
 	}
-	//solidifierChan <- transaction
 }
+
+const WORKER_COUNT = 400

plugins/webapi/plugin.go

@@ -33,7 +33,7 @@ func configure(plugin *node.Plugin) {
 func run(plugin *node.Plugin) {
 	plugin.LogInfo("Starting Web Server ...")
 
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("WebAPI Server", func() {
 		plugin.LogSuccess("Starting Web Server ... done")
 
 		if err := Server.Start(":8080"); err != nil {

plugins/zeromq/plugin.go

@@ -45,7 +45,7 @@ func run(plugin *node.Plugin) {
 
 	plugin.LogInfo("Starting ZeroMQ Publisher (port " + strconv.Itoa(*PORT.Value) + ") ...")
 
-	daemon.BackgroundWorker(func() {
+	daemon.BackgroundWorker("ZeroMQ Publisher", func() {
 		if err := startPublisher(plugin); err != nil {
 			plugin.LogFailure("Stopping ZeroMQ Publisher: " + err.Error())
 		} else {