Merge pull request #408 from iotaledger/feat/drng-integration-test

dRNG integration test

.github/workflows/integration-tests.yml

@@ -15,6 +15,9 @@ jobs:
       - name: Build GoShimmer image
         run: docker build -t iotaledger/goshimmer .
 
+      - name: Pull drand image
+        run: docker pull angelocapossele/drand:latest
+        
       - name: Run integration tests
         run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
 

tools/integration-tests/runTests.sh

@@ -3,6 +3,9 @@
 echo "Build GoShimmer image"
 docker build -t iotaledger/goshimmer ../../.
 
+echo "Pulling drand image"
+docker pull angelocapossele/drand:latest
+
 echo "Run integration tests"
 docker-compose -f tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
 

tools/integration-tests/tester/framework/docker.go

@@ -23,7 +23,7 @@ func newDockerClient() (*client.Client, error) {
 	)
 }
 
-// Wrapper object for a Docker container.
+// DockerContainer is a wrapper object for a Docker container.
 type DockerContainer struct {
 	client *client.Client
 	id     string
@@ -71,7 +71,7 @@ func (d *DockerContainer) CreateGoShimmerEntryNode(name string, seed string) err
 }
 
 // CreateGoShimmerPeer creates a new container with the GoShimmer peer's configuration.
-func (d *DockerContainer) CreateGoShimmerPeer(name string, seed string, entryNodeHost string, entryNodePublicKey string, bootstrap bool) error {
+func (d *DockerContainer) CreateGoShimmerPeer(config GoShimmerConfig) error {
 	// configure GoShimmer container instance
 	containerConfig := &container.Config{
 		Image: "iotaledger/goshimmer",
@@ -81,17 +81,41 @@ func (d *DockerContainer) CreateGoShimmerPeer(name string, seed string, entryNod
 		Cmd: strslice.StrSlice{
 			"--skip-config=true",
 			"--logger.level=debug",
-			fmt.Sprintf("--node.disablePlugins=%s", disabledPluginsPeer),
+			fmt.Sprintf("--node.disablePlugins=%s", config.DisabledPlugins),
 			fmt.Sprintf("--node.enablePlugins=%s", func() string {
-				if bootstrap {
+				if config.Bootstrap {
 					return "Bootstrap"
 				}
 				return ""
 			}()),
 			"--webapi.bindAddress=0.0.0.0:8080",
-			fmt.Sprintf("--autopeering.seed=%s", seed),
-			fmt.Sprintf("--autopeering.entryNodes=%s@%s:14626", entryNodePublicKey, entryNodeHost),
+			fmt.Sprintf("--autopeering.seed=%s", config.Seed),
+			fmt.Sprintf("--autopeering.entryNodes=%s@%s:14626", config.EntryNodePublicKey, config.EntryNodeHost),
+			fmt.Sprintf("--drng.instanceId=%d", config.DRNGInstance),
+			fmt.Sprintf("--drng.threshold=%d", config.DRNGThreshold),
+			fmt.Sprintf("--drng.committeeMembers=%s", config.DRNGCommittee),
+			fmt.Sprintf("--drng.distributedPubKey=%s", config.DRNGDistKey),
+		},
+	}
+
+	return d.CreateContainer(config.Name, containerConfig)
+}
+
+// CreateDrandMember creates a new container with the drand configuration.
+func (d *DockerContainer) CreateDrandMember(name string, goShimmerAPI string, leader bool) error {
+	// configure drand container instance
+	env := []string{}
+	if leader {
+		env = append(env, "LEADER=1")
+	}
+	env = append(env, "GOSHIMMER=http://"+goShimmerAPI)
+	containerConfig := &container.Config{
+		Image: "angelocapossele/drand:latest",
+		ExposedPorts: nat.PortSet{
+			nat.Port("8000/tcp"): {},
 		},
+		Env:        env,
+		Entrypoint: strslice.StrSlice{"/data/client-script.sh"},
 	}
 
 	return d.CreateContainer(name, containerConfig)
@@ -109,13 +133,13 @@ func (d *DockerContainer) CreateContainer(name string, containerConfig *containe
 }
 
 // ConnectToNetwork connects a container to an existent network in the docker host.
-func (d *DockerContainer) ConnectToNetwork(networkId string) error {
-	return d.client.NetworkConnect(context.Background(), networkId, d.id, nil)
+func (d *DockerContainer) ConnectToNetwork(networkID string) error {
+	return d.client.NetworkConnect(context.Background(), networkID, d.id, nil)
 }
 
 // DisconnectFromNetwork disconnects a container from an existent network in the docker host.
-func (d *DockerContainer) DisconnectFromNetwork(networkId string) error {
-	return d.client.NetworkDisconnect(context.Background(), networkId, d.id, true)
+func (d *DockerContainer) DisconnectFromNetwork(networkID string) error {
+	return d.client.NetworkDisconnect(context.Background(), networkID, d.id, true)
 }
 
 // Start sends a request to the docker daemon to start a container.

tools/integration-tests/tester/framework/drand.go

+package framework
+
+import (
+	"fmt"
+
+	"github.com/drand/drand/core"
+)
+
+// Drand represents a drand node (committe member) inside the Docker network
+type Drand struct {
+	// name of the drand instance, Docker container and hostname
+	name string
+
+	// Web API of this drand node
+	*core.Client
+
+	// the DockerContainer that this peer is running in
+	*DockerContainer
+}
+
+// newDrand creates a new instance of Drand with the given information.
+func newDrand(name string, dockerContainer *DockerContainer) *Drand {
+	return &Drand{
+		name:            name,
+		Client:          core.NewGrpcClient(),
+		DockerContainer: dockerContainer,
+	}
+}
+
+func (d *Drand) String() string {
+	return fmt.Sprintf("Drand:{%s}", d.name)
+}

tools/integration-tests/tester/framework/drngnetwork.go

+package framework
+
+import (
+	"encoding/hex"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/docker/docker/client"
+	hive_ed25519 "github.com/iotaledger/hive.go/crypto/ed25519"
+	"github.com/iotaledger/hive.go/identity"
+)
+
+// DRNGNetwork represents a complete drand with GoShimmer network within Docker.
+// Including an entry node, drand members and arbitrary many peers.
+type DRNGNetwork struct {
+	network *Network
+	members []*Drand
+	distKey []byte
+}
+
+// newDRNGNetwork returns a DRNGNetwork instance, creates its underlying Docker network and adds the tester container to the network.
+func newDRNGNetwork(dockerClient *client.Client, name string, tester *DockerContainer) (*DRNGNetwork, error) {
+	network, err := newNetwork(dockerClient, name, tester)
+	if err != nil {
+		return nil, err
+	}
+	return &DRNGNetwork{
+		network: network,
+	}, nil
+}
+
+// CreatePeer creates a new peer/GoShimmer node in the network and returns it.
+func (n *DRNGNetwork) CreatePeer(c GoShimmerConfig, publicKey hive_ed25519.PublicKey) (*Peer, error) {
+	name := n.network.namePrefix(fmt.Sprintf("%s%d", containerNameReplica, len(n.network.peers)))
+
+	config := c
+	config.Name = name
+	config.EntryNodeHost = n.network.namePrefix(containerNameEntryNode)
+	config.EntryNodePublicKey = n.network.entryNodePublicKey()
+	config.DisabledPlugins = disabledPluginsPeer
+
+	// create Docker container
+	container := NewDockerContainer(n.network.dockerClient)
+	err := container.CreateGoShimmerPeer(config)
+	if err != nil {
+		return nil, err
+	}
+	err = container.ConnectToNetwork(n.network.id)
+	if err != nil {
+		return nil, err
+	}
+	err = container.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	peer := newPeer(name, identity.New(publicKey), container)
+	n.network.peers = append(n.network.peers, peer)
+	return peer, nil
+}
+
+// CreateMember creates a new member/drand node in the network and returns it.
+// Passing leader true enables the leadership on the given peer.
+func (n *DRNGNetwork) CreateMember(leader bool) (*Drand, error) {
+	name := n.network.namePrefix(fmt.Sprintf("%s%d", containerNameDrand, len(n.members)))
+
+	// create Docker container
+	container := NewDockerContainer(n.network.dockerClient)
+	err := container.CreateDrandMember(name, fmt.Sprintf("%s:8080", n.network.namePrefix(fmt.Sprintf("%s%d", containerNameReplica, len(n.members)))), leader)
+	if err != nil {
+		return nil, err
+	}
+	err = container.ConnectToNetwork(n.network.id)
+	if err != nil {
+		return nil, err
+	}
+	err = container.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	member := newDrand(name, container)
+	n.members = append(n.members, member)
+	return member, nil
+}
+
+// Shutdown creates logs and removes network and containers.
+// Should always be called when a network is not needed anymore!
+func (n *DRNGNetwork) Shutdown() error {
+	// stop drand members
+	for _, p := range n.members {
+		err := p.Stop()
+		if err != nil {
+			return err
+		}
+	}
+
+	// retrieve logs
+	for _, p := range n.members {
+		logs, err := p.Logs()
+		if err != nil {
+			return err
+		}
+		err = createLogFile(p.name, logs)
+		if err != nil {
+			return err
+		}
+	}
+
+	// remove containers
+	for _, p := range n.members {
+		err := p.Remove()
+		if err != nil {
+			return err
+		}
+	}
+
+	return n.network.Shutdown()
+}
+
+// WaitForDKG waits until all members have concluded the DKG phase.
+func (n *DRNGNetwork) WaitForDKG() error {
+	log.Printf("Waiting for DKG...\n")
+	defer log.Printf("Waiting for DKG... done\n")
+
+	for i := dkgMaxTries; i > 0; i-- {
+		if dkey, err := n.members[0].Client.DistKey(n.members[0].name+":8000", false); err == nil {
+			n.SetDistKey(dkey.Key)
+			log.Printf("DistKey: %v", hex.EncodeToString(n.distKey))
+			return nil
+		}
+
+		log.Println("Not done yet. Try again in 5 seconds...")
+		time.Sleep(5 * time.Second)
+	}
+
+	return fmt.Errorf("DKG not successful")
+}
+
+// SetDistKey sets the distributed key.
+func (n *DRNGNetwork) SetDistKey(key []byte) {
+	n.distKey = key
+}
+
+// Peers returns the list of peers.
+func (n *DRNGNetwork) Peers() []*Peer {
+	return n.network.Peers()
+}

tools/integration-tests/tester/framework/framework.go

@@ -4,11 +4,16 @@
 package framework
 
 import (
+	"crypto/ed25519"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/docker/docker/client"
+	hive_ed25519 "github.com/iotaledger/hive.go/crypto/ed25519"
 )
 
 var (
@@ -84,3 +89,75 @@ func (f *Framework) CreateNetwork(name string, peers int, minimumNeighbors int)
 
 	return network, nil
 }
+
+// CreateDRNGNetwork creates and returns a (Docker) Network that contains drand and `peers` GoShimmer nodes.
+func (f *Framework) CreateDRNGNetwork(name string, members, peers, minimumNeighbors int) (*DRNGNetwork, error) {
+	drng, err := newDRNGNetwork(f.dockerClient, strings.ToLower(name), f.tester)
+	if err != nil {
+		return nil, err
+	}
+
+	err = drng.network.createEntryNode()
+	if err != nil {
+		return nil, err
+	}
+
+	// create members/drand nodes
+	for i := 0; i < members; i++ {
+		leader := i == 0
+		if _, err = drng.CreateMember(leader); err != nil {
+			return nil, err
+		}
+	}
+
+	// wait until containers are fully started
+	time.Sleep(1 * time.Second)
+	err = drng.WaitForDKG()
+	if err != nil {
+		return nil, err
+	}
+
+	// create GoShimmer identities
+	pubKeys := make([]hive_ed25519.PublicKey, peers)
+	privKeys := make([]hive_ed25519.PrivateKey, peers)
+	var drngCommittee string
+
+	for i := 0; i < peers; i++ {
+		pubKeys[i], privKeys[i], err = hive_ed25519.GenerateKey()
+		if err != nil {
+			return nil, err
+		}
+
+		if i < members {
+			if drngCommittee != "" {
+				drngCommittee += fmt.Sprintf(",")
+			}
+			drngCommittee += pubKeys[i].String()
+		}
+	}
+
+	config := GoShimmerConfig{
+		DRNGInstance:  1,
+		DRNGThreshold: 3,
+		DRNGDistKey:   hex.EncodeToString(drng.distKey),
+		DRNGCommittee: drngCommittee,
+	}
+
+	// create peers/GoShimmer nodes
+	for i := 0; i < peers; i++ {
+		config.Bootstrap = i == 0
+		config.Seed = base64.StdEncoding.EncodeToString(ed25519.PrivateKey(privKeys[i].Bytes()).Seed())
+		if _, err = drng.CreatePeer(config, pubKeys[i]); err != nil {
+			return nil, err
+		}
+	}
+
+	// wait until peers are fully started and connected
+	time.Sleep(1 * time.Second)
+	err = drng.network.WaitForAutopeering(minimumNeighbors)
+	if err != nil {
+		return nil, err
+	}
+
+	return drng, nil
+}

tools/integration-tests/tester/framework/network.go

@@ -95,7 +95,14 @@ func (n *Network) CreatePeer(bootstrap bool) (*Peer, error) {
 
 	// create Docker container
 	container := NewDockerContainer(n.dockerClient)
-	err = container.CreateGoShimmerPeer(name, seed, n.namePrefix(containerNameEntryNode), n.entryNodePublicKey(), bootstrap)
+	err = container.CreateGoShimmerPeer(GoShimmerConfig{
+		Name:               name,
+		Seed:               seed,
+		EntryNodeHost:      n.namePrefix(containerNameEntryNode),
+		EntryNodePublicKey: n.entryNodePublicKey(),
+		Bootstrap:          bootstrap,
+		DisabledPlugins:    disabledPluginsPeer,
+	})
 	if err != nil {
 		return nil, err
 	}

tools/integration-tests/tester/framework/parameters.go

@@ -8,6 +8,7 @@ const (
 	containerNameTester    = "/tester"
 	containerNameEntryNode = "entry_node"
 	containerNameReplica   = "replica_"
+	containerNameDrand     = "drand_"
 
 	logsDir = "/tmp/logs/"
 
@@ -15,4 +16,21 @@ const (
 	disabledPluginsPeer      = "portcheck,dashboard,analysis"
 
 	dockerLogsPrefixLen = 8
+
+	dkgMaxTries = 50
 )
+
+// GoShimmerConfig defines the config of a goshimmer node.
+type GoShimmerConfig struct {
+	Seed               string
+	Name               string
+	EntryNodeHost      string
+	EntryNodePublicKey string
+	Bootstrap          bool
+	DisabledPlugins    string
+
+	DRNGCommittee string
+	DRNGDistKey   string
+	DRNGInstance  int
+	DRNGThreshold int
+}

tools/integration-tests/tester/framework/peer.go

@@ -32,7 +32,7 @@ func newPeer(name string, identity *identity.Identity, dockerContainer *DockerCo
 	return &Peer{
 		name:            name,
 		Identity:        identity,
-		GoShimmerAPI:    client.NewGoShimmerAPI(getWebApiBaseUrl(name), http.Client{Timeout: 30 * time.Second}),
+		GoShimmerAPI:    client.NewGoShimmerAPI(getWebAPIBaseURL(name), http.Client{Timeout: 30 * time.Second}),
 		DockerContainer: dockerContainer,
 	}
 }

tools/integration-tests/tester/framework/util.go

@@ -7,8 +7,8 @@ import (
 	"os"
 )
 
-// getWebApiBaseUrl returns the web API base url for the given IP.
-func getWebApiBaseUrl(hostname string) string {
+// getWebAPIBaseURL returns the web API base url for the given IP.
+func getWebAPIBaseURL(hostname string) string {
 	return fmt.Sprintf("http://%s:%s", hostname, apiPort)
 }
 
@@ -25,7 +25,16 @@ func createLogFile(name string, logs io.ReadCloser) error {
 	// remove non-ascii chars at beginning of line
 	scanner := bufio.NewScanner(logs)
 	for scanner.Scan() {
-		bytes := append(scanner.Bytes()[dockerLogsPrefixLen:], '\n')
+		line := scanner.Bytes()
+
+		// in case of an error there is no Docker prefix
+		var bytes []byte
+		if len(line) < dockerLogsPrefixLen {
+			bytes = append(line, '\n')
+		} else {
+			bytes = append(line[dockerLogsPrefixLen:], '\n')
+		}
+
 		_, err = f.Write(bytes)
 		if err != nil {
 			return err

tools/integration-tests/tester/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/drand/drand v0.8.1
 	github.com/iotaledger/goshimmer v0.1.3
 	github.com/iotaledger/hive.go v0.0.0-20200507170830-a7c8444003b7
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect

tools/integration-tests/tester/tests/drng_test.go

+package tests
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/iotaledger/goshimmer/tools/integration-tests/tester/framework"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	errWrongRound = fmt.Errorf("wrong round")
+)
+
+// TestDRNG checks whether drng messages are actually relayed/gossiped through the network
+// by checking the messages' existence on all nodes after a cool down.
+func TestDRNG(t *testing.T) {
+	var wg sync.WaitGroup
+
+	drng, err := f.CreateDRNGNetwork("TestDRNG", 5, 8, 3)
+	require.NoError(t, err)
+	defer drng.Shutdown()
+
+	// wait for randomness generation to be started
+	log.Printf("Waiting for randomness generation to be started...\n")
+
+	// randomness starts at round = 2
+	firstRound := uint64(2)
+	_, err = waitForRound(t, drng.Peers()[0], firstRound, 200)
+	require.NoError(t, err)
+
+	log.Printf("Waiting for randomness generation to be started... done\n")
+
+	ticker := time.NewTimer(0)
+	defer ticker.Stop()
+
+	numChecks := 3
+	i := 0
+	for {
+		select {
+		case <-ticker.C:
+			ticker.Reset(10 * time.Second)
+
+			// check for randomness on every peer
+			for _, peer := range drng.Peers() {
+				wg.Add(1)
+				go func(peer *framework.Peer) {
+					defer wg.Done()
+					s, err := waitForRound(t, peer, firstRound+uint64(i), 8)
+					require.NoError(t, err, peer.ID().String(), s)
+					t.Log(peer.ID().String(), s)
+				}(peer)
+			}
+
+			wg.Wait()
+			i++
+
+			if i == numChecks {
+				return
+			}
+		}
+	}
+}
+
+func waitForRound(t *testing.T, peer *framework.Peer, round uint64, maxAttempts int) (string, error) {
+	var b []byte
+	for i := 0; i < maxAttempts; i++ {
+		resp, err := peer.GetRandomness()
+		require.NoError(t, err)
+		b, _ = json.MarshalIndent(resp, "", " ")
+		if resp.Round == round {
+			return string(b), nil
+		}
+		time.Sleep(1 * time.Second)
+	}
+	return string(b), errWrongRound
+}