diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f096d4eed5d035c466954af8d2c00bd43f98e5d5..3e36c388fe2358b881eb3207be5c260cc31d28a6 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,6 +15,9 @@ jobs: - name: Build GoShimmer image run: docker build -t iotaledger/goshimmer . + - name: Pull drand image + run: docker pull angelocapossele/drand:latest + - name: Run integration tests run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build diff --git a/tools/integration-tests/runTests.sh b/tools/integration-tests/runTests.sh index 9f885f01f363c656b2dc6bebdeb4186392c989b3..bb0674097775df377d9b9aa36ef94c03870a5983 100755 --- a/tools/integration-tests/runTests.sh +++ b/tools/integration-tests/runTests.sh @@ -3,6 +3,9 @@ echo "Build GoShimmer image" docker build -t iotaledger/goshimmer ../../. +echo "Pulling drand image" +docker pull angelocapossele/drand:latest + echo "Run integration tests" docker-compose -f tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build diff --git a/tools/integration-tests/tester/framework/docker.go b/tools/integration-tests/tester/framework/docker.go index b00dc3993e55c344cd2ba728c6ff77b2dcdbd2ac..42b7ddec0eb17b4b0467de832d646170a1a26cbf 100644 --- a/tools/integration-tests/tester/framework/docker.go +++ b/tools/integration-tests/tester/framework/docker.go @@ -23,7 +23,7 @@ func newDockerClient() (*client.Client, error) { ) } -// Wrapper object for a Docker container. +// DockerContainer is a wrapper object for a Docker container. type DockerContainer struct { client *client.Client id string @@ -71,7 +71,7 @@ func (d *DockerContainer) CreateGoShimmerEntryNode(name string, seed string) err } // CreateGoShimmerPeer creates a new container with the GoShimmer peer's configuration. -func (d *DockerContainer) CreateGoShimmerPeer(name string, seed string, entryNodeHost string, entryNodePublicKey string, bootstrap bool) error { +func (d *DockerContainer) CreateGoShimmerPeer(config GoShimmerConfig) error { // configure GoShimmer container instance containerConfig := &container.Config{ Image: "iotaledger/goshimmer", @@ -81,17 +81,41 @@ func (d *DockerContainer) CreateGoShimmerPeer(name string, seed string, entryNod Cmd: strslice.StrSlice{ "--skip-config=true", "--logger.level=debug", - fmt.Sprintf("--node.disablePlugins=%s", disabledPluginsPeer), + fmt.Sprintf("--node.disablePlugins=%s", config.DisabledPlugins), fmt.Sprintf("--node.enablePlugins=%s", func() string { - if bootstrap { + if config.Bootstrap { return "Bootstrap" } return "" }()), "--webapi.bindAddress=0.0.0.0:8080", - fmt.Sprintf("--autopeering.seed=%s", seed), - fmt.Sprintf("--autopeering.entryNodes=%s@%s:14626", entryNodePublicKey, entryNodeHost), + fmt.Sprintf("--autopeering.seed=%s", config.Seed), + fmt.Sprintf("--autopeering.entryNodes=%s@%s:14626", config.EntryNodePublicKey, config.EntryNodeHost), + fmt.Sprintf("--drng.instanceId=%d", config.DRNGInstance), + fmt.Sprintf("--drng.threshold=%d", config.DRNGThreshold), + fmt.Sprintf("--drng.committeeMembers=%s", config.DRNGCommittee), + fmt.Sprintf("--drng.distributedPubKey=%s", config.DRNGDistKey), + }, + } + + return d.CreateContainer(config.Name, containerConfig) +} + +// CreateDrandMember creates a new container with the drand configuration. +func (d *DockerContainer) CreateDrandMember(name string, goShimmerAPI string, leader bool) error { + // configure drand container instance + env := []string{} + if leader { + env = append(env, "LEADER=1") + } + env = append(env, "GOSHIMMER=http://"+goShimmerAPI) + containerConfig := &container.Config{ + Image: "angelocapossele/drand:latest", + ExposedPorts: nat.PortSet{ + nat.Port("8000/tcp"): {}, }, + Env: env, + Entrypoint: strslice.StrSlice{"/data/client-script.sh"}, } return d.CreateContainer(name, containerConfig) @@ -109,13 +133,13 @@ func (d *DockerContainer) CreateContainer(name string, containerConfig *containe } // ConnectToNetwork connects a container to an existent network in the docker host. -func (d *DockerContainer) ConnectToNetwork(networkId string) error { - return d.client.NetworkConnect(context.Background(), networkId, d.id, nil) +func (d *DockerContainer) ConnectToNetwork(networkID string) error { + return d.client.NetworkConnect(context.Background(), networkID, d.id, nil) } // DisconnectFromNetwork disconnects a container from an existent network in the docker host. -func (d *DockerContainer) DisconnectFromNetwork(networkId string) error { - return d.client.NetworkDisconnect(context.Background(), networkId, d.id, true) +func (d *DockerContainer) DisconnectFromNetwork(networkID string) error { + return d.client.NetworkDisconnect(context.Background(), networkID, d.id, true) } // Start sends a request to the docker daemon to start a container. diff --git a/tools/integration-tests/tester/framework/drand.go b/tools/integration-tests/tester/framework/drand.go new file mode 100644 index 0000000000000000000000000000000000000000..67277a3971609707002935c94d9c1a78c20ca1d9 --- /dev/null +++ b/tools/integration-tests/tester/framework/drand.go @@ -0,0 +1,32 @@ +package framework + +import ( + "fmt" + + "github.com/drand/drand/core" +) + +// Drand represents a drand node (committe member) inside the Docker network +type Drand struct { + // name of the drand instance, Docker container and hostname + name string + + // Web API of this drand node + *core.Client + + // the DockerContainer that this peer is running in + *DockerContainer +} + +// newDrand creates a new instance of Drand with the given information. +func newDrand(name string, dockerContainer *DockerContainer) *Drand { + return &Drand{ + name: name, + Client: core.NewGrpcClient(), + DockerContainer: dockerContainer, + } +} + +func (d *Drand) String() string { + return fmt.Sprintf("Drand:{%s}", d.name) +} diff --git a/tools/integration-tests/tester/framework/drngnetwork.go b/tools/integration-tests/tester/framework/drngnetwork.go new file mode 100644 index 0000000000000000000000000000000000000000..80da857995fa1cc325900ba9746915c096cf7430 --- /dev/null +++ b/tools/integration-tests/tester/framework/drngnetwork.go @@ -0,0 +1,149 @@ +package framework + +import ( + "encoding/hex" + "fmt" + "log" + "time" + + "github.com/docker/docker/client" + hive_ed25519 "github.com/iotaledger/hive.go/crypto/ed25519" + "github.com/iotaledger/hive.go/identity" +) + +// DRNGNetwork represents a complete drand with GoShimmer network within Docker. +// Including an entry node, drand members and arbitrary many peers. +type DRNGNetwork struct { + network *Network + members []*Drand + distKey []byte +} + +// newDRNGNetwork returns a DRNGNetwork instance, creates its underlying Docker network and adds the tester container to the network. +func newDRNGNetwork(dockerClient *client.Client, name string, tester *DockerContainer) (*DRNGNetwork, error) { + network, err := newNetwork(dockerClient, name, tester) + if err != nil { + return nil, err + } + return &DRNGNetwork{ + network: network, + }, nil +} + +// CreatePeer creates a new peer/GoShimmer node in the network and returns it. +func (n *DRNGNetwork) CreatePeer(c GoShimmerConfig, publicKey hive_ed25519.PublicKey) (*Peer, error) { + name := n.network.namePrefix(fmt.Sprintf("%s%d", containerNameReplica, len(n.network.peers))) + + config := c + config.Name = name + config.EntryNodeHost = n.network.namePrefix(containerNameEntryNode) + config.EntryNodePublicKey = n.network.entryNodePublicKey() + config.DisabledPlugins = disabledPluginsPeer + + // create Docker container + container := NewDockerContainer(n.network.dockerClient) + err := container.CreateGoShimmerPeer(config) + if err != nil { + return nil, err + } + err = container.ConnectToNetwork(n.network.id) + if err != nil { + return nil, err + } + err = container.Start() + if err != nil { + return nil, err + } + + peer := newPeer(name, identity.New(publicKey), container) + n.network.peers = append(n.network.peers, peer) + return peer, nil +} + +// CreateMember creates a new member/drand node in the network and returns it. +// Passing leader true enables the leadership on the given peer. +func (n *DRNGNetwork) CreateMember(leader bool) (*Drand, error) { + name := n.network.namePrefix(fmt.Sprintf("%s%d", containerNameDrand, len(n.members))) + + // create Docker container + container := NewDockerContainer(n.network.dockerClient) + err := container.CreateDrandMember(name, fmt.Sprintf("%s:8080", n.network.namePrefix(fmt.Sprintf("%s%d", containerNameReplica, len(n.members)))), leader) + if err != nil { + return nil, err + } + err = container.ConnectToNetwork(n.network.id) + if err != nil { + return nil, err + } + err = container.Start() + if err != nil { + return nil, err + } + + member := newDrand(name, container) + n.members = append(n.members, member) + return member, nil +} + +// Shutdown creates logs and removes network and containers. +// Should always be called when a network is not needed anymore! +func (n *DRNGNetwork) Shutdown() error { + // stop drand members + for _, p := range n.members { + err := p.Stop() + if err != nil { + return err + } + } + + // retrieve logs + for _, p := range n.members { + logs, err := p.Logs() + if err != nil { + return err + } + err = createLogFile(p.name, logs) + if err != nil { + return err + } + } + + // remove containers + for _, p := range n.members { + err := p.Remove() + if err != nil { + return err + } + } + + return n.network.Shutdown() +} + +// WaitForDKG waits until all members have concluded the DKG phase. +func (n *DRNGNetwork) WaitForDKG() error { + log.Printf("Waiting for DKG...\n") + defer log.Printf("Waiting for DKG... done\n") + + for i := dkgMaxTries; i > 0; i-- { + if dkey, err := n.members[0].Client.DistKey(n.members[0].name+":8000", false); err == nil { + n.SetDistKey(dkey.Key) + log.Printf("DistKey: %v", hex.EncodeToString(n.distKey)) + return nil + } + + log.Println("Not done yet. Try again in 5 seconds...") + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("DKG not successful") +} + +// SetDistKey sets the distributed key. +func (n *DRNGNetwork) SetDistKey(key []byte) { + n.distKey = key +} + +// Peers returns the list of peers. +func (n *DRNGNetwork) Peers() []*Peer { + return n.network.Peers() +} diff --git a/tools/integration-tests/tester/framework/framework.go b/tools/integration-tests/tester/framework/framework.go index 0770c3a5f868d961a61d79f6ce160be97755053f..75383ec28027a4d1b72f2abe367fa3857c1ce0b6 100644 --- a/tools/integration-tests/tester/framework/framework.go +++ b/tools/integration-tests/tester/framework/framework.go @@ -4,11 +4,16 @@ package framework import ( + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "fmt" "strings" "sync" "time" "github.com/docker/docker/client" + hive_ed25519 "github.com/iotaledger/hive.go/crypto/ed25519" ) var ( @@ -84,3 +89,75 @@ func (f *Framework) CreateNetwork(name string, peers int, minimumNeighbors int) return network, nil } + +// CreateDRNGNetwork creates and returns a (Docker) Network that contains drand and `peers` GoShimmer nodes. +func (f *Framework) CreateDRNGNetwork(name string, members, peers, minimumNeighbors int) (*DRNGNetwork, error) { + drng, err := newDRNGNetwork(f.dockerClient, strings.ToLower(name), f.tester) + if err != nil { + return nil, err + } + + err = drng.network.createEntryNode() + if err != nil { + return nil, err + } + + // create members/drand nodes + for i := 0; i < members; i++ { + leader := i == 0 + if _, err = drng.CreateMember(leader); err != nil { + return nil, err + } + } + + // wait until containers are fully started + time.Sleep(1 * time.Second) + err = drng.WaitForDKG() + if err != nil { + return nil, err + } + + // create GoShimmer identities + pubKeys := make([]hive_ed25519.PublicKey, peers) + privKeys := make([]hive_ed25519.PrivateKey, peers) + var drngCommittee string + + for i := 0; i < peers; i++ { + pubKeys[i], privKeys[i], err = hive_ed25519.GenerateKey() + if err != nil { + return nil, err + } + + if i < members { + if drngCommittee != "" { + drngCommittee += fmt.Sprintf(",") + } + drngCommittee += pubKeys[i].String() + } + } + + config := GoShimmerConfig{ + DRNGInstance: 1, + DRNGThreshold: 3, + DRNGDistKey: hex.EncodeToString(drng.distKey), + DRNGCommittee: drngCommittee, + } + + // create peers/GoShimmer nodes + for i := 0; i < peers; i++ { + config.Bootstrap = i == 0 + config.Seed = base64.StdEncoding.EncodeToString(ed25519.PrivateKey(privKeys[i].Bytes()).Seed()) + if _, err = drng.CreatePeer(config, pubKeys[i]); err != nil { + return nil, err + } + } + + // wait until peers are fully started and connected + time.Sleep(1 * time.Second) + err = drng.network.WaitForAutopeering(minimumNeighbors) + if err != nil { + return nil, err + } + + return drng, nil +} diff --git a/tools/integration-tests/tester/framework/network.go b/tools/integration-tests/tester/framework/network.go index 0a0c875f81d4517f2428bac922c7ce3e9608c42a..d818829340dd84cc3be2f2686a9e665cf9e429ea 100644 --- a/tools/integration-tests/tester/framework/network.go +++ b/tools/integration-tests/tester/framework/network.go @@ -95,7 +95,14 @@ func (n *Network) CreatePeer(bootstrap bool) (*Peer, error) { // create Docker container container := NewDockerContainer(n.dockerClient) - err = container.CreateGoShimmerPeer(name, seed, n.namePrefix(containerNameEntryNode), n.entryNodePublicKey(), bootstrap) + err = container.CreateGoShimmerPeer(GoShimmerConfig{ + Name: name, + Seed: seed, + EntryNodeHost: n.namePrefix(containerNameEntryNode), + EntryNodePublicKey: n.entryNodePublicKey(), + Bootstrap: bootstrap, + DisabledPlugins: disabledPluginsPeer, + }) if err != nil { return nil, err } diff --git a/tools/integration-tests/tester/framework/parameters.go b/tools/integration-tests/tester/framework/parameters.go index 04fad4a6efdff6392355267e1b561384b6f609c2..a0b082ce9a5d3bde47df2f94da4d6c6064fd5360 100644 --- a/tools/integration-tests/tester/framework/parameters.go +++ b/tools/integration-tests/tester/framework/parameters.go @@ -8,6 +8,7 @@ const ( containerNameTester = "/tester" containerNameEntryNode = "entry_node" containerNameReplica = "replica_" + containerNameDrand = "drand_" logsDir = "/tmp/logs/" @@ -15,4 +16,21 @@ const ( disabledPluginsPeer = "portcheck,dashboard,analysis" dockerLogsPrefixLen = 8 + + dkgMaxTries = 50 ) + +// GoShimmerConfig defines the config of a goshimmer node. +type GoShimmerConfig struct { + Seed string + Name string + EntryNodeHost string + EntryNodePublicKey string + Bootstrap bool + DisabledPlugins string + + DRNGCommittee string + DRNGDistKey string + DRNGInstance int + DRNGThreshold int +} diff --git a/tools/integration-tests/tester/framework/peer.go b/tools/integration-tests/tester/framework/peer.go index cd8adf6f03612099f4b3fadc016280673ecd5467..2ad98d4125599a3b87848a41c1d4ab8d08b6f806 100644 --- a/tools/integration-tests/tester/framework/peer.go +++ b/tools/integration-tests/tester/framework/peer.go @@ -32,7 +32,7 @@ func newPeer(name string, identity *identity.Identity, dockerContainer *DockerCo return &Peer{ name: name, Identity: identity, - GoShimmerAPI: client.NewGoShimmerAPI(getWebApiBaseUrl(name), http.Client{Timeout: 30 * time.Second}), + GoShimmerAPI: client.NewGoShimmerAPI(getWebAPIBaseURL(name), http.Client{Timeout: 30 * time.Second}), DockerContainer: dockerContainer, } } diff --git a/tools/integration-tests/tester/framework/util.go b/tools/integration-tests/tester/framework/util.go index 98b5ced90269669d5ee0dd7730cd8b3113e2f6a0..09a7805dc1798a5d641d9d530473f511c5c05137 100644 --- a/tools/integration-tests/tester/framework/util.go +++ b/tools/integration-tests/tester/framework/util.go @@ -7,8 +7,8 @@ import ( "os" ) -// getWebApiBaseUrl returns the web API base url for the given IP. -func getWebApiBaseUrl(hostname string) string { +// getWebAPIBaseURL returns the web API base url for the given IP. +func getWebAPIBaseURL(hostname string) string { return fmt.Sprintf("http://%s:%s", hostname, apiPort) } @@ -25,7 +25,16 @@ func createLogFile(name string, logs io.ReadCloser) error { // remove non-ascii chars at beginning of line scanner := bufio.NewScanner(logs) for scanner.Scan() { - bytes := append(scanner.Bytes()[dockerLogsPrefixLen:], '\n') + line := scanner.Bytes() + + // in case of an error there is no Docker prefix + var bytes []byte + if len(line) < dockerLogsPrefixLen { + bytes = append(line, '\n') + } else { + bytes = append(line[dockerLogsPrefixLen:], '\n') + } + _, err = f.Write(bytes) if err != nil { return err diff --git a/tools/integration-tests/tester/go.mod b/tools/integration-tests/tester/go.mod index fe16ed3afa3e204ed8bc6002614ff295a302b4ab..a6b87aa8694a8159187ac3bfc343a15d6fbe5d73 100644 --- a/tools/integration-tests/tester/go.mod +++ b/tools/integration-tests/tester/go.mod @@ -8,6 +8,7 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect + github.com/drand/drand v0.8.1 github.com/iotaledger/goshimmer v0.1.3 github.com/iotaledger/hive.go v0.0.0-20200507170830-a7c8444003b7 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/tools/integration-tests/tester/tests/drng_test.go b/tools/integration-tests/tester/tests/drng_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0832f04ae9553522cd09097c5964ae2c6ab4ff3d --- /dev/null +++ b/tools/integration-tests/tester/tests/drng_test.go @@ -0,0 +1,81 @@ +package tests + +import ( + "encoding/json" + "fmt" + "log" + "sync" + "testing" + "time" + + "github.com/iotaledger/goshimmer/tools/integration-tests/tester/framework" + "github.com/stretchr/testify/require" +) + +var ( + errWrongRound = fmt.Errorf("wrong round") +) + +// TestDRNG checks whether drng messages are actually relayed/gossiped through the network +// by checking the messages' existence on all nodes after a cool down. +func TestDRNG(t *testing.T) { + var wg sync.WaitGroup + + drng, err := f.CreateDRNGNetwork("TestDRNG", 5, 8, 3) + require.NoError(t, err) + defer drng.Shutdown() + + // wait for randomness generation to be started + log.Printf("Waiting for randomness generation to be started...\n") + + // randomness starts at round = 2 + firstRound := uint64(2) + _, err = waitForRound(t, drng.Peers()[0], firstRound, 200) + require.NoError(t, err) + + log.Printf("Waiting for randomness generation to be started... done\n") + + ticker := time.NewTimer(0) + defer ticker.Stop() + + numChecks := 3 + i := 0 + for { + select { + case <-ticker.C: + ticker.Reset(10 * time.Second) + + // check for randomness on every peer + for _, peer := range drng.Peers() { + wg.Add(1) + go func(peer *framework.Peer) { + defer wg.Done() + s, err := waitForRound(t, peer, firstRound+uint64(i), 8) + require.NoError(t, err, peer.ID().String(), s) + t.Log(peer.ID().String(), s) + }(peer) + } + + wg.Wait() + i++ + + if i == numChecks { + return + } + } + } +} + +func waitForRound(t *testing.T, peer *framework.Peer, round uint64, maxAttempts int) (string, error) { + var b []byte + for i := 0; i < maxAttempts; i++ { + resp, err := peer.GetRandomness() + require.NoError(t, err) + b, _ = json.MarshalIndent(resp, "", " ") + if resp.Round == round { + return string(b), nil + } + time.Sleep(1 * time.Second) + } + return string(b), errWrongRound +}