Flexible integration tests (#339)

* Introduce concept of networks

* Adjust tests to use flexible Docker networks

* Adjust test execution workflow

* Separately build GoShimmer Docker image

* Generate identities to create containers from seeds

* Move Docker network functionality

* Add comments and docs

* Update readme

* Update readme

* Address PR comments

* Use log instead of fmt

* Introduce proper error handling

* Use named return for error

.github/workflows/integration-tests.yml

@@ -12,30 +12,16 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Build GoShimmer Docker network
-        run: docker-compose -f tools/integration-tests/docker-compose.yml up -d --scale peer_replica=5 --build
-
-      - name: Dispay containers
-        run: docker ps -a
+      - name: Build GoShimmer image
+        run: docker build -t iotaledger/goshimmer .
 
       - name: Run integration tests
         run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
 
-      - name: Stop GoShimmer Docker network
-        if: always()
-        run: docker-compose -f tools/integration-tests/docker-compose.yml stop
-
-      - name: Create logs from containers in network
+      - name: Create logs from tester
         if: always()
         run: |
-          docker logs entry_node > tools/integration-tests/logs/entry_node.log
-          docker logs peer_master > tools/integration-tests/logs/peer_master.log
-          docker logs integration-tests_peer_replica_1 > tools/integration-tests/logs/peer_replica_1.log
-          docker logs integration-tests_peer_replica_2 > tools/integration-tests/logs/peer_replica_2.log
-          docker logs integration-tests_peer_replica_3 > tools/integration-tests/logs/peer_replica_3.log
-          docker logs integration-tests_peer_replica_4 > tools/integration-tests/logs/peer_replica_4.log
-          docker logs integration-tests_peer_replica_5 > tools/integration-tests/logs/peer_replica_5.log
-          docker logs tester > tools/integration-tests/logs/tester.log
+          docker logs tester &> tools/integration-tests/logs/tester.log
 
       - name: Save logs as artifacts
         if: always()
@@ -46,4 +32,4 @@ jobs:
 
       - name: Clean up
         if: always()
-        run: docker-compose -f tools/integration-tests/docker-compose.yml down
+        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml down

tools/docker-network/README.md

+# GoShimmer Network with Docker
+
+![Docker network](../../images/docker-network.png)
+
+Running `docker-compose` spins up a GoShimmer network within Docker as schematically shown in the figure above.
+`N` defines the number of `peer_replicas` and can be specified when running the network.
+The peers can communicate freely within the Docker network 
+while the autopeering network visualizer, `master_peer's` dashboard and web API are reachable from the host system on the respective ports.
+
+The different containers (`entry_node`, `peer_master`, `peer_replica`) are based on the same config file 
+and separate config file and modified as necessary, respectively. 
+
+## How to use as development tool
+Using a standalone throwaway Docker network can be really helpful as a development tool. 
+
+Prerequisites: 
+- Docker 17.12.0+
+- Docker compose: file format 3.5
+
+Reachable from the host system
+- autopeering visualizer: http://localhost:9000
+- `master_peer's` dashboard: http: http://localhost:8081
+- `master_peer's` web API: http: http://localhost:8080
+
+It is therefore possible to send messages to the local network via the `master_peer` and observe log messages either 
+via `docker logs --follow CONTAINER` or by starting the Docker network without the `-d` option, as follows.
+
+```
+docker-compose up --scale peer_replica=5
+
+# remove containers and network
+docker-compose down
+```
+
+Sometimes when changing files Docker does not detect the changes on a rebuild. 
+Then the option `--build` needs to be used with `docker-compose`.
\ No newline at end of file

tools/integration-tests/docker-compose.yml → tools/docker-network/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     volumes:
       - ./config.docker.json:/config.json:ro
     ports:
-      - "9000:9000/tcp" # visualizer
+      - "127.0.0.1:9000:9000/tcp" # autopeering visualizer
     expose:
       - "1888/tcp" # analysis server (within Docker network)
     networks:
@@ -24,8 +24,8 @@ services:
     volumes:
       - ./config.docker.json:/config.json:ro
     ports:
-      - "8080:8080/tcp" # web API
-      - "8081:8081/tcp" # dashboard
+      - "127.0.0.1:8080:8080/tcp" # web API
+      - "127.0.0.1:8081:8081/tcp" # dashboard
     depends_on:
       - entry_node
     networks:

tools/integration-tests/README.md

 # Integration tests with Docker
 
-![Integration testing setup](../../images/integration-testing-setup.png)
+![Integration testing](../../images/integration-testing.png)
 
-Running the integration tests spins up a GoShimmer network within Docker as schematically shown in the figure above.
-`N` defines the number of `peer_replicas` and can be specified when running the network.
-The peers can communicate freely within the Docker network and this is exactly how the tests are run using the `tester` container.
-Test can be written in regular Go style while the framework provides convenience functions to access a specific peer's web API or logs.
+Running the integration tests spins up a `tester` container within which every test can specify its own GoShimmer network with Docker as schematically shown in the figure above.
 
-The autopeering network visualizer, `master_peer's` dashboard and web API are reachable from the host system on the respective ports.
-
-The different containers (`entry_node`, `peer_master`, `peer_replica`) load separate config files that can be modified as necessary, respectively. 
+Peers can communicate freely within their Docker network and this is exactly how the tests are run using the `tester` container.
+Test can be written in regular Go style while the framework provides convenience functions to create a new network, access a specific peer's web API or logs.
 
 ## How to run
 Prerequisites: 
-- Docker
-- Docker compose
+- Docker 17.12.0+
+- Docker compose: file format 3.5
 
 ```
 # Mac & Linux
 ./runTests.sh
 ```
-The tests produce `*.log` files for every peer in the `logs` folder after every run.
+The tests produce `*.log` files for every networks' peer in the `logs` folder after every run.
 
-Currently, the integration tests are configured to run on every push to GitHub with `peer_replica=5`. 
-The logs of every peer are stored as artifacts and can be downloaded for closer inspection once the job finishes.
+On GitHub logs of every peer are stored as artifacts and can be downloaded for closer inspection once the job finishes.
 
 ## Creating tests
 Tests can be written in regular Go style. Each tested component should reside in its own test file in `tester/tests`.
 `main_test` with its `TestMain` function is executed before any test in the package and initializes the integration test framework.
 
-
-## Use as development tool
-Using a standalone throwaway Docker network can be really helpful as a development tool as well. 
-
-Reachable from the host system
-- visualizer: http://localhost:9000
-- `master_peer's` dashboard: http: http://localhost:8081
-- `master_peer's` web API: http: http://localhost:8080
-
-It is therefore possible to send messages to the local network via the `master_peer` and observe log messages either 
-via `docker logs --follow CONTAINER` or by starting the Docker network without the `-d` option, as follows.
-
+Each test has to specify its network where the tests are run. This can be done via the framework at the beginning of a test.
+```go
+// create a network with name 'testnetwork' with 6 peers and wait until every peer has at least 3 neighbors
+n := f.CreateNetwork("testnetwork", 6, 3)
+// must be called to create log files and properly clean up
+defer n.Shutdown() 
 ```
-docker-compose -f docker-compose.yml up --scale peer_replica=5
-
-# 1. test manually with master_peer
 
-# 2. or run in separate terminal window
-docker-compose -f tester/docker-compose.yml up --exit-code-from tester
-```
-Sometimes when changing files, either in the tests or in GoShimmer, Docker does not detect the changes on a rebuild. 
-Then the option `--build` needs to be used with `docker-compose`.
\ No newline at end of file
+## Other tips
+Useful for development is to only execute the test you're currently building. For that matter, simply modify the `docker-compose.yml` file as follows:
+```yaml
+entrypoint: go test ./tests -run <YOUR_TEST_NAME> -v -mod=readonly
+```
\ No newline at end of file

tools/integration-tests/runTests.sh

 #!/bin/bash
 
-if [ -z "$1" ]; then
-    echo "Usage: `basename $0` number_of_peer_replicas"
-    exit 0
-fi
-
-REPLICAS=$1
-
-echo "Build GoShimmer Docker network"
-docker-compose -f docker-compose.yml up -d --scale peer_replica=$REPLICAS --build
-if [ $? -ne 0 ]; then { echo "Failed, aborting." ; exit 1; } fi
-
-echo "Dispay containers"
-docker ps -a
+echo "Build GoShimmer image"
+docker build -t iotaledger/goshimmer ../../.
 
 echo "Run integration tests"
 docker-compose -f tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
 
 echo "Create logs from containers in network"
-docker-compose -f docker-compose.yml stop
-docker logs tester > logs/tester.log
-docker logs entry_node > logs/entry_node.log
-docker logs peer_master > logs/peer_master.log
-for (( c=1; c<=$REPLICAS; c++ ))
-do
-docker logs integration-tests_peer_replica_$c > logs/peer_replica_$c.log
-done
+docker logs tester &> logs/tester.log
 
 echo "Clean up"
-docker-compose -f tester/docker-compose.yml down
-docker-compose -f docker-compose.yml down
+docker-compose -f tester/docker-compose.yml down
\ No newline at end of file

tools/integration-tests/tester/docker-compose.yml

@@ -9,9 +9,5 @@ services:
     volumes:
     - /var/run/docker.sock:/var/run/docker.sock:ro
     - ../../..:/go/src/github.com/iotaledger/goshimmer:ro
-    networks:
-      - integration-test
+    - ../logs:/tmp/logs
 
-networks:
-  integration-test:
-    external: true

tools/integration-tests/tester/framework/docker.go

+package framework
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
+)
+
+// newDockerClient creates a Docker client that communicates via the Docker socket.
+func newDockerClient() (*client.Client, error) {
+	return client.NewClient(
+		"unix:///var/run/docker.sock",
+		"",
+		nil,
+		nil,
+	)
+}
+
+// Wrapper object for a Docker container.
+type DockerContainer struct {
+	client *client.Client
+	id     string
+}
+
+// NewDockerContainer creates a new DockerContainer.
+func NewDockerContainer(c *client.Client) *DockerContainer {
+	return &DockerContainer{client: c}
+}
+
+// NewDockerContainerFromExisting creates a new DockerContainer from an already existing Docker container by name.
+func NewDockerContainerFromExisting(c *client.Client, name string) (*DockerContainer, error) {
+	containers, err := c.ContainerList(context.Background(), types.ContainerListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, cont := range containers {
+		if cont.Names[0] == name {
+			return &DockerContainer{
+				client: c,
+				id:     cont.ID,
+			}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("could not find container with name '%s'", name)
+}
+
+// CreateGoShimmerEntryNode creates a new container with the GoShimmer entry node's configuration.
+func (d *DockerContainer) CreateGoShimmerEntryNode(name string, seed string) error {
+	containerConfig := &container.Config{
+		Image:        "iotaledger/goshimmer",
+		ExposedPorts: nil,
+		Cmd: strslice.StrSlice{
+			fmt.Sprintf("--node.disablePlugins=%s", disabledPluginsEntryNode),
+			"--autopeering.entryNodes=",
+			fmt.Sprintf("--autopeering.seed=%s", seed),
+		},
+	}
+
+	return d.CreateContainer(name, containerConfig)
+}
+
+// CreateGoShimmerPeer creates a new container with the GoShimmer peer's configuration.
+func (d *DockerContainer) CreateGoShimmerPeer(name string, seed string, entryNodeHost string, entryNodePublicKey string) error {
+	// configure GoShimmer container instance
+	containerConfig := &container.Config{
+		Image: "iotaledger/goshimmer",
+		ExposedPorts: nat.PortSet{
+			nat.Port("8080/tcp"): {},
+		},
+		Cmd: strslice.StrSlice{
+			fmt.Sprintf("--node.disablePlugins=%s", disabledPluginsPeer),
+			"--webapi.bindAddress=0.0.0.0:8080",
+			fmt.Sprintf("--autopeering.seed=%s", seed),
+			fmt.Sprintf("--autopeering.entryNodes=%s@%s:14626", entryNodePublicKey, entryNodeHost),
+		},
+	}
+
+	return d.CreateContainer(name, containerConfig)
+}
+
+// CreateContainer creates a new container with the given configuration.
+func (d *DockerContainer) CreateContainer(name string, containerConfig *container.Config) error {
+	resp, err := d.client.ContainerCreate(context.Background(), containerConfig, nil, nil, name)
+	if err != nil {
+		return err
+	}
+
+	d.id = resp.ID
+	return nil
+}
+
+// ConnectToNetwork connects a container to an existent network in the docker host.
+func (d *DockerContainer) ConnectToNetwork(networkId string) error {
+	return d.client.NetworkConnect(context.Background(), networkId, d.id, nil)
+}
+
+// DisconnectFromNetwork disconnects a container from an existent network in the docker host.
+func (d *DockerContainer) DisconnectFromNetwork(networkId string) error {
+	return d.client.NetworkDisconnect(context.Background(), networkId, d.id, true)
+}
+
+// Start sends a request to the docker daemon to start a container.
+func (d *DockerContainer) Start() error {
+	return d.client.ContainerStart(context.Background(), d.id, types.ContainerStartOptions{})
+}
+
+// Remove kills and removes a container from the docker host.
+func (d *DockerContainer) Remove() error {
+	return d.client.ContainerRemove(context.Background(), d.id, types.ContainerRemoveOptions{Force: true})
+}
+
+// Stop stops a container without terminating the process.
+// The process is blocked until the container stops or the timeout expires.
+func (d *DockerContainer) Stop() error {
+	duration := 10 * time.Second
+	return d.client.ContainerStop(context.Background(), d.id, &duration)
+}
+
+// Logs returns the logs of the container as io.ReadCloser.
+func (d *DockerContainer) Logs() (io.ReadCloser, error) {
+	options := types.ContainerLogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Since:      "",
+		Timestamps: false,
+		Follow:     false,
+		Tail:       "",
+		Details:    false,
+	}
+
+	return d.client.ContainerLogs(context.Background(), d.id, options)
+}

tools/integration-tests/tester/framework/framework.go

 // Package framework provides integration test functionality for GoShimmer with a Docker network.
-// It effectively abstracts away all complexity with discovering peers,
-// waiting for them to autopeer and offers easy access to the peers' web API
-// and logs via Docker.
+// It effectively abstracts away all complexity with creating a custom Docker network per test,
+// discovering peers, waiting for them to autopeer and offers easy access to the peers' web API and logs.
 package framework
 
 import (
-	"fmt"
-	"math/rand"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/docker/docker/client"
 )
 
-// Framework is a wrapper encapsulating all peers
+var (
+	once     sync.Once
+	instance *Framework
+)
+
+// Framework is a wrapper that provides the integration testing functionality.
 type Framework struct {
-	peers     []*Peer
-	dockerCli *client.Client
+	tester       *DockerContainer
+	dockerClient *client.Client
 }
 
-// New creates a new instance of Framework, gets all available peers within the Docker network and
-// waits for them to autopeer.
-// Panics if no peer is found.
-func New() *Framework {
-	fmt.Printf("Finding available peers...\n")
+// Instance returns the singleton Framework instance.
+func Instance() (f *Framework, err error) {
+	once.Do(func() {
+		f, err = newFramework()
+		instance = f
+	})
 
-	cli, err := client.NewClient(
-		"unix:///var/run/docker.sock",
-		"",
-		nil,
-		nil,
-	)
+	return instance, err
+}
+
+// newFramework creates a new instance of Framework, creates a DockerClient
+// and creates a DockerContainer for the tester container where the tests are running in.
+func newFramework() (*Framework, error) {
+	dockerClient, err := newDockerClient()
 	if err != nil {
-		fmt.Println("Could not create docker CLI client.")
-		panic(err)
+		return nil, err
 	}
 
-	f := &Framework{
-		dockerCli: cli,
-		peers:     getAvailablePeers(cli),
+	tester, err := NewDockerContainerFromExisting(dockerClient, containerNameTester)
+	if err != nil {
+		return nil, err
 	}
 
-	if len(f.peers) == 0 {
-		panic("Could not find any peers in Docker network.")
+	f := &Framework{
+		dockerClient: dockerClient,
+		tester:       tester,
 	}
-	fmt.Printf("Finding available peers... done. Peers: %v\n", f.peers)
-
-	fmt.Printf("Waiting for autopeering...\n")
-	f.waitForAutopeering()
-	fmt.Printf("Waiting for autopeering... done\n")
 
-	return f
+	return f, nil
 }
 
-// waitForAutopeering waits until all peers have reached a minimum amount of neighbors.
-// Panics if this minimum is not reached after autopeeringMaxTries.
-func (f *Framework) waitForAutopeering() {
-	maxTries := autopeeringMaxTries
-	for maxTries > 0 {
+// CreateNetwork creates and returns a (Docker) Network that contains `peers` GoShimmer nodes.
+// It waits for the peers to autopeer until the minimum neighbors criteria is met for every peer.
+func (f *Framework) CreateNetwork(name string, peers int, minimumNeighbors int) (*Network, error) {
+	network, err := newNetwork(f.dockerClient, strings.ToLower(name), f.tester)
+	if err != nil {
+		return nil, err
+	}
 
-		for _, p := range f.peers {
-			if resp, err := p.GetNeighbors(false); err != nil {
-				fmt.Printf("request error: %v\n", err)
-			} else {
-				p.SetNeighbors(resp.Chosen, resp.Accepted)
-			}
-		}
+	err = network.createEntryNode()
+	if err != nil {
+		return nil, err
+	}
 
-		// verify neighbor requirement
-		min := 100
-		total := 0
-		for _, p := range f.peers {
-			neighbors := p.TotalNeighbors()
-			if neighbors < min {
-				min = neighbors
-			}
-			total += neighbors
+	// create peers/GoShimmer nodes
+	for i := 0; i < peers; i++ {
+		_, err = network.CreatePeer()
+		if err != nil {
+			return nil, err
 		}
-		if min >= autopeeringMinimumNeighbors {
-			fmt.Printf("Neighbors: min=%d avg=%.2f\n", min, float64(total)/float64(len(f.peers)))
-			return
-		}
-
-		fmt.Println("Not done yet. Try again in 5 seconds...")
-		time.Sleep(5 * time.Second)
-		maxTries--
 	}
-	panic("Peering not successful.")
-}
 
-// Peers returns all available peers.
-func (f *Framework) Peers() []*Peer {
-	return f.peers
-}
+	// wait until containers are fully started
+	time.Sleep(1 * time.Second)
+	err = network.WaitForAutopeering(minimumNeighbors)
+	if err != nil {
+		return nil, err
+	}
 
-// RandomPeer returns a random peer out of the list of peers.
-func (f *Framework) RandomPeer() *Peer {
-	return f.peers[rand.Intn(len(f.peers))]
+	return network, nil
 }

tools/integration-tests/tester/framework/network.go

+package framework
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"math/rand"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	hive_ed25519 "github.com/iotaledger/hive.go/crypto/ed25519"
+	"github.com/iotaledger/hive.go/identity"
+)
+
+// Network represents a complete GoShimmer network within Docker.
+// Including an entry node and arbitrary many peers.
+type Network struct {
+	id   string
+	name string
+
+	peers  []*Peer
+	tester *DockerContainer
+
+	entryNode         *DockerContainer
+	entryNodeIdentity *identity.Identity
+
+	dockerClient *client.Client
+}
+
+// newNetwork returns a Network instance, creates its underlying Docker network and adds the tester container to the network.
+func newNetwork(dockerClient *client.Client, name string, tester *DockerContainer) (*Network, error) {
+	// create Docker network
+	resp, err := dockerClient.NetworkCreate(context.Background(), name, types.NetworkCreate{})
+	if err != nil {
+		return nil, err
+	}
+
+	// the tester container needs to join the Docker network in order to communicate with the peers
+	err = tester.ConnectToNetwork(resp.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Network{
+		id:           resp.ID,
+		name:         name,
+		tester:       tester,
+		dockerClient: dockerClient,
+	}, nil
+}
+
+// createEntryNode creates the network's entry node.
+func (n *Network) createEntryNode() error {
+	// create identity
+	publicKey, privateKey, err := hive_ed25519.GenerateKey()
+	if err != nil {
+		return err
+	}
+
+	n.entryNodeIdentity = identity.New(publicKey)
+	seed := base64.StdEncoding.EncodeToString(ed25519.PrivateKey(privateKey.Bytes()).Seed())
+
+	// create entry node container
+	n.entryNode = NewDockerContainer(n.dockerClient)
+	err = n.entryNode.CreateGoShimmerEntryNode(n.namePrefix(containerNameEntryNode), seed)
+	if err != nil {
+		return err
+	}
+	err = n.entryNode.ConnectToNetwork(n.id)
+	if err != nil {
+		return err
+	}
+	err = n.entryNode.Start()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CreatePeer creates a new peer/GoShimmer node in the network and returns it.
+func (n *Network) CreatePeer() (*Peer, error) {
+	name := n.namePrefix(fmt.Sprintf("%s%d", containerNameReplica, len(n.peers)))
+
+	// create identity
+	publicKey, privateKey, err := hive_ed25519.GenerateKey()
+	if err != nil {
+		return nil, err
+	}
+	seed := base64.StdEncoding.EncodeToString(ed25519.PrivateKey(privateKey.Bytes()).Seed())
+
+	// create Docker container
+	container := NewDockerContainer(n.dockerClient)
+	err = container.CreateGoShimmerPeer(name, seed, n.namePrefix(containerNameEntryNode), n.entryNodePublicKey())
+	if err != nil {
+		return nil, err
+	}
+	err = container.ConnectToNetwork(n.id)
+	if err != nil {
+		return nil, err
+	}
+	err = container.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	peer := newPeer(name, identity.New(publicKey), container)
+	n.peers = append(n.peers, peer)
+	return peer, nil
+}
+
+// Shutdown creates logs and removes network and containers.
+// Should always be called when a network is not needed anymore!
+func (n *Network) Shutdown() error {
+	// stop containers
+	err := n.entryNode.Stop()
+	if err != nil {
+		return err
+	}
+	for _, p := range n.peers {
+		err = p.Stop()
+		if err != nil {
+			return err
+		}
+	}
+
+	// retrieve logs
+	logs, err := n.entryNode.Logs()
+	if err != nil {
+		return err
+	}
+	err = createLogFile(n.namePrefix(containerNameEntryNode), logs)
+	if err != nil {
+		return err
+	}
+	for _, p := range n.peers {
+		logs, err = p.Logs()
+		if err != nil {
+			return err
+		}
+		err = createLogFile(p.name, logs)
+		if err != nil {
+			return err
+		}
+	}
+
+	// remove containers
+	err = n.entryNode.Remove()
+	if err != nil {
+		return err
+	}
+	for _, p := range n.peers {
+		err = p.Remove()
+		if err != nil {
+			return err
+		}
+	}
+
+	// disconnect tester from network otherwise the network can't be removed
+	err = n.tester.DisconnectFromNetwork(n.id)
+	if err != nil {
+		return err
+	}
+
+	// remove network
+	err = n.dockerClient.NetworkRemove(context.Background(), n.id)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// WaitForAutopeering waits until all peers have reached the minimum amount of neighbors.
+// Returns error if this minimum is not reached after autopeeringMaxTries.
+func (n *Network) WaitForAutopeering(minimumNeighbors int) error {
+	log.Printf("Waiting for autopeering...\n")
+	defer log.Printf("Waiting for autopeering... done\n")
+
+	for i := autopeeringMaxTries; i > 0; i-- {
+
+		for _, p := range n.peers {
+			if resp, err := p.GetNeighbors(false); err != nil {
+				log.Printf("request error: %v\n", err)
+			} else {
+				p.SetNeighbors(resp.Chosen, resp.Accepted)
+			}
+		}
+
+		// verify neighbor requirement
+		min := 100
+		total := 0
+		for _, p := range n.peers {
+			neighbors := p.TotalNeighbors()
+			if neighbors < min {
+				min = neighbors
+			}
+			total += neighbors
+		}
+		if min >= minimumNeighbors {
+			log.Printf("Neighbors: min=%d avg=%.2f\n", min, float64(total)/float64(len(n.peers)))
+			return nil
+		}
+
+		log.Println("Not done yet. Try again in 5 seconds...")
+		time.Sleep(5 * time.Second)
+	}
+
+	return fmt.Errorf("autopeering not successful")
+}
+
+// namePrefix returns the suffix prefixed with the name.
+func (n *Network) namePrefix(suffix string) string {
+	return fmt.Sprintf("%s-%s", n.name, suffix)
+}
+
+// entryNodePublicKey returns the entry node's public key encoded as base64
+func (n *Network) entryNodePublicKey() string {
+	return base64.StdEncoding.EncodeToString(n.entryNodeIdentity.PublicKey().Bytes())
+}
+
+// Peers returns all available peers in the network.
+func (n *Network) Peers() []*Peer {
+	return n.peers
+}
+
+// RandomPeer returns a random peer out of the list of peers.
+func (n *Network) RandomPeer() *Peer {
+	return n.peers[rand.Intn(len(n.peers))]
+}

tools/integration-tests/tester/framework/parameters.go

 package framework
 
 const (
-	hostnamePeerMaster        = "peer_master"
-	hostnamePeerReplicaPrefix = "integration-tests_peer_replica_"
-
-	autopeeringMaxTries         = 50
-	autopeeringMinimumNeighbors = 2
+	autopeeringMaxTries = 50
 
 	apiPort = "8080"
+
+	containerNameTester    = "/tester"
+	containerNameEntryNode = "entry_node"
+	containerNameReplica   = "replica_"
+
+	logsDir = "/tmp/logs/"
+
+	disabledPluginsEntryNode = "portcheck,spa,analysis,gossip,webapi,webapibroadcastdataendpoint,webapifindtransactionhashesendpoint,webapigetneighborsendpoint,webapigettransactionobjectsbyhashendpoint,webapigettransactiontrytesbyhashendpoint"
+	disabledPluginsPeer      = "portcheck,spa,analysis"
+
+	dockerLogsPrefixLen = 8
 )

tools/integration-tests/tester/framework/peer.go

 package framework
 
 import (
-	"context"
 	"fmt"
-	"io"
-	"net"
 	"net/http"
 	"time"
 
-	"github.com/docker/docker/api/types"
-	dockerclient "github.com/docker/docker/client"
-
 	"github.com/iotaledger/goshimmer/client"
 	"github.com/iotaledger/goshimmer/plugins/webapi/autopeering"
+	"github.com/iotaledger/hive.go/identity"
 )
 
 // Peer represents a GoShimmer node inside the Docker network
 type Peer struct {
+	// name of the GoShimmer instance, Docker container and hostname
 	name string
-	ip   net.IP
+	// GoShimmer identity
+	identity *identity.Identity
+
+	// Web API of this peer
 	*client.GoShimmerAPI
-	dockerCli *dockerclient.Client
-	chosen    []autopeering.Neighbor
-	accepted  []autopeering.Neighbor
+
+	// the DockerContainer that this peer is running in
+	*DockerContainer
+
+	chosen   []autopeering.Neighbor
+	accepted []autopeering.Neighbor
 }
 
-// NewPeer creates a new instance of Peer with the given information.
-func NewPeer(name string, ip net.IP, dockerCli *dockerclient.Client) *Peer {
+// newPeer creates a new instance of Peer with the given information.
+func newPeer(name string, identity *identity.Identity, dockerContainer *DockerContainer) *Peer {
 	return &Peer{
-		name:         name,
-		ip:           ip,
-		GoShimmerAPI: client.NewGoShimmerAPI(getWebApiBaseUrl(ip), http.Client{Timeout: 30 * time.Second}),
-		dockerCli:    dockerCli,
+		name:            name,
+		identity:        identity,
+		GoShimmerAPI:    client.NewGoShimmerAPI(getWebApiBaseUrl(name), http.Client{Timeout: 30 * time.Second}),
+		DockerContainer: dockerContainer,
 	}
 }
 
 func (p *Peer) String() string {
-	return fmt.Sprintf("Peer:{%s, %s, %s, %d}", p.name, p.ip.String(), p.BaseUrl(), p.TotalNeighbors())
+	return fmt.Sprintf("Peer:{%s, %s, %s, %d}", p.name, p.identity.ID().String(), p.BaseUrl(), p.TotalNeighbors())
 }
 
 // TotalNeighbors returns the total number of neighbors the peer has.
@@ -52,51 +54,3 @@ func (p *Peer) SetNeighbors(chosen, accepted []autopeering.Neighbor) {
 	p.accepted = make([]autopeering.Neighbor, len(accepted))
 	copy(p.accepted, accepted)
 }
-
-// Logs returns the logs of the peer as io.ReadCloser.
-// Logs are returned via Docker and contain every log entry since start of the container/GoShimmer node.
-func (p *Peer) Logs() (io.ReadCloser, error) {
-	return p.dockerCli.ContainerLogs(
-		context.Background(),
-		p.name,
-		types.ContainerLogsOptions{
-			ShowStdout: true,
-			ShowStderr: true,
-			Since:      "",
-			Timestamps: false,
-			Follow:     false,
-			Tail:       "",
-			Details:    false,
-		})
-}
-
-// getAvailablePeers gets all available peers in the Docker network.
-// It uses the expected Docker hostnames and tries to resolve them.
-// If that does not work it means the host is not available in the network
-func getAvailablePeers(dockerCli *dockerclient.Client) (peers []*Peer) {
-	// get peer master
-	if addr, err := net.LookupIP(hostnamePeerMaster); err != nil {
-		fmt.Printf("Could not resolve %s\n", hostnamePeerMaster)
-	} else {
-		p := NewPeer(hostnamePeerMaster, addr[0], dockerCli)
-		peers = append(peers, p)
-	}
-
-	// get peer replicas
-	for i := 1; ; i++ {
-		peerName := fmt.Sprintf("%s%d", hostnamePeerReplicaPrefix, i)
-		if addr, err := net.LookupIP(peerName); err != nil {
-			//fmt.Printf("Could not resolve %s\n", peerName)
-			break
-		} else {
-			p := NewPeer(peerName, addr[0], dockerCli)
-			peers = append(peers, p)
-		}
-	}
-	return
-}
-
-// getWebApiBaseUrl returns the web API base url for the given IP.
-func getWebApiBaseUrl(ip net.IP) string {
-	return fmt.Sprintf("http://%s:%s", ip.String(), apiPort)
-}

tools/integration-tests/tester/framework/util.go

+package framework
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+)
+
+// getWebApiBaseUrl returns the web API base url for the given IP.
+func getWebApiBaseUrl(hostname string) string {
+	return fmt.Sprintf("http://%s:%s", hostname, apiPort)
+}
+
+// createLogFile creates a log file from the given logs ReadCloser.
+func createLogFile(name string, logs io.ReadCloser) error {
+	defer logs.Close()
+
+	f, err := os.Create(fmt.Sprintf("%s%s.log", logsDir, name))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// remove non-ascii chars at beginning of line
+	scanner := bufio.NewScanner(logs)
+	for scanner.Scan() {
+		bytes := append(scanner.Bytes()[dockerLogsPrefixLen:], '\n')
+		_, err = f.Write(bytes)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = f.Sync()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

tools/integration-tests/tester/go.mod

@@ -6,9 +6,10 @@ require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/iotaledger/goshimmer v0.1.3
+	github.com/iotaledger/hive.go v0.0.0-20200403132600-4c10556e08a0
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/stretchr/testify v1.5.1
 )

tools/integration-tests/tester/tests/dockerlogs_test.go

@@ -12,9 +12,13 @@ import (
 // TestDockerLogs simply verifies that a peer's log message contains "GoShimmer".
 // Using the combination of logs and regular expressions can be useful to test a certain peer's behavior.
 func TestDockerLogs(t *testing.T) {
+	n, err := f.CreateNetwork("TestDockerLogs", 3, 1)
+	require.NoError(t, err)
+	defer n.Shutdown()
+
 	r := regexp.MustCompile("GoShimmer")
 
-	for _, p := range f.Peers() {
+	for _, p := range n.Peers() {
 		log, err := p.Logs()
 		require.NoError(t, err)
 

tools/integration-tests/tester/tests/main_test.go

 // Package tests provides the possibility to write integration tests in regular Go style.
 // The integration test framework is initialized before any test in the package runs and
-// thus can readily be used to make requests to peers and read their logs.
+// thus can readily be used to create networks.
 //
-// Each tested feature should reside in its own test file and define tests cases as necessary.
+// Each tested feature should reside in its own test file and define tests cases and networks as necessary.
 package tests
 
 import (
@@ -17,7 +17,11 @@ var f *framework.Framework
 // TestMain gets called by the test utility and is executed before any other test in this package.
 // It is therefore used to initialize the integration testing framework.
 func TestMain(m *testing.M) {
-	f = framework.New()
+	var err error
+	f, err = framework.Instance()
+	if err != nil {
+		panic(err)
+	}
 
 	// call the tests
 	os.Exit(m.Run())

tools/integration-tests/tester/tests/relaymessage_test.go

@@ -11,24 +11,29 @@ import (
 // TestRelayMessages checks whether messages are actually relayed/gossiped through the network
 // by checking the messages' existence on all nodes after a cool down.
 func TestRelayMessages(t *testing.T) {
-	numMessages := 100
+	n, err := f.CreateNetwork("TestRelayMessages", 6, 3)
+	require.NoError(t, err)
+	defer n.Shutdown()
+
+	numMessages := 105
 	ids := make([]string, numMessages)
 
 	data := []byte("Test")
 
 	// create messages on random peers
 	for i := 0; i < numMessages; i++ {
-		id, err := f.RandomPeer().Data(data)
+		peer := n.RandomPeer()
+		id, err := peer.Data(data)
 		require.NoError(t, err)
 
 		ids[i] = id
 	}
 
 	// wait for messages to be gossiped
-	time.Sleep(5 * time.Second)
+	time.Sleep(10 * time.Second)
 
 	// check for messages on every peer
-	for _, peer := range f.Peers() {
+	for _, peer := range n.Peers() {
 		resp, err := peer.FindMessageById(ids)
 		require.NoError(t, err)