Consider exit status of containers in integration tests (#387)

* Add status code check to network shutdown so that a test fails if a container stops caused by an error

* Speed up Docker network through little caching trick

tools/integration-tests/tester/framework/docker.go

@@ -160,6 +160,16 @@ func (d *DockerContainer) Stop() error {
 	return d.client.ContainerStop(context.Background(), d.id, &duration)
 }
 
+// ExitStatus returns the exit status according to the container information.
+func (d *DockerContainer) ExitStatus() (int, error) {
+	resp, err := d.client.ContainerInspect(context.Background(), d.id)
+	if err != nil {
+		return -1, err
+	}
+
+	return resp.State.ExitCode, nil
+}
+
 // Logs returns the logs of the container as io.ReadCloser.
 func (d *DockerContainer) Logs() (io.ReadCloser, error) {
 	options := types.ContainerLogsOptions{

tools/integration-tests/tester/framework/network.go

@@ -155,6 +155,19 @@ func (n *Network) Shutdown() error {
 		}
 	}
 
+	// save exit status of containers to check at end of shutdown process
+	exitStatus := make(map[string]int, len(n.peers)+1)
+	exitStatus[containerNameEntryNode], err = n.entryNode.ExitStatus()
+	if err != nil {
+		return err
+	}
+	for _, p := range n.peers {
+		exitStatus[p.name], err = p.ExitStatus()
+		if err != nil {
+			return err
+		}
+	}
+
 	// remove containers
 	err = n.entryNode.Remove()
 	if err != nil {
@@ -179,6 +192,13 @@ func (n *Network) Shutdown() error {
 		return err
 	}
 
+	// check exit codes of containers
+	for name, status := range exitStatus {
+		if status != exitStatusSuccessful {
+			return fmt.Errorf("container %s exited with code %d", name, status)
+		}
+	}
+
 	return nil
 }
 

tools/integration-tests/tester/framework/parameters.go

@@ -18,6 +18,8 @@ const (
 	dockerLogsPrefixLen = 8
 
 	dkgMaxTries = 50
+
+	exitStatusSuccessful = 0
 )
 
 // GoShimmerConfig defines the config of a GoShimmer node.

tools/integration-tests/tester/tests/common_test.go

@@ -19,7 +19,7 @@ func TestSynchronization(t *testing.T) {
 	initalPeers := 4
 	n, err := f.CreateNetwork("common_TestSynchronization", initalPeers, 2, config)
 	require.NoError(t, err)
-	defer n.Shutdown()
+	defer ShutdownNetwork(t, n)
 
 	// wait for peers to change their state to synchronized
 	time.Sleep(5 * time.Second)

tools/integration-tests/tester/tests/drng_test.go

@@ -23,7 +23,7 @@ func TestDRNG(t *testing.T) {
 
 	drng, err := f.CreateDRNGNetwork("TestDRNG", 5, 8, 3)
 	require.NoError(t, err)
-	defer drng.Shutdown()
+	defer ShutdownNetwork(t, drng)
 
 	// wait for randomness generation to be started
 	log.Printf("Waiting for randomness generation to be started...\n")

tools/integration-tests/tester/tests/message_test.go

@@ -11,7 +11,7 @@ import (
 func TestPersistence(t *testing.T) {
 	n, err := f.CreateNetwork("message_TestPersistence", 4, 2)
 	require.NoError(t, err)
-	defer n.Shutdown()
+	defer ShutdownNetwork(t, n)
 
 	// wait for peers to change their state to synchronized
 	time.Sleep(5 * time.Second)

tools/integration-tests/tester/tests/testutil.go

@@ -19,6 +19,10 @@ type DataMessageSent struct {
 	issuerPublicKey string
 }
 
+type Shutdowner interface {
+	Shutdown() error
+}
+
 // sendDataMessagesOnRandomPeer sends data messages on a random peer and saves the sent message to a map.
 func sendDataMessagesOnRandomPeer(t *testing.T, peers []*framework.Peer, numMessages int, idsMap ...map[string]DataMessageSent) map[string]DataMessageSent {
 	var ids map[string]DataMessageSent
@@ -90,3 +94,9 @@ func checkForMessageIds(t *testing.T, peers []*framework.Peer, ids map[string]Da
 		}
 	}
 }
+
+// ShutdownNetwork shuts down the network and reports errors.
+func ShutdownNetwork(t *testing.T, n Shutdowner) {
+	err := n.Shutdown()
+	require.NoError(t, err)
+}