Integration tests (#312)

* Add docker compose for running integration tests.
Runs entry_node and arbitrary number of peers in docker network

* Fix permission denied in container if run without mounting a `rw` volume making it possible to run as throw-away container.
Remove `VOLUME` from Dockerfile as this only pollutes host system with anonymous volumes.

* Use named network for easier external use

* Add test container that discovers peers and waits for autopeering to happen

* Fix min waitForNeighbors

* Add go.sum

* Run integration tests with Github Actions

* Added framework that abstracts the docker network and provides convenience functionality

* Update directory in Github Actions

* Add bash script for automated local test execution

* Add getMessageByHash endpoint

* Adjust to merge changes

* Add methods to easily do HTTP POST requests

* Added relay message test

* Increase client timeout

* Verbose output for tests makes it easier to follow the execution

* Introduce small API wrapper for GoShimmer HTTP API

* Adjust relay test to use new API wrapper

* WIP: Docker logs

* Fix issue with serving visualizer analysis server of entry node

* Persist logs of containers after CI run

* Fix test file

* Fix uploading of artifacts

* Save all containers' logs as artifacts

* Create logs files also with local run

* Add possibility to retrieve logs from a peer via Docker logs

* Make tester part of the goshimmer module to make code sharing possible

* Use client/lib to make HTTP requests in tester

* Fix unit test directory

* Add comments/doc to the code

* Add readme

* Move tester to own module and don't build container but use existing golang one instead

* Address PR comments

* Adjust to merge

* Only use 1 config file for all containers

* go mod tidy

* Rename client lib base url

.github/workflows/integration-tests.yml

+name: Test GoShimmer
+on:
+  push:
+  pull_request:
+    types: [opened, reopened]
+jobs:
+
+  integration-test:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Build GoShimmer Docker network
+        run: docker-compose -f tools/integration-tests/docker-compose.yml up -d --scale peer_replica=5 --build
+
+      - name: Dispay containers
+        run: docker ps -a
+
+      - name: Run integration tests
+        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+
+      - name: Stop GoShimmer Docker network
+        if: always()
+        run: docker-compose -f tools/integration-tests/docker-compose.yml stop
+
+      - name: Create logs from containers in network
+        if: always()
+        run: |
+          docker logs entry_node > tools/integration-tests/logs/entry_node.log
+          docker logs peer_master > tools/integration-tests/logs/peer_master.log
+          docker logs integration-tests_peer_replica_1 > tools/integration-tests/logs/peer_replica_1.log
+          docker logs integration-tests_peer_replica_2 > tools/integration-tests/logs/peer_replica_2.log
+          docker logs integration-tests_peer_replica_3 > tools/integration-tests/logs/peer_replica_3.log
+          docker logs integration-tests_peer_replica_4 > tools/integration-tests/logs/peer_replica_4.log
+          docker logs integration-tests_peer_replica_5 > tools/integration-tests/logs/peer_replica_5.log
+          docker logs tester > tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: container-logs
+          path: tools/integration-tests/logs
+
+      - name: Clean up
+        if: always()
+        run: docker-compose -f tools/integration-tests/docker-compose.yml down

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
 jobs:
 
   build:
-    name: Test GoShimmer
+    name: Unit tests
     runs-on: ubuntu-latest
     steps:
 

Dockerfile

@@ -34,8 +34,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
 # user:group is nonroot:nonroot, uid:gid = 65532:65532
 FROM gcr.io/distroless/static@sha256:23aa732bba4c8618c0d97c26a72a32997363d591807b0d4c31b0bbc8a774bddf
 
-VOLUME /mainnetdb
-
 EXPOSE 14666/tcp
 EXPOSE 14626/udp
 
@@ -44,4 +42,4 @@ COPY --from=build /go/bin/goshimmer /run/goshimmer
 # Copy the default config
 COPY config.default.json /config.json
 
-ENTRYPOINT ["/run/goshimmer", "--config-dir=/", "--database.directory=/mainnetdb"]
+ENTRYPOINT ["/run/goshimmer", "--config-dir=/", "--database.directory=/tmp/mainnetdb"]

client/autopeering.go

@@ -8,7 +8,7 @@ import (
 )
 
 const (
-	routeGetNeighbors = "getNeighbors"
+	routeGetNeighbors = "autopeering/neighbors"
 )
 
 // GetNeighbors gets the chosen/accepted neighbors.

client/lib.go

@@ -24,17 +24,17 @@ const (
 	contentTypeJSON = "application/json"
 )
 
-func NewGoShimmerAPI(node string, httpClient ...http.Client) *GoShimmerAPI {
+func NewGoShimmerAPI(baseUrl string, httpClient ...http.Client) *GoShimmerAPI {
 	if len(httpClient) > 0 {
-		return &GoShimmerAPI{node: node, httpClient: httpClient[0]}
+		return &GoShimmerAPI{baseUrl: baseUrl, httpClient: httpClient[0]}
 	}
-	return &GoShimmerAPI{node: node}
+	return &GoShimmerAPI{baseUrl: baseUrl}
 }
 
 // GoShimmerAPI is an API wrapper over the web API of GoShimmer.
 type GoShimmerAPI struct {
 	httpClient http.Client
-	node       string
+	baseUrl    string
 	jwt        string
 }
 
@@ -62,7 +62,7 @@ func interpretBody(res *http.Response, decodeTo interface{}) error {
 	case http.StatusInternalServerError:
 		return fmt.Errorf("%w: %s", ErrInternalServerError, errRes.Error)
 	case http.StatusNotFound:
-		return fmt.Errorf("%w: %s", ErrNotFound, errRes.Error)
+		return fmt.Errorf("%w: %s", ErrNotFound, res.Request.URL.String())
 	case http.StatusBadRequest:
 		return fmt.Errorf("%w: %s", ErrBadRequest, errRes.Error)
 	case http.StatusUnauthorized:
@@ -86,7 +86,7 @@ func (api *GoShimmerAPI) do(method string, route string, reqObj interface{}, res
 	}
 
 	// construct request
-	req, err := http.NewRequest(method, fmt.Sprintf("%s/%s", api.node, route), func() io.Reader {
+	req, err := http.NewRequest(method, fmt.Sprintf("%s/%s", api.baseUrl, route), func() io.Reader {
 		if data == nil {
 			return nil
 		}
@@ -121,3 +121,7 @@ func (api *GoShimmerAPI) do(method string, route string, reqObj interface{}, res
 	}
 	return nil
 }
+
+func (api *GoShimmerAPI) BaseUrl() string {
+	return api.baseUrl
+}

client/message.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	routeFindById = "findById"
+	routeFindById = "message/findById"
 )
 
 // FindMessageById finds messages by the given ids. The messages are returned in the same order as

docker-compose.yml

@@ -14,7 +14,7 @@ services:
     # make sure to give read/write access to the folder ./mainnetdb (e.g., chmod -R 777 ./mainnetdb)
     # optionally, you can mount a config.json into the container
     volumes:
-      - ./mainnetdb/:/mainnetdb/:rw
+      - ./mainnetdb/:/tmp/mainnetdb/:rw
       #- ./config.json:/config.json:ro
     # Expose ports:
     # gossip:       - "14666:14666/tcp"

plugins/analysis/webinterface/httpserver/static/js/main.js

-const ANALYSIS_SERVER_URL = "116.202.49.178" + "/datastream";
+const ANALYSIS_SERVER_URL = window.location.host + "/datastream";
 const NODE_ID_LENGTH = 64;
 
 // for some strange reason color formats for edges and nodes need to be different... careful!

plugins/webapi/getMessageByHash/plugin.go

+package getMessageByHash
+
+import (
+	"net/http"
+
+	"github.com/iotaledger/goshimmer/packages/binary/messagelayer/message"
+	"github.com/iotaledger/goshimmer/plugins/messagelayer"
+	"github.com/labstack/echo"
+
+	"github.com/iotaledger/goshimmer/plugins/webapi"
+
+	"github.com/iotaledger/hive.go/logger"
+	"github.com/iotaledger/hive.go/node"
+)
+
+var PLUGIN = node.NewPlugin("WebAPI getMessageByHash Endpoint", node.Enabled, configure)
+var log *logger.Logger
+
+func configure(plugin *node.Plugin) {
+	log = logger.NewLogger("API-getMessageByHash")
+	webapi.Server.POST("getMessageByHash", getMessageByHash)
+}
+
+// getMessageByHash returns the array of messages for the
+// given message hashes (in the same order as the parameters).
+// If a node doesn't have the message for a given message hash in its ledger,
+// the value at the index of that message hash is empty.
+func getMessageByHash(c echo.Context) error {
+	var request Request
+	if err := c.Bind(&request); err != nil {
+		log.Info(err.Error())
+		return c.JSON(http.StatusBadRequest, Response{Error: err.Error()})
+	}
+
+	var result []Message
+	for _, hash := range request.Hashes {
+		log.Info("Received:", hash)
+
+		msgId, err := message.NewId(hash)
+		if err != nil {
+			log.Info(err)
+			continue
+		}
+
+		msgObject := messagelayer.Tangle.Message(msgId)
+		if !msgObject.Exists() {
+			continue
+		}
+
+		msg := msgObject.Unwrap()
+		msgResp := Message{
+			MessageId:           msg.Id().String(),
+			TrunkTransactionId:  msg.TrunkId().String(),
+			BranchTransactionId: msg.BranchId().String(),
+			IssuerPublicKey:     msg.IssuerPublicKey().String(),
+			IssuingTime:         msg.IssuingTime().String(),
+			SequenceNumber:      msg.SequenceNumber(),
+			Payload:             msg.Payload().String(),
+			Signature:           msg.Signature().String(),
+		}
+		result = append(result, msgResp)
+		msgObject.Release()
+	}
+
+	return c.JSON(http.StatusOK, Response{Messages: result})
+}
+
+type Response struct {
+	Messages []Message `json:"messages,omitempty"`
+	Error    string    `json:"error,omitempty"`
+}
+
+type Request struct {
+	Hashes []string `json:"hashes"`
+}
+
+type Message struct {
+	MessageId           string `json:"messageId,omitempty"`
+	TrunkTransactionId  string `json:"trunkTransactionId,omitempty"`
+	BranchTransactionId string `json:"branchTransactionId,omitempty"`
+	IssuerPublicKey     string `json:"issuerPublicKey,omitempty"`
+	IssuingTime         string `json:"issuingTime,omitempty"`
+	SequenceNumber      uint64 `json:"sequenceNumber,omitempty"`
+	Payload             string `json:"payload,omitempty"`
+	Signature           string `json:"signature,omitempty"`
+}

tools/integration-tests/README.md

+# Integration tests with Docker
+
+![Integration testing setup](../../images/integration-testing-setup.png)
+
+Running the integration tests spins up a GoShimmer network within Docker as schematically shown in the figure above.
+`N` defines the number of `peer_replicas` and can be specified when running the network.
+The peers can communicate freely within the Docker network and this is exactly how the tests are run using the `tester` container.
+Test can be written in regular Go style while the framework provides convenience functions to access a specific peer's web API or logs.
+
+The autopeering network visualizer, `master_peer's` dashboard and web API are reachable from the host system on the respective ports.
+
+The different containers (`entry_node`, `peer_master`, `peer_replica`) load separate config files that can be modified as necessary, respectively. 
+
+## How to run
+Prerequisites: 
+- Docker
+- Docker compose
+
+```
+# Mac & Linux
+./runTests.sh
+```
+The tests produce `*.log` files for every peer in the `logs` folder after every run.
+
+Currently, the integration tests are configured to run on every push to GitHub with `peer_replica=5`. 
+The logs of every peer are stored as artifacts and can be downloaded for closer inspection once the job finishes.
+
+## Creating tests
+Tests can be written in regular Go style. Each tested component should reside in its own test file in `tester/tests`.
+`main_test` with its `TestMain` function is executed before any test in the package and initializes the integration test framework.
+
+
+## Use as development tool
+Using a standalone throwaway Docker network can be really helpful as a development tool as well. 
+
+Reachable from the host system
+- visualizer: http://localhost:9000
+- `master_peer's` dashboard: http: http://localhost:8081
+- `master_peer's` web API: http: http://localhost:8080
+
+It is therefore possible to send messages to the local network via the `master_peer` and observe log messages either 
+via `docker logs --follow CONTAINER` or by starting the Docker network without the `-d` option, as follows.
+
+```
+docker-compose -f docker-compose.yml up --scale peer_replica=5
+
+# 1. test manually with master_peer
+
+# 2. or run in separate terminal window
+docker-compose -f tester/docker-compose.yml up --exit-code-from tester
+```
+Sometimes when changing files, either in the tests or in GoShimmer, Docker does not detect the changes on a rebuild. 
+Then the option `--build` needs to be used with `docker-compose`.
\ No newline at end of file

tools/integration-tests/config.docker.json

+{
+  "analysis": {
+    "client": {
+      "serverAddress": "entry_node:1888"
+    },
+    "server": {
+      "port": 0
+    },
+    "httpServer": {
+      "bindAddress": "0.0.0.0:9000",
+      "dev": false
+    }
+  },
+  "autopeering": {
+    "entryNodes": [
+      "X2cmCzYnZDjmsvdAH90Q7oKmhNeTdwJdj2FX84adLzo=@entry_node:14626"
+    ],
+    "port": 14626,
+    "version": 1
+  },
+  "dashboard": {
+    "bindAddress": "0.0.0.0:8081",
+    "dev": false,
+    "basic_auth": {
+      "enabled": false,
+      "username": "goshimmer",
+      "password": "goshimmer"
+    }
+  },
+  "database": {
+    "directory": "mainnetdb"
+  },
+  "gossip": {
+    "port": 14666
+  },
+  "graph": {
+    "bindAddress": "127.0.0.1:8082",
+    "domain": "",
+    "networkName": "GoShimmer",
+    "socketIOPath": "socket.io-client/dist/socket.io.js",
+    "webrootPath": "IOTAtangle/webroot"
+  },
+  "logger": {
+    "level": "info",
+    "disableCaller": false,
+    "disableStacktrace": false,
+    "encoding": "console",
+    "outputPaths": [
+      "stdout",
+      "goshimmer.log"
+    ],
+    "disableEvents": true,
+    "remotelog": {
+      "serverAddress": "remotelog.goshimmer.iota.cafe:5213"
+    }
+  },
+  "network": {
+    "bindAddress": "0.0.0.0",
+    "externalAddress": "auto"
+  },
+  "node": {
+    "disablePlugins": "portcheck",
+    "enablePlugins": []
+  },
+  "webapi": {
+    "auth": {
+      "password": "goshimmer",
+      "privateKey": "",
+      "username": "goshimmer"
+    },
+    "bindAddress": "0.0.0.0:8080"
+  }
+}
\ No newline at end of file

tools/integration-tests/docker-compose.yml

+version: "3.5"
+
+services:
+  entry_node:
+    command: --autopeering.seed=uuDCzsjyLNQ17/7fWKPNMYmr4IWuaVRf7qKqRL0v/6c= --autopeering.entryNodes= --analysis.server.port=1888 --node.disablePlugins=gossip,portcheck,spa,webapi,webapibroadcastdataendpoint,webapifindtransactionhashesendpoint,webapigetneighborsendpoint,webapigettransactionobjectsbyhashendpoint,webapigettransactiontrytesbyhashendpoint
+    container_name: entry_node
+    image: iotaledger/goshimmer
+    build:
+      context: ../..
+    volumes:
+      - ./config.docker.json:/config.json:ro
+    ports:
+      - "9000:9000/tcp" # visualizer
+    expose:
+      - "1888/tcp" # analysis server (within Docker network)
+    networks:
+      - integration-test
+
+  peer_master:
+    container_name: peer_master
+    image: iotaledger/goshimmer
+    build:
+      context: ../..
+    volumes:
+      - ./config.docker.json:/config.json:ro
+    ports:
+      - "8080:8080/tcp" # web API
+      - "8081:8081/tcp" # dashboard
+    depends_on:
+      - entry_node
+    networks:
+      - integration-test
+
+  peer_replica:
+    command: --node.disablePlugins=spa
+    image: iotaledger/goshimmer
+    build:
+      context: ../..
+    volumes:
+      - ./config.docker.json:/config.json:ro
+    expose:
+      - "8080/tcp" # web API (within Docker network)
+    depends_on:
+      - entry_node
+    networks:
+      - integration-test
+
+networks:
+  integration-test:
+    driver: bridge
+    name: integration-test

tools/integration-tests/runTests.sh

+#!/bin/bash
+
+if [ -z "$1" ]; then
+    echo "Usage: `basename $0` number_of_peer_replicas"
+    exit 0
+fi
+
+REPLICAS=$1
+
+echo "Build GoShimmer Docker network"
+docker-compose -f docker-compose.yml up -d --scale peer_replica=$REPLICAS --build
+if [ $? -ne 0 ]; then { echo "Failed, aborting." ; exit 1; } fi
+
+echo "Dispay containers"
+docker ps -a
+
+echo "Run integration tests"
+docker-compose -f tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+
+echo "Create logs from containers in network"
+docker-compose -f docker-compose.yml stop
+docker logs tester > logs/tester.log
+docker logs entry_node > logs/entry_node.log
+docker logs peer_master > logs/peer_master.log
+for (( c=1; c<=$REPLICAS; c++ ))
+do
+docker logs integration-tests_peer_replica_$c > logs/peer_replica_$c.log
+done
+
+echo "Clean up"
+docker-compose -f tester/docker-compose.yml down
+docker-compose -f docker-compose.yml down

tools/integration-tests/tester/docker-compose.yml

+version: "3.5"
+
+services:
+  tester:
+    container_name: tester
+    image: golang:1.14
+    working_dir: /go/src/github.com/iotaledger/goshimmer/tools/integration-tests/tester
+    entrypoint: go test ./tests -v -mod=readonly
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - ../../..:/go/src/github.com/iotaledger/goshimmer:ro
+    networks:
+      - integration-test
+
+networks:
+  integration-test:
+    external: true

tools/integration-tests/tester/framework/framework.go

+// Package framework provides integration test functionality for GoShimmer with a Docker network.
+// It effectively abstracts away all complexity with discovering peers,
+// waiting for them to autopeer and offers easy access to the peers' web API
+// and logs via Docker.
+package framework
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/docker/docker/client"
+)
+
+// Framework is a wrapper encapsulating all peers
+type Framework struct {
+	peers     []*Peer
+	dockerCli *client.Client
+}
+
+// New creates a new instance of Framework, gets all available peers within the Docker network and
+// waits for them to autopeer.
+// Panics if no peer is found.
+func New() *Framework {
+	fmt.Printf("Finding available peers...\n")
+
+	cli, err := client.NewClient(
+		"unix:///var/run/docker.sock",
+		"",
+		nil,
+		nil,
+	)
+	if err != nil {
+		fmt.Println("Could not create docker CLI client.")
+		panic(err)
+	}
+
+	f := &Framework{
+		dockerCli: cli,
+		peers:     getAvailablePeers(cli),
+	}
+
+	if len(f.peers) == 0 {
+		panic("Could not find any peers in Docker network.")
+	}
+	fmt.Printf("Finding available peers... done. Peers: %v\n", f.peers)
+
+	fmt.Printf("Waiting for autopeering...\n")
+	f.waitForAutopeering()
+	fmt.Printf("Waiting for autopeering... done\n")
+
+	return f
+}
+
+// waitForAutopeering waits until all peers have reached a minimum amount of neighbors.
+// Panics if this minimum is not reached after autopeeringMaxTries.
+func (f *Framework) waitForAutopeering() {
+	maxTries := autopeeringMaxTries
+	for maxTries > 0 {
+
+		for _, p := range f.peers {
+			if resp, err := p.GetNeighbors(false); err != nil {
+				fmt.Printf("request error: %v\n", err)
+			} else {
+				p.SetNeighbors(resp.Chosen, resp.Accepted)
+			}
+		}
+
+		// verify neighbor requirement
+		min := 100
+		total := 0
+		for _, p := range f.peers {
+			neighbors := p.TotalNeighbors()
+			if neighbors < min {
+				min = neighbors
+			}
+			total += neighbors
+		}
+		if min >= autopeeringMinimumNeighbors {
+			fmt.Printf("Neighbors: min=%d avg=%.2f\n", min, float64(total)/float64(len(f.peers)))
+			return
+		}
+
+		fmt.Println("Not done yet. Try again in 5 seconds...")
+		time.Sleep(5 * time.Second)
+		maxTries--
+	}
+	panic("Peering not successful.")
+}
+
+// Peers returns all available peers.
+func (f *Framework) Peers() []*Peer {
+	return f.peers
+}
+
+// RandomPeer returns a random peer out of the list of peers.
+func (f *Framework) RandomPeer() *Peer {
+	return f.peers[rand.Intn(len(f.peers))]
+}

tools/integration-tests/tester/framework/parameters.go

+package framework
+
+const (
+	hostnamePeerMaster        = "peer_master"
+	hostnamePeerReplicaPrefix = "integration-tests_peer_replica_"
+
+	autopeeringMaxTries         = 50
+	autopeeringMinimumNeighbors = 2
+
+	apiPort = "8080"
+)

tools/integration-tests/tester/framework/peer.go

+package framework
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	dockerclient "github.com/docker/docker/client"
+
+	"github.com/iotaledger/goshimmer/client"
+	"github.com/iotaledger/goshimmer/plugins/webapi/autopeering"
+)
+
+// Peer represents a GoShimmer node inside the Docker network
+type Peer struct {
+	name string
+	ip   net.IP
+	*client.GoShimmerAPI
+	dockerCli *dockerclient.Client
+	chosen    []autopeering.Neighbor
+	accepted  []autopeering.Neighbor
+}
+
+// NewPeer creates a new instance of Peer with the given information.
+func NewPeer(name string, ip net.IP, dockerCli *dockerclient.Client) *Peer {
+	return &Peer{
+		name:         name,
+		ip:           ip,
+		GoShimmerAPI: client.NewGoShimmerAPI(getWebApiBaseUrl(ip), http.Client{Timeout: 30 * time.Second}),
+		dockerCli:    dockerCli,
+	}
+}
+
+func (p *Peer) String() string {
+	return fmt.Sprintf("Peer:{%s, %s, %s, %d}", p.name, p.ip.String(), p.BaseUrl(), p.TotalNeighbors())
+}
+
+// TotalNeighbors returns the total number of neighbors the peer has.
+func (p *Peer) TotalNeighbors() int {
+	return len(p.chosen) + len(p.accepted)
+}
+
+// SetNeighbors sets the neighbors of the peer accordingly.
+func (p *Peer) SetNeighbors(chosen, accepted []autopeering.Neighbor) {
+	p.chosen = make([]autopeering.Neighbor, len(chosen))
+	copy(p.chosen, chosen)
+
+	p.accepted = make([]autopeering.Neighbor, len(accepted))
+	copy(p.accepted, accepted)
+}
+
+// Logs returns the logs of the peer as io.ReadCloser.
+// Logs are returned via Docker and contain every log entry since start of the container/GoShimmer node.
+func (p *Peer) Logs() (io.ReadCloser, error) {
+	return p.dockerCli.ContainerLogs(
+		context.Background(),
+		p.name,
+		types.ContainerLogsOptions{
+			ShowStdout: true,
+			ShowStderr: true,
+			Since:      "",
+			Timestamps: false,
+			Follow:     false,
+			Tail:       "",
+			Details:    false,
+		})
+}
+
+// getAvailablePeers gets all available peers in the Docker network.
+// It uses the expected Docker hostnames and tries to resolve them.
+// If that does not work it means the host is not available in the network
+func getAvailablePeers(dockerCli *dockerclient.Client) (peers []*Peer) {
+	// get peer master
+	if addr, err := net.LookupIP(hostnamePeerMaster); err != nil {
+		fmt.Printf("Could not resolve %s\n", hostnamePeerMaster)
+	} else {
+		p := NewPeer(hostnamePeerMaster, addr[0], dockerCli)
+		peers = append(peers, p)
+	}
+
+	// get peer replicas
+	for i := 1; ; i++ {
+		peerName := fmt.Sprintf("%s%d", hostnamePeerReplicaPrefix, i)
+		if addr, err := net.LookupIP(peerName); err != nil {
+			//fmt.Printf("Could not resolve %s\n", peerName)
+			break
+		} else {
+			p := NewPeer(peerName, addr[0], dockerCli)
+			peers = append(peers, p)
+		}
+	}
+	return
+}
+
+// getWebApiBaseUrl returns the web API base url for the given IP.
+func getWebApiBaseUrl(ip net.IP) string {
+	return fmt.Sprintf("http://%s:%s", ip.String(), apiPort)
+}