fix race condition in e2e tests (#184)

In our CI environment there is an artifical delay in between starting
the Rekor services via docker-compose and when the E2E tests are
actually executed due to Go modules being downloaded. In a local
development environment, the download may not be required so the tests
can start before the docker-compose services are actually running.

This introduces a healthcheck for services (where possible), and blocks
the start of the e2e tests until the services are reporting as healthy.
It also forces the use of an empty homedir and rekor config file to
ensure no collision between the tests and the developer's environment.

Fixes #183

Signed-off-by: Bob Callaway <bcallawa@redhat.com>

docker-compose.yml

@@ -8,6 +8,12 @@ services:
       - MYSQL_USER=test
       - MYSQL_PASSWORD=zaphod
     restart: always # keep the MySQL server running
+    healthcheck:
+      test: ["CMD", "/etc/init.d/mysql", "status"]
+      interval: 30s
+      timeout: 3s
+      retries: 3
+      start_period: 10s
   redis-server:
     image: docker.io/redis:5.0.10
     command: [
@@ -19,6 +25,12 @@ services:
     ports:
       - "6379:6379"
     restart: always # keep the redis server running
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 5s
   trillian-log-server:
     image: gcr.io/trillian-opensource-ci/log_server
     command: [
@@ -72,3 +84,9 @@ services:
       - mysql
       - redis-server
       - trillian-log-server
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 5s

tests/e2e-test.sh

 #!/bin/bash
-set -ex
+#set -ex
 testdir=$(dirname "$0")
 
+echo "starting services"
 docker-compose up -d
 
+echo "building CLI"
 go build -o rekor-cli ./cmd/cli
 
-go test -tags=e2e ./tests/
+count=0
+
+echo -n "waiting up to 60 sec for system to start"
+until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
+do
+    if [ $count -eq 6 ]; then
+       echo "! timeout reached"
+       exit 1
+    else
+       echo -n "."
+       sleep 10
+       let 'count+=1'
+    fi
+done
+
+echo
+echo "running tests"
+TMPDIR="$(mktemp -d -t rekor_test)"
+touch $TMPDIR.rekor.yaml
+trap "rm -rf $TMPDIR" EXIT
+TMPDIR=$TMPDIR go test -tags=e2e ./tests/
\ No newline at end of file

tests/util.go

@@ -6,6 +6,7 @@ import (
 	"encoding/base64"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"os/exec"
 	"strings"
 	"testing"
@@ -30,6 +31,10 @@ func run(t *testing.T, stdin, cmd string, arg ...string) string {
 	if stdin != "" {
 		c.Stdin = strings.NewReader(stdin)
 	}
+	if os.Getenv("TMPDIR") != "" {
+		// ensure that we use a clean state.json file for each run
+		c.Env = append(c.Env, "HOME="+os.Getenv("TMPDIR"))
+	}
 	b, err := c.CombinedOutput()
 	if err != nil {
 		t.Log(string(b))
@@ -41,6 +46,10 @@ func run(t *testing.T, stdin, cmd string, arg ...string) string {
 func runCli(t *testing.T, arg ...string) string {
 	t.Helper()
 	arg = append(arg, "--rekor_server=http://localhost:3000")
+	// use a blank config file to ensure no collision
+	if os.Getenv("TMPDIR") != "" {
+		arg = append(arg, "--config="+os.Getenv("TMPDIR")+".rekor.yaml")
+	}
 	return run(t, "", cli, arg...)
 }