From b6e91ea4e8565dd8243f8270eb5bcadbdeeb70bf Mon Sep 17 00:00:00 2001
From: Bob Callaway <bobcallaway@users.noreply.github.com>
Date: Wed, 3 Mar 2021 14:01:40 -0500
Subject: [PATCH] fix race condition in e2e tests (#184)
In our CI environment there is an artifical delay in between starting
the Rekor services via docker-compose and when the E2E tests are
actually executed due to Go modules being downloaded. In a local
development environment, the download may not be required so the tests
can start before the docker-compose services are actually running.
This introduces a healthcheck for services (where possible), and blocks
the start of the e2e tests until the services are reporting as healthy.
It also forces the use of an empty homedir and rekor config file to
ensure no collision between the tests and the developer's environment.
Fixes #183
Signed-off-by: Bob Callaway <bcallawa@redhat.com>
---
docker-compose.yml | 18 ++++++++++++++++++
tests/e2e-test.sh | 26 ++++++++++++++++++++++++--
tests/util.go | 9 +++++++++
3 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 3a5a1dd..be6962b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,6 +8,12 @@ services:
- MYSQL_USER=test
- MYSQL_PASSWORD=zaphod
restart: always # keep the MySQL server running
+ healthcheck:
+ test: ["CMD", "/etc/init.d/mysql", "status"]
+ interval: 30s
+ timeout: 3s
+ retries: 3
+ start_period: 10s
redis-server:
image: docker.io/redis:5.0.10
command: [
@@ -19,6 +25,12 @@ services:
ports:
- "6379:6379"
restart: always # keep the redis server running
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 10s
+ timeout: 3s
+ retries: 3
+ start_period: 5s
trillian-log-server:
image: gcr.io/trillian-opensource-ci/log_server
command: [
@@ -72,3 +84,9 @@ services:
- mysql
- redis-server
- trillian-log-server
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:3000/ping"]
+ interval: 10s
+ timeout: 3s
+ retries: 3
+ start_period: 5s
diff --git a/tests/e2e-test.sh b/tests/e2e-test.sh
index f5d211f..13d8a02 100755
--- a/tests/e2e-test.sh
+++ b/tests/e2e-test.sh
@@ -1,9 +1,31 @@
#!/bin/bash
-set -ex
+#set -ex
testdir=$(dirname "$0")
+echo "starting services"
docker-compose up -d
+echo "building CLI"
go build -o rekor-cli ./cmd/cli
-go test -tags=e2e ./tests/
+count=0
+
+echo -n "waiting up to 60 sec for system to start"
+until [ $(docker-compose ps | grep -c "(healthy)") == 3 ];
+do
+ if [ $count -eq 6 ]; then
+ echo "! timeout reached"
+ exit 1
+ else
+ echo -n "."
+ sleep 10
+ let 'count+=1'
+ fi
+done
+
+echo
+echo "running tests"
+TMPDIR="$(mktemp -d -t rekor_test)"
+touch $TMPDIR.rekor.yaml
+trap "rm -rf $TMPDIR" EXIT
+TMPDIR=$TMPDIR go test -tags=e2e ./tests/
\ No newline at end of file
diff --git a/tests/util.go b/tests/util.go
index 84668de..6491f49 100644
--- a/tests/util.go
+++ b/tests/util.go
@@ -6,6 +6,7 @@ import (
"encoding/base64"
"io/ioutil"
"math/rand"
+ "os"
"os/exec"
"strings"
"testing"
@@ -30,6 +31,10 @@ func run(t *testing.T, stdin, cmd string, arg ...string) string {
if stdin != "" {
c.Stdin = strings.NewReader(stdin)
}
+ if os.Getenv("TMPDIR") != "" {
+ // ensure that we use a clean state.json file for each run
+ c.Env = append(c.Env, "HOME="+os.Getenv("TMPDIR"))
+ }
b, err := c.CombinedOutput()
if err != nil {
t.Log(string(b))
@@ -41,6 +46,10 @@ func run(t *testing.T, stdin, cmd string, arg ...string) string {
func runCli(t *testing.T, arg ...string) string {
t.Helper()
arg = append(arg, "--rekor_server=http://localhost:3000")
+ // use a blank config file to ensure no collision
+ if os.Getenv("TMPDIR") != "" {
+ arg = append(arg, "--config="+os.Getenv("TMPDIR")+".rekor.yaml")
+ }
return run(t, "", cli, arg...)
}
--
GitLab