Remove gzip processing flow completely from rekor (#221)

* Remove gzip processing flow completely from rekor

Issue #208 reported different handling of gzipped content via fetch vs
direct upload to rekor server. The code should be consistent, regardless
of whether content was compressed or not - by always attempting to
verify the signature against the (unmodified) byte stream.

This patch removes the gzip decoding completely from rekor and verifies
the bytes supplied or referenced.

Also fixes issue in E2E tests where sending SIGKILL to watch process
caused message to be printed to stderr, which fails the test when
running on MacOS.

Fixes #208

Signed-off-by: Bob Callaway <bcallawa@redhat.com>

pkg/api/index.go

@@ -53,7 +53,7 @@ func SearchIndexHandler(params index.SearchIndexParams) middleware.Responder {
 	}
 	if params.Query.PublicKey != nil {
 		af := pki.NewArtifactFactory(swag.StringValue(params.Query.PublicKey.Format))
-		keyReader, err := util.FileOrURLReadCloser(httpReqCtx, params.Query.PublicKey.URL.String(), params.Query.PublicKey.Content, true)
+		keyReader, err := util.FileOrURLReadCloser(httpReqCtx, params.Query.PublicKey.URL.String(), params.Query.PublicKey.Content)
 		if err != nil {
 			return handleRekorAPIError(params, http.StatusBadRequest, err, malformedPublicKey)
 		}

pkg/types/rekord/v0.0.1/entry.go

@@ -195,7 +195,7 @@ func (v *V001Entry) FetchExternalEntities(ctx context.Context) error {
 		defer hashW.Close()
 		defer sigW.Close()
 
-		dataReadCloser, err := util.FileOrURLReadCloser(ctx, v.RekordObj.Data.URL.String(), v.RekordObj.Data.Content, false)
+		dataReadCloser, err := util.FileOrURLReadCloser(ctx, v.RekordObj.Data.URL.String(), v.RekordObj.Data.Content)
 		if err != nil {
 			return closePipesOnError(err)
 		}
@@ -237,7 +237,7 @@ func (v *V001Entry) FetchExternalEntities(ctx context.Context) error {
 		defer close(sigResult)
 
 		sigReadCloser, err := util.FileOrURLReadCloser(ctx, v.RekordObj.Signature.URL.String(),
-			v.RekordObj.Signature.Content, false)
+			v.RekordObj.Signature.Content)
 		if err != nil {
 			return closePipesOnError(err)
 		}
@@ -262,7 +262,7 @@ func (v *V001Entry) FetchExternalEntities(ctx context.Context) error {
 		defer close(keyResult)
 
 		keyReadCloser, err := util.FileOrURLReadCloser(ctx, v.RekordObj.Signature.PublicKey.URL.String(),
-			v.RekordObj.Signature.PublicKey.Content, false)
+			v.RekordObj.Signature.PublicKey.Content)
 		if err != nil {
 			return closePipesOnError(err)
 		}

pkg/types/rpm/v0.0.1/entry.go

@@ -198,7 +198,7 @@ func (v *V001Entry) FetchExternalEntities(ctx context.Context) error {
 		defer sigW.Close()
 		defer rpmW.Close()
 
-		dataReadCloser, err := util.FileOrURLReadCloser(ctx, v.RPMModel.Package.URL.String(), v.RPMModel.Package.Content, true)
+		dataReadCloser, err := util.FileOrURLReadCloser(ctx, v.RPMModel.Package.URL.String(), v.RPMModel.Package.Content)
 		if err != nil {
 			return closePipesOnError(err)
 		}
@@ -236,7 +236,7 @@ func (v *V001Entry) FetchExternalEntities(ctx context.Context) error {
 
 	g.Go(func() error {
 		keyReadCloser, err := util.FileOrURLReadCloser(ctx, v.RPMModel.PublicKey.URL.String(),
-			v.RPMModel.PublicKey.Content, false)
+			v.RPMModel.PublicKey.Content)
 		if err != nil {
 			return closePipesOnError(err)
 		}

pkg/util/fetch.go

@@ -16,9 +16,7 @@ limitations under the License.
 package util
 
 import (
-	"bufio"
 	"bytes"
-	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -27,7 +25,7 @@ import (
 )
 
 // FileOrURLReadCloser Note: caller is responsible for closing ReadCloser returned from method!
-func FileOrURLReadCloser(ctx context.Context, url string, content []byte, checkGZIP bool) (io.ReadCloser, error) {
+func FileOrURLReadCloser(ctx context.Context, url string, content []byte) (io.ReadCloser, error) {
 	var dataReader io.ReadCloser
 	if url != "" {
 		//TODO: set timeout here, SSL settings?
@@ -44,22 +42,7 @@ func FileOrURLReadCloser(ctx context.Context, url string, content []byte, checkG
 			return nil, fmt.Errorf("error received while fetching artifact: %v", resp.Status)
 		}
 
-		if checkGZIP {
-			// read first 512 bytes to determine if content is gzip compressed
-			bufReader := bufio.NewReaderSize(resp.Body, 512)
-			ctBuf, err := bufReader.Peek(512)
-			if err != nil && err != bufio.ErrBufferFull && err != io.EOF {
-				return nil, err
-			}
-
-			if http.DetectContentType(ctBuf) == "application/x-gzip" {
-				dataReader, _ = gzip.NewReader(io.MultiReader(bufReader, resp.Body))
-			} else {
-				dataReader = ioutil.NopCloser(io.MultiReader(bufReader, resp.Body))
-			}
-		} else {
-			dataReader = resp.Body
-		}
+		dataReader = resp.Body
 	} else {
 		dataReader = ioutil.NopCloser(bytes.NewReader(content))
 	}

tests/e2e-test.sh

 #!/bin/bash
-#set -ex
+set -e
 testdir=$(dirname "$0")
 
 echo "starting services"
@@ -26,7 +26,7 @@ done
 
 echo
 echo "running tests"
-TMPDIR="$(mktemp -d -t rekor_test)"
+TMPDIR="$(mktemp -d -t rekor_test.XXXXXX)"
 touch $TMPDIR.rekor.yaml
 trap "rm -rf $TMPDIR" EXIT
-TMPDIR=$TMPDIR go test -tags=e2e ./tests/
\ No newline at end of file
+TMPDIR=$TMPDIR go test -tags=e2e ./tests/

tests/e2e_test.go

@@ -303,8 +303,10 @@ func TestWatch(t *testing.T) {
 	go func() {
 		b, err := cmd.CombinedOutput()
 		t.Log(string(b))
-		if err != nil {
-			t.Fatal(err)
+		if cmd.ProcessState.Exited() && cmd.ProcessState.ExitCode() != 0 {
+			if err != nil {
+				t.Fatal(err)
+			}
 		}
 	}()