Add sharding package and update validators (#583)

* Add sharding package

Signed-off-by: Lily Sturmann <lsturman@redhat.com>

* Update validators to support future FullID

A FullID is a UUID prepended by a TreeID. This will be used for log sharding

Signed-off-by: Lily Sturmann <lsturman@redhat.com>

cmd/rekor-cli/app/pflags.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/sigstore/rekor/pkg/pki"
+	"github.com/sigstore/rekor/pkg/sharding"
 	"github.com/sigstore/rekor/pkg/util"
 
 	"github.com/spf13/pflag"
@@ -56,8 +57,10 @@ var pflagValueFuncMap map[FlagType]newPFlagValueFunc
 func initializePFlagMap() {
 	pflagValueFuncMap = map[FlagType]newPFlagValueFunc{
 		uuidFlag: func() pflag.Value {
-			// this corresponds to the merkle leaf hash of entries, which is represented by a 64 character hexadecimal string
-			return valueFactory(uuidFlag, validateString("required,len=64,hexadecimal"), "")
+			// this validates a UUID with or without a prepended TreeID;
+			// the UUID corresponds to the merkle leaf hash of entries,
+			// which is represented by a 64 character hexadecimal string
+			return valueFactory(uuidFlag, validateID, "")
 		},
 		shaFlag: func() pflag.Value {
 			// this validates a valid sha256 checksum which is optionally prefixed with 'sha256:'
@@ -190,6 +193,19 @@ func validateFileOrURL(v string) error {
 	return valGen().Set(v)
 }
 
+// validateID ensures the ID is either a FullID (TreeID + UUID) or a UUID
+func validateID(v string) error {
+	if len(v) != sharding.FullIDHexStringLen && len(v) != sharding.UUIDHexStringLen {
+		return fmt.Errorf("ID len error, expected %v (FullID) or %v (UUID) but got len %v for ID %v", sharding.FullIDHexStringLen, sharding.UUIDHexStringLen, len(v), v)
+	}
+
+	if err := validateString("required,hexadecimal")(v); err != nil {
+		return fmt.Errorf("invalid uuid: %v", v)
+	}
+
+	return nil
+}
+
 // validateLogIndex ensures that the supplied string is a valid log index (integer >= 0)
 func validateLogIndex(v string) error {
 	i, err := strconv.Atoi(v)

openapi.yaml

@@ -65,7 +65,7 @@ paths:
             items:
               type: string
               description: Entry UUID in transparency log
-              pattern: '^[0-9a-fA-F]{64}$'
+              pattern: '(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)'
         400:
           $ref: '#/responses/BadContent'
         default:
@@ -203,7 +203,7 @@ paths:
           name: entryUUID
           type: string
           required: true
-          pattern: '^[0-9a-fA-F]{64}$'
+          pattern: '(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)'
           description: the UUID of the entry for which the inclusion proof information should be returned
       responses:
         200:

pkg/generated/restapi/embedded_spec.go

@@ -78,7 +78,7 @@ func init() {
               "items": {
                 "description": "Entry UUID in transparency log",
                 "type": "string",
-                "pattern": "^[0-9a-fA-F]{64}$"
+                "pattern": "(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)"
               }
             }
           },
@@ -236,7 +236,7 @@ func init() {
         "operationId": "getLogEntryByUUID",
         "parameters": [
           {
-            "pattern": "^[0-9a-fA-F]{64}$",
+            "pattern": "(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)",
             "type": "string",
             "description": "the UUID of the entry for which the inclusion proof information should be returned",
             "name": "entryUUID",
@@ -990,7 +990,7 @@ func init() {
               "items": {
                 "description": "Entry UUID in transparency log",
                 "type": "string",
-                "pattern": "^[0-9a-fA-F]{64}$"
+                "pattern": "(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)"
               }
             }
           },
@@ -1182,7 +1182,7 @@ func init() {
         "operationId": "getLogEntryByUUID",
         "parameters": [
           {
-            "pattern": "^[0-9a-fA-F]{64}$",
+            "pattern": "(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)",
             "type": "string",
             "description": "the UUID of the entry for which the inclusion proof information should be returned",
             "name": "entryUUID",

pkg/generated/restapi/operations/entries/get_log_entry_by_uuid_parameters.go

@@ -49,7 +49,7 @@ type GetLogEntryByUUIDParams struct {
 
 	/*the UUID of the entry for which the inclusion proof information should be returned
 	  Required: true
-	  Pattern: ^[0-9a-fA-F]{64}$
+	  Pattern: (^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)
 	  In: path
 	*/
 	EntryUUID string
@@ -95,7 +95,7 @@ func (o *GetLogEntryByUUIDParams) bindEntryUUID(rawData []string, hasKey bool, f
 // validateEntryUUID carries on validations for parameter EntryUUID
 func (o *GetLogEntryByUUIDParams) validateEntryUUID(formats strfmt.Registry) error {
 
-	if err := validate.Pattern("entryUUID", "path", o.EntryUUID, `^[0-9a-fA-F]{64}$`); err != nil {
+	if err := validate.Pattern("entryUUID", "path", o.EntryUUID, `(^[0-9a-fA-F]{64}|[0-9a-fA-F]{80}$)`); err != nil {
 		return err
 	}
 

pkg/sharding/sharding.go

+//
+// Copyright 2021 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sharding
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	"github.com/sigstore/rekor/pkg/api"
+)
+
+// A FullID refers to a specific artifact's ID and is made of two components,
+// the TreeID and the UUID. The TreeID is a hex-encoded uint64 (8 bytes)
+// referring to the specific trillian tree (also known as log or shard) where
+// the artifact can be found. The UUID is a hex-encoded 32-byte number
+// referring to the artifact's merkle leaf hash from trillian. Artifact lookup
+// by UUID occurs by finding the UUID within the tree specified by the TreeID.
+//
+// A FullID is 40 bytes long and looks like this:
+// FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF
+// |_______  ________| |_____________________________________  ______________________________________|
+//         \/                                                \/
+// TreeID (8 bytes, hex)                             UUID (32 bytes, hex)
+
+const TreeIDHexStringLen = 16
+const UUIDHexStringLen = 64
+const FullIDHexStringLen = TreeIDHexStringLen + UUIDHexStringLen
+
+// TODO: replace this with the actual LogRanges struct when logic is hooked up
+var dummy = api.LogRanges{
+	Ranges: []api.LogRange{},
+}
+
+type FullID struct {
+	TreeID string
+	UUID   string
+}
+
+func CreateFullID(treeid string, uuid string) (FullID, error) {
+	if len(treeid) != TreeIDHexStringLen {
+		err := fmt.Errorf("invalid treeid len: %v", len(treeid))
+		return createEmptyFullID(), err
+	}
+
+	if len(uuid) != UUIDHexStringLen {
+		err := fmt.Errorf("invalid uuid len: %v", len(uuid))
+		return createEmptyFullID(), err
+	}
+
+	if _, err := hex.DecodeString(treeid); err != nil {
+		err := fmt.Errorf("treeid is not a valid hex string: %v", treeid)
+		return createEmptyFullID(), err
+	}
+
+	if _, err := hex.DecodeString(uuid); err != nil {
+		err := fmt.Errorf("uuid is not a valid hex string: %v", uuid)
+		return createEmptyFullID(), err
+	}
+
+	return FullID{
+		TreeID: treeid,
+		UUID:   uuid}, nil
+}
+
+func createEmptyFullID() FullID {
+	return FullID{
+		TreeID: "",
+		UUID:   ""}
+}
+
+func PrependActiveTreeID(uuid string) (FullID, error) {
+	// TODO: Update this to be the global LogRanges struct
+	active := dummy.ActiveIndex()
+	return CreateFullID(strconv.FormatUint(active, 10), uuid)
+}