diff --git a/client/client_test.go b/client/client_test.go index fb4ee329..e4edaaa2 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -18,6 +18,7 @@ import ( cjson "github.com/tent/canonical-json-go" tuf "github.com/theupdateframework/go-tuf" "github.com/theupdateframework/go-tuf/data" + "github.com/theupdateframework/go-tuf/internal/sets" "github.com/theupdateframework/go-tuf/pkg/keys" "github.com/theupdateframework/go-tuf/sign" "github.com/theupdateframework/go-tuf/util" @@ -362,7 +363,7 @@ func (s *ClientSuite) TestNewRoot(c *C) { } role := client.db.GetRole(name) c.Assert(role, NotNil) - c.Assert(role.KeyIDs, DeepEquals, util.StringSliceToSet(ids)) + c.Assert(role.KeyIDs, DeepEquals, sets.StringSliceToSet(ids)) } } @@ -602,7 +603,7 @@ func (s *ClientSuite) TestNewTimestampKey(c *C) { } role := client.db.GetRole("timestamp") c.Assert(role, NotNil) - c.Assert(role.KeyIDs, DeepEquals, util.StringSliceToSet(newIDs)) + c.Assert(role.KeyIDs, DeepEquals, sets.StringSliceToSet(newIDs)) } func (s *ClientSuite) TestNewSnapshotKey(c *C) { @@ -642,7 +643,7 @@ func (s *ClientSuite) TestNewSnapshotKey(c *C) { } role := client.db.GetRole("snapshot") c.Assert(role, NotNil) - c.Assert(role.KeyIDs, DeepEquals, util.StringSliceToSet(newIDs)) + c.Assert(role.KeyIDs, DeepEquals, sets.StringSliceToSet(newIDs)) } func (s *ClientSuite) TestNewTargetsKey(c *C) { @@ -685,7 +686,7 @@ func (s *ClientSuite) TestNewTargetsKey(c *C) { } role := client.db.GetRole("targets") c.Assert(role, NotNil) - c.Assert(role.KeyIDs, DeepEquals, util.StringSliceToSet(newIDs)) + c.Assert(role.KeyIDs, DeepEquals, sets.StringSliceToSet(newIDs)) } func (s *ClientSuite) TestLocalExpired(c *C) { diff --git a/client/delegations.go b/client/delegations.go index f66a302a..6b17df66 100644 --- a/client/delegations.go +++ b/client/delegations.go @@ -20,7 +20,7 @@ func (c *Client) getTargetFileMeta(target string) (data.TargetFileMeta, error) { // - filter delegations with paths or path_hash_prefixes matching searched target // - 5.6.7.1 cycles protection // - 5.6.7.2 terminations - delegations := targets.NewDelegationsIterator(target) + delegations := targets.NewDelegationsIterator(target, c.db) for i := 0; i < c.MaxDelegations; i++ { d, ok := delegations.Next() if !ok { @@ -28,7 +28,7 @@ func (c *Client) getTargetFileMeta(target string) (data.TargetFileMeta, error) { } // covers 5.6.{1,2,3,4,5,6} - targets, err := c.loadDelegatedTargets(snapshot, d.Delegatee.Name, d.Verifier) + targets, err := c.loadDelegatedTargets(snapshot, d.Delegatee.Name, d.DB) if err != nil { return data.TargetFileMeta{}, err } @@ -39,11 +39,11 @@ func (c *Client) getTargetFileMeta(target string) (data.TargetFileMeta, error) { } if targets.Delegations != nil { - delegationsVerifier, err := verify.NewDelegationsVerifier(targets.Delegations) + delegationsDB, err := verify.NewDBFromDelegations(targets.Delegations) if err != nil { return data.TargetFileMeta{}, err } - err = delegations.Add(targets.Delegations.Roles, d.Delegatee.Name, delegationsVerifier) + err = delegations.Add(targets.Delegations.Roles, d.Delegatee.Name, delegationsDB) if err != nil { return data.TargetFileMeta{}, err } @@ -75,7 +75,7 @@ func (c *Client) loadLocalSnapshot() (*data.Snapshot, error) { } // loadDelegatedTargets downloads, decodes, verifies and stores targets -func (c *Client) loadDelegatedTargets(snapshot *data.Snapshot, role string, verifier verify.DelegationsVerifier) (*data.Targets, error) { +func (c *Client) loadDelegatedTargets(snapshot *data.Snapshot, role string, db *verify.DB) (*data.Targets, error) { var err error fileName := role + ".json" fileMeta, ok := snapshot.Meta[fileName] @@ -98,11 +98,7 @@ func (c *Client) loadDelegatedTargets(snapshot *data.Snapshot, role string, veri // 5.6.3 verify signature with parent public keys // 5.6.5 verify that the targets is not expired // role "targets" is a top role verified by root keys loaded in the client db - if role == "targets" { - err = c.db.Unmarshal(raw, targets, role, fileMeta.Version) - } else { - err = verifier.Unmarshal(raw, targets, role, fileMeta.Version) - } + err = db.Unmarshal(raw, targets, role, fileMeta.Version) if err != nil { return nil, ErrDecodeFailed{fileName, err} } diff --git a/data/types.go b/data/types.go index 700fa156..557f3ab8 100644 --- a/data/types.go +++ b/data/types.go @@ -82,12 +82,13 @@ func DefaultExpires(role string) time.Time { switch role { case "root": t = time.Now().AddDate(1, 0, 0) - case "targets": - t = time.Now().AddDate(0, 3, 0) case "snapshot": t = time.Now().AddDate(0, 0, 7) case "timestamp": t = time.Now().AddDate(0, 0, 1) + default: + // Target + t = time.Now().AddDate(0, 3, 0) } return t.UTC().Round(time.Second) } diff --git a/errors.go b/errors.go index c471dc46..f381b15c 100644 --- a/errors.go +++ b/errors.go @@ -86,3 +86,11 @@ type ErrPassphraseRequired struct { func (e ErrPassphraseRequired) Error() string { return fmt.Sprintf("tuf: a passphrase is required to access the encrypted %s keys file", e.Role) } + +type ErrNoDelegatedTarget struct { + Path string +} + +func (e ErrNoDelegatedTarget) Error() string { + return fmt.Sprintf("tuf: no delegated target for path %s", e.Path) +} diff --git a/internal/roles/roles.go b/internal/roles/roles.go new file mode 100644 index 00000000..f7841c26 --- /dev/null +++ b/internal/roles/roles.go @@ -0,0 +1,41 @@ +package roles + +import ( + "strconv" + "strings" +) + +var TopLevelRoles = map[string]struct{}{ + "root": {}, + "targets": {}, + "snapshot": {}, + "timestamp": {}, +} + +func IsTopLevelRole(name string) bool { + _, ok := TopLevelRoles[name] + return ok +} + +func IsDelegatedTargetsRole(name string) bool { + return !IsTopLevelRole(name) +} + +func IsTopLevelManifest(name string) bool { + return IsTopLevelRole(strings.TrimSuffix(name, ".json")) +} + +func IsDelegatedTargetsManifest(name string) bool { + return !IsTopLevelManifest(name) +} + +func IsVersionedManifest(name string) bool { + parts := strings.Split(name, ".") + // Versioned manifests have the form "x.role.json" + if len(parts) < 3 { + return false + } + + _, err := strconv.Atoi(parts[0]) + return err == nil +} diff --git a/internal/sets/strings.go b/internal/sets/strings.go new file mode 100644 index 00000000..07ee8945 --- /dev/null +++ b/internal/sets/strings.go @@ -0,0 +1,24 @@ +package sets + +func StringSliceToSet(items []string) map[string]struct{} { + s := make(map[string]struct{}) + for _, item := range items { + s[item] = struct{}{} + } + return s +} + +func StringSetToSlice(items map[string]struct{}) []string { + ret := []string{} + + for k := range items { + ret = append(ret, k) + } + + return ret +} + +func DeduplicateStrings(items []string) []string { + s := StringSliceToSet(items) + return StringSetToSlice(s) +} diff --git a/internal/targets/delegation.go b/internal/targets/delegation.go index 8e09c05c..3823df4e 100644 --- a/internal/targets/delegation.go +++ b/internal/targets/delegation.go @@ -2,13 +2,14 @@ package targets import ( "github.com/theupdateframework/go-tuf/data" + "github.com/theupdateframework/go-tuf/internal/sets" "github.com/theupdateframework/go-tuf/verify" ) type Delegation struct { Delegator string - Verifier verify.DelegationsVerifier Delegatee data.DelegatedRole + DB *verify.DB } type delegationsIterator struct { @@ -18,13 +19,23 @@ type delegationsIterator struct { } // NewDelegationsIterator initialises an iterator with a first step -// on top level targets -func NewDelegationsIterator(target string) *delegationsIterator { +// on top level targets. +func NewDelegationsIterator(target string, topLevelKeysDB *verify.DB) *delegationsIterator { + role := topLevelKeysDB.GetRole("targets") + keyIDs := []string{} + if role != nil { + keyIDs = sets.StringSetToSlice(role.KeyIDs) + } + i := &delegationsIterator{ target: target, stack: []Delegation{ { - Delegatee: data.DelegatedRole{Name: "targets"}, + Delegatee: data.DelegatedRole{ + Name: "targets", + KeyIDs: keyIDs, + }, + DB: topLevelKeysDB, }, }, visitedRoles: make(map[string]struct{}), @@ -57,7 +68,7 @@ func (d *delegationsIterator) Next() (value Delegation, ok bool) { return delegation, true } -func (d *delegationsIterator) Add(roles []data.DelegatedRole, delegator string, verifier verify.DelegationsVerifier) error { +func (d *delegationsIterator) Add(roles []data.DelegatedRole, delegator string, db *verify.DB) error { for i := len(roles) - 1; i >= 0; i-- { // Push the roles onto the stack in reverse so we get an preorder traversal // of the delegations graph. @@ -70,7 +81,7 @@ func (d *delegationsIterator) Add(roles []data.DelegatedRole, delegator string, delegation := Delegation{ Delegator: delegator, Delegatee: r, - Verifier: verifier, + DB: db, } d.stack = append(d.stack, delegation) } diff --git a/internal/targets/delegation_test.go b/internal/targets/delegation_test.go index bb460b18..f157c077 100644 --- a/internal/targets/delegation_test.go +++ b/internal/targets/delegation_test.go @@ -14,6 +14,7 @@ var ( ) func TestDelegationsIterator(t *testing.T) { + defaultKeyIDs := []string{"26b878ad73362774b8b69dd4fdeb2cc6a2688e4133ed5ace9e18a06e9d998a6d"} var iteratorTests = []struct { testName string roles map[string][]data.DelegatedRole @@ -25,23 +26,23 @@ func TestDelegationsIterator(t *testing.T) { testName: "no termination", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", Paths: defaultPathPatterns}, - {Name: "e", Paths: defaultPathPatterns}, + {Name: "b", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "e", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": { - {Name: "c", Paths: defaultPathPatterns}, + {Name: "c", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "c": { - {Name: "d", Paths: defaultPathPatterns}, + {Name: "d", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "e": { - {Name: "f", Paths: defaultPathPatterns}, - {Name: "g", Paths: defaultPathPatterns}, + {Name: "f", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "g", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "g": { - {Name: "h", Paths: defaultPathPatterns}, - {Name: "i", Paths: defaultPathPatterns}, - {Name: "j", Paths: defaultPathPatterns}, + {Name: "h", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "i", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "j", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", @@ -51,12 +52,12 @@ func TestDelegationsIterator(t *testing.T) { testName: "terminated in b", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", Paths: defaultPathPatterns, Terminating: true}, - {Name: "e", Paths: defaultPathPatterns}, + {Name: "b", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs, Terminating: true}, + {Name: "e", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": { - {Name: "c", Paths: defaultPathPatterns}, - {Name: "d", Paths: defaultPathPatterns}, + {Name: "c", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "d", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", @@ -66,12 +67,12 @@ func TestDelegationsIterator(t *testing.T) { testName: "path does not match b", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", Paths: noMatchPathPatterns}, - {Name: "e", Paths: defaultPathPatterns}, + {Name: "b", Paths: noMatchPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "e", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": { - {Name: "c", Paths: defaultPathPatterns}, - {Name: "d", Paths: defaultPathPatterns}, + {Name: "c", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "d", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", @@ -81,12 +82,13 @@ func TestDelegationsIterator(t *testing.T) { testName: "path does not match b - path prefixes", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", PathHashPrefixes: []string{"33472a4909"}}, - {Name: "c", PathHashPrefixes: []string{"34c85d1ee84f61f10d7dc633"}}, + {Name: "b", PathHashPrefixes: []string{"33472a4909"}, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "c", PathHashPrefixes: []string{"34c85d1ee84f61f10d7dc633"}, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "c": { - {Name: "d", PathHashPrefixes: []string{"8baf"}}, - {Name: "e", PathHashPrefixes: []string{"34c85d1ee84f61f10d7dc633472a49096ed87f8f764bd597831eac371f40ac39"}}, + + {Name: "d", PathHashPrefixes: []string{"8baf"}, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "e", PathHashPrefixes: []string{"34c85d1ee84f61f10d7dc633472a49096ed87f8f764bd597831eac371f40ac39"}, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "/e/f/g.txt", @@ -96,7 +98,7 @@ func TestDelegationsIterator(t *testing.T) { testName: "err paths and pathHashPrefixes are set", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", Paths: defaultPathPatterns, PathHashPrefixes: defaultPathPatterns}, + {Name: "b", Paths: defaultPathPatterns, PathHashPrefixes: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": {}, }, @@ -108,48 +110,54 @@ func TestDelegationsIterator(t *testing.T) { testName: "cycle avoided 1", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", Paths: defaultPathPatterns}, - {Name: "e", Paths: defaultPathPatterns}, + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + }, + "a": { + {Name: "b", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "e", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": { - {Name: "targets", Paths: defaultPathPatterns}, - {Name: "d", Paths: defaultPathPatterns}, + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "d", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", - resultOrder: []string{"targets", "b", "d", "e"}, + resultOrder: []string{"targets", "a", "b", "d", "e"}, }, { testName: "cycle avoided 2", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "targets", Paths: defaultPathPatterns}, - {Name: "b", Paths: defaultPathPatterns}, + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + }, + "a": { + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "b", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": { - {Name: "targets", Paths: defaultPathPatterns}, - {Name: "b", Paths: defaultPathPatterns}, - {Name: "c", Paths: defaultPathPatterns}, + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "b", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "c", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "c": { - {Name: "c", Paths: defaultPathPatterns}, + {Name: "c", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", - resultOrder: []string{"targets", "b", "c"}, + resultOrder: []string{"targets", "a", "b", "c"}, }, { testName: "diamond delegation", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "b", Paths: defaultPathPatterns}, - {Name: "c", Paths: defaultPathPatterns}, + {Name: "b", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, + {Name: "c", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "b": { - {Name: "d", Paths: defaultPathPatterns}, + {Name: "d", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "c": { - {Name: "d", Paths: defaultPathPatterns}, + {Name: "d", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", @@ -159,10 +167,10 @@ func TestDelegationsIterator(t *testing.T) { testName: "simple cycle", roles: map[string][]data.DelegatedRole{ "targets": { - {Name: "a", Paths: defaultPathPatterns}, + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, "a": { - {Name: "a", Paths: defaultPathPatterns}, + {Name: "a", Paths: defaultPathPatterns, Threshold: 1, KeyIDs: defaultKeyIDs}, }, }, file: "", @@ -172,7 +180,17 @@ func TestDelegationsIterator(t *testing.T) { for _, tt := range iteratorTests { t.Run(tt.testName, func(t *testing.T) { - d := NewDelegationsIterator(tt.file) + flattened := []data.DelegatedRole{} + for _, roles := range tt.roles { + flattened = append(flattened, roles...) + } + db, err := verify.NewDBFromDelegations(&data.Delegations{ + Roles: flattened, + }) + + assert.NoError(t, err) + d := NewDelegationsIterator(tt.file, db) + var iterationOrder []string for { r, ok := d.Next() @@ -184,7 +202,13 @@ func TestDelegationsIterator(t *testing.T) { if !ok { continue } - err := d.Add(delegations, r.Delegatee.Name, verify.DelegationsVerifier{}) + + db, err := verify.NewDBFromDelegations(&data.Delegations{ + Roles: delegations, + }) + assert.NoError(t, err) + + err = d.Add(delegations, r.Delegatee.Name, db) assert.Equal(t, tt.err, err) } assert.Equal(t, tt.resultOrder, iterationOrder) diff --git a/internal/targets/hash_bins.go b/internal/targets/hash_bins.go new file mode 100644 index 00000000..12116dff --- /dev/null +++ b/internal/targets/hash_bins.go @@ -0,0 +1,82 @@ +package targets + +import ( + "strconv" + "strings" +) + +// hexEncode formats x as a hex string, left padded with zeros to padWidth. +func hexEncode(x uint64, padWidth int) string { + // Benchmarked to be more than 10x faster than padding with Sprintf. + s := strconv.FormatUint(x, 16) + if len(s) >= padWidth { + return s + } + return strings.Repeat("0", padWidth-len(s)) + s +} + +// HashBin represents a hex prefix range. First should be less than Last. +type HashBin struct { + First uint64 + Last uint64 +} + +// Name returns the of the role that signs for the HashBin. +func (b HashBin) Name(prefix string, padWidth int) string { + if b.First == b.Last { + return prefix + hexEncode(b.First, padWidth) + } + + return prefix + hexEncode(b.First, padWidth) + "-" + hexEncode(b.Last, padWidth) +} + +// Enumerate returns a slice of hash prefixes in the range from First to Last. +func (b HashBin) Enumerate(padWidth int) []string { + n := int(b.Last - b.First + 1) + ret := make([]string, int(n)) + + x := b.First + for i := 0; i < n; i++ { + ret[i] = hexEncode(x, padWidth) + x++ + } + + return ret +} + +// HashPrefixLength returns the width of hash prefixes if there are +// 2^(log2NumBins) hash bins. +func HashPrefixLength(log2NumBins uint8) int { + if log2NumBins == 0 { + // Hash prefix of "" is represented equivalently as "0-f". + return 1 + } + + // ceil(log2NumBins / 4.0) + return int((log2NumBins-1)/4) + 1 +} + +// GenerateHashBins returns a slice of length 2^(log2NumBins) that partitions +// the space of path hashes into HashBin ranges. +func GenerateHashBins(log2NumBins uint8) []HashBin { + numBins := uint64(1) << log2NumBins + + // numPrefixes = 16^(HashPrefixLength(log2NumBins)) + numPrefixes := uint64(1) << (4 * HashPrefixLength(log2NumBins)) + + p := make([]HashBin, numBins) + + first := uint64(0) + interval := numPrefixes / numBins + last := first + interval - 1 + for i := uint64(0); i < numBins; i++ { + p[i] = HashBin{ + First: first, + Last: last, + } + first += interval + last += interval + } + + return p +} diff --git a/internal/targets/hash_bins_test.go b/internal/targets/hash_bins_test.go new file mode 100644 index 00000000..b2042abb --- /dev/null +++ b/internal/targets/hash_bins_test.go @@ -0,0 +1,95 @@ +package targets + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkHexEncode1(b *testing.B) { + for n := 0; n <= b.N; n++ { + for x := uint64(0); x <= 0xf; x += 1 { + hexEncode(x, 1) + } + } +} + +func BenchmarkHexEncode4(b *testing.B) { + for n := 0; n <= b.N; n++ { + for x := uint64(0); x <= 0xffff; x += 1 { + hexEncode(x, 4) + } + } +} + +func TestHashBin(t *testing.T) { + h := HashBin{ + First: 0x0, + Last: 0xf, + } + assert.Equal(t, "abc_0-f", h.Name("abc_", 1)) + assert.Equal(t, "abc_0000-000f", h.Name("abc_", 4)) + assert.Equal(t, []string{ + "00", "01", "02", "03", "04", "05", "06", "07", + "08", "09", "0a", "0b", "0c", "0d", "0e", "0f", + }, h.Enumerate(2)) + + h = HashBin{ + First: 0xcd, + Last: 0xce, + } + assert.Equal(t, "abc_00cd-00ce", h.Name("abc_", 4)) + assert.Equal(t, []string{"00cd", "00ce"}, h.Enumerate(4)) + + h = HashBin{ + First: 0x0abc, + Last: 0xbcde, + } + assert.Equal(t, "test_0abc-bcde", h.Name("test_", 4)) +} + +func TestHashPrefixLength(t *testing.T) { + assert.Equal(t, 1, HashPrefixLength(0)) + assert.Equal(t, 1, HashPrefixLength(1)) + assert.Equal(t, 1, HashPrefixLength(2)) + assert.Equal(t, 1, HashPrefixLength(3)) + assert.Equal(t, 1, HashPrefixLength(4)) + assert.Equal(t, 2, HashPrefixLength(5)) + assert.Equal(t, 2, HashPrefixLength(6)) + assert.Equal(t, 2, HashPrefixLength(7)) + assert.Equal(t, 2, HashPrefixLength(8)) + assert.Equal(t, 3, HashPrefixLength(9)) + assert.Equal(t, 3, HashPrefixLength(10)) + assert.Equal(t, 3, HashPrefixLength(11)) + assert.Equal(t, 3, HashPrefixLength(12)) +} + +func TestGenerateHashBins(t *testing.T) { + tcs := []struct { + Log2NumBins uint8 + BinNames []string + }{ + {0, []string{"0-f"}}, + {1, []string{"0-7", "8-f"}}, + {2, []string{"0-3", "4-7", "8-b", "c-f"}}, + {3, []string{"0-1", "2-3", "4-5", "6-7", "8-9", "a-b", "c-d", "e-f"}}, + {4, []string{ + "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", "a", "b", "c", "d", "e", "f", + }}, + {5, []string{ + "00-07", "08-0f", "10-17", "18-1f", "20-27", "28-2f", "30-37", "38-3f", + "40-47", "48-4f", "50-57", "58-5f", "60-67", "68-6f", "70-77", "78-7f", + "80-87", "88-8f", "90-97", "98-9f", "a0-a7", "a8-af", "b0-b7", "b8-bf", + "c0-c7", "c8-cf", "d0-d7", "d8-df", "e0-e7", "e8-ef", "f0-f7", "f8-ff", + }}, + } + for _, tc := range tcs { + bn := []string{} + bins := GenerateHashBins(tc.Log2NumBins) + for _, b := range bins { + bn = append(bn, b.Name("", HashPrefixLength(tc.Log2NumBins))) + } + assert.Equal(t, tc.BinNames, bn, "GenerateHashBins(%v)", tc.Log2NumBins) + } +} diff --git a/local_store.go b/local_store.go index bc926528..a8515a8c 100644 --- a/local_store.go +++ b/local_store.go @@ -11,6 +11,7 @@ import ( "github.com/theupdateframework/go-tuf/data" "github.com/theupdateframework/go-tuf/encrypted" + "github.com/theupdateframework/go-tuf/internal/sets" "github.com/theupdateframework/go-tuf/pkg/keys" "github.com/theupdateframework/go-tuf/util" ) @@ -27,15 +28,44 @@ func signers(privateKeys []*data.PrivateKey) []keys.Signer { return res } +type LocalStore interface { + // GetMeta returns a map from metadata file names (e.g. root.json) to their raw JSON payload or an error. + GetMeta() (map[string]json.RawMessage, error) + + // SetMeta is used to update a metadata file name with a JSON payload. + SetMeta(name string, meta json.RawMessage) error + + // WalkStagedTargets calls targetsFn for each staged target file in paths. + // + // If paths is empty, all staged target files will be walked. + WalkStagedTargets(paths []string, targetsFn TargetsWalkFunc) error + + // Commit is used to publish staged files to the repository + Commit(consistentSnapshot bool, versions map[string]int, hashes map[string]data.Hashes) error + + // GetSigners return a list of signers for a role. + GetSigners(role string) ([]keys.Signer, error) + + // SaveSigner adds a signer to a role. + SaveSigner(role string, signer keys.Signer) error + + // SignersForRole return a list of signing keys for a role. + SignersForKeyIDs(keyIDs []string) []keys.Signer + + // Clean is used to remove all staged manifests. + Clean() error +} + func MemoryStore(meta map[string]json.RawMessage, files map[string][]byte) LocalStore { if meta == nil { meta = make(map[string]json.RawMessage) } return &memoryStore{ - meta: meta, - stagedMeta: make(map[string]json.RawMessage), - files: files, - signers: make(map[string][]keys.Signer), + meta: meta, + stagedMeta: make(map[string]json.RawMessage), + files: files, + signerForKeyID: make(map[string]keys.Signer), + keyIDsForRole: make(map[string][]string), } } @@ -43,7 +73,9 @@ type memoryStore struct { meta map[string]json.RawMessage stagedMeta map[string]json.RawMessage files map[string][]byte - signers map[string][]keys.Signer + + signerForKeyID map[string]keys.Signer + keyIDsForRole map[string][]string } func (m *memoryStore) GetMeta() (map[string]json.RawMessage, error) { @@ -95,14 +127,53 @@ func (m *memoryStore) Commit(consistentSnapshot bool, versions map[string]int, h } func (m *memoryStore) GetSigners(role string) ([]keys.Signer, error) { - return m.signers[role], nil + keyIDs, ok := m.keyIDsForRole[role] + if ok { + return m.SignersForKeyIDs(keyIDs), nil + } + + return nil, nil } func (m *memoryStore) SaveSigner(role string, signer keys.Signer) error { - m.signers[role] = append(m.signers[role], signer) + keyIDs := signer.PublicData().IDs() + + for _, keyID := range keyIDs { + m.signerForKeyID[keyID] = signer + } + + mergedKeyIDs := sets.DeduplicateStrings(append(m.keyIDsForRole[role], keyIDs...)) + m.keyIDsForRole[role] = mergedKeyIDs return nil } +func (m *memoryStore) SignersForKeyIDs(keyIDs []string) []keys.Signer { + signers := []keys.Signer{} + keyIDsSeen := map[string]struct{}{} + + for _, keyID := range keyIDs { + signer, ok := m.signerForKeyID[keyID] + if !ok { + continue + } + addSigner := false + + for _, skid := range signer.PublicData().IDs() { + if _, seen := keyIDsSeen[skid]; !seen { + addSigner = true + } + + keyIDsSeen[skid] = struct{}{} + } + + if addSigner { + signers = append(signers, signer) + } + } + + return signers +} + func (m *memoryStore) Clean() error { return nil } @@ -116,7 +187,8 @@ func FileSystemStore(dir string, p util.PassphraseFunc) LocalStore { return &fileSystemStore{ dir: dir, passphraseFunc: p, - signers: make(map[string][]keys.Signer), + signerForKeyID: make(map[string]keys.Signer), + keyIDsForRole: make(map[string][]string), } } @@ -124,8 +196,8 @@ type fileSystemStore struct { dir string passphraseFunc util.PassphraseFunc - // signers is a cache of persisted keys to avoid decrypting multiple times - signers map[string][]keys.Signer + signerForKeyID map[string]keys.Signer + keyIDsForRole map[string][]string } func (f *fileSystemStore) repoDir() string { @@ -317,18 +389,62 @@ func (f *fileSystemStore) Commit(consistentSnapshot bool, versions map[string]in } func (f *fileSystemStore) GetSigners(role string) ([]keys.Signer, error) { - if keys, ok := f.signers[role]; ok { - return keys, nil + keyIDs, ok := f.keyIDsForRole[role] + if ok { + return f.SignersForKeyIDs(keyIDs), nil } - keys, _, err := f.loadPrivateKeys(role) + + privKeys, _, err := f.loadPrivateKeys(role) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } - f.signers[role] = signers(keys) - return f.signers[role], nil + + signers := []keys.Signer{} + for _, key := range privKeys { + signer, err := keys.GetSigner(key) + if err != nil { + return nil, err + } + + // Cache the signers. + for _, keyID := range signer.PublicData().IDs() { + f.keyIDsForRole[role] = append(f.keyIDsForRole[role], keyID) + f.signerForKeyID[keyID] = signer + } + } + + return signers, nil +} + +func (f *fileSystemStore) SignersForKeyIDs(keyIDs []string) []keys.Signer { + signers := []keys.Signer{} + keyIDsSeen := map[string]struct{}{} + + for _, keyID := range keyIDs { + signer, ok := f.signerForKeyID[keyID] + if !ok { + continue + } + + addSigner := false + + for _, skid := range signer.PublicData().IDs() { + if _, seen := keyIDsSeen[skid]; !seen { + addSigner = true + } + + keyIDsSeen[skid] = struct{}{} + } + + if addSigner { + signers = append(signers, signer) + } + } + + return signers } func (f *fileSystemStore) SaveSigner(role string, signer keys.Signer) error { @@ -337,7 +453,7 @@ func (f *fileSystemStore) SaveSigner(role string, signer keys.Signer) error { } // add the key to the existing keys (if any) - keys, pass, err := f.loadPrivateKeys(role) + privKeys, pass, err := f.loadPrivateKeys(role) if err != nil && !os.IsNotExist(err) { return err } @@ -345,7 +461,7 @@ func (f *fileSystemStore) SaveSigner(role string, signer keys.Signer) error { if err != nil { return err } - keys = append(keys, key) + privKeys = append(privKeys, key) // if loadPrivateKeys didn't return a passphrase (because no keys yet exist) // and passphraseFunc is set, get the passphrase so the keys file can @@ -360,13 +476,13 @@ func (f *fileSystemStore) SaveSigner(role string, signer keys.Signer) error { pk := &persistedKeys{} if pass != nil { - pk.Data, err = encrypted.Marshal(keys, pass) + pk.Data, err = encrypted.Marshal(privKeys, pass) if err != nil { return err } pk.Encrypted = true } else { - pk.Data, err = json.MarshalIndent(keys, "", "\t") + pk.Data, err = json.MarshalIndent(privKeys, "", "\t") if err != nil { return err } @@ -378,7 +494,23 @@ func (f *fileSystemStore) SaveSigner(role string, signer keys.Signer) error { if err := util.AtomicallyWriteFile(f.keysPath(role), append(data, '\n'), 0600); err != nil { return err } - f.signers[role] = append(f.signers[role], signer) + + for _, key := range privKeys { + signer, err := keys.GetSigner(key) + if err != nil { + return err + } + + keyIDs := signer.PublicData().IDs() + + for _, keyID := range keyIDs { + f.signerForKeyID[keyID] = signer + } + + mergedKeyIDs := sets.DeduplicateStrings(append(f.keyIDsForRole[role], keyIDs...)) + f.keyIDsForRole[role] = mergedKeyIDs + } + return nil } diff --git a/repo.go b/repo.go index c74094a0..3266d04e 100644 --- a/repo.go +++ b/repo.go @@ -12,13 +12,20 @@ import ( cjson "github.com/tent/canonical-json-go" "github.com/theupdateframework/go-tuf/data" + "github.com/theupdateframework/go-tuf/internal/roles" "github.com/theupdateframework/go-tuf/internal/signer" + "github.com/theupdateframework/go-tuf/internal/targets" "github.com/theupdateframework/go-tuf/pkg/keys" "github.com/theupdateframework/go-tuf/sign" "github.com/theupdateframework/go-tuf/util" "github.com/theupdateframework/go-tuf/verify" ) +const ( + // The maximum number of delegations to visit while traversing the delegations graph. + defaultMaxDelegations = 32 +) + // topLevelMetadata determines the order signatures are verified when committing. var topLevelMetadata = []string{ "root.json", @@ -27,41 +34,11 @@ var topLevelMetadata = []string{ "timestamp.json", } -var snapshotMetadata = []string{ - "root.json", - "targets.json", -} - // TargetsWalkFunc is a function of a target path name and a target payload used to // execute some function on each staged target file. For example, it may normalize path // names and generate target file metadata with additional custom metadata. type TargetsWalkFunc func(path string, target io.Reader) error -type LocalStore interface { - // GetMeta returns a map from metadata file names (e.g. root.json) to their raw JSON payload or an error. - GetMeta() (map[string]json.RawMessage, error) - - // SetMeta is used to update a metadata file name with a JSON payload. - SetMeta(string, json.RawMessage) error - - // WalkStagedTargets calls targetsFn for each staged target file in paths. - // - // If paths is empty, all staged target files will be walked. - WalkStagedTargets(paths []string, targetsFn TargetsWalkFunc) error - - // Commit is used to publish staged files to the repository - Commit(bool, map[string]int, map[string]data.Hashes) error - - // GetSigners return a list of signers for a role. - GetSigners(string) ([]keys.Signer, error) - - // SavePrivateKey adds a signer to a role. - SaveSigner(string, keys.Signer) error - - // Clean is used to remove all staged metadata files. - Clean() error -} - type Repo struct { local LocalStore hashAlgorithms []string @@ -97,7 +74,7 @@ func NewRepoIndent(local LocalStore, prefix string, indent string, hashAlgorithm } func (r *Repo) Init(consistentSnapshot bool) error { - t, err := r.targets() + t, err := r.topLevelTargets() if err != nil { return err } @@ -109,7 +86,7 @@ func (r *Repo) Init(consistentSnapshot bool) error { return r.setMeta("root.json", root) } -func (r *Repo) db() (*verify.DB, error) { +func (r *Repo) topLevelKeysDB() (*verify.DB, error) { db := verify.NewDB() root, err := r.root() if err != nil { @@ -133,6 +110,7 @@ func (r *Repo) db() (*verify.DB, error) { return nil, err } } + return db, nil } @@ -190,7 +168,7 @@ func (r *Repo) GetThreshold(keyRole string) (int, error) { } func (r *Repo) SetThreshold(keyRole string, t int) error { - if !validMetadata(keyRole + ".json") { + if !roles.IsTopLevelRole(keyRole) { // Delegations are not currently supported, so return an error if this is not a // top-level metadata file. return ErrInvalidRole{keyRole} @@ -216,7 +194,7 @@ func (r *Repo) SetThreshold(keyRole string, t int) error { } func (r *Repo) Targets() (data.TargetFiles, error) { - targets, err := r.targets() + targets, err := r.topLevelTargets() if err != nil { return nil, err } @@ -224,7 +202,7 @@ func (r *Repo) Targets() (data.TargetFiles, error) { } func (r *Repo) SetTargetsVersion(v int) error { - t, err := r.targets() + t, err := r.topLevelTargets() if err != nil { return err } @@ -233,7 +211,7 @@ func (r *Repo) SetTargetsVersion(v int) error { } func (r *Repo) TargetsVersion() (int, error) { - t, err := r.targets() + t, err := r.topLevelTargets() if err != nil { return -1, err } @@ -277,18 +255,22 @@ func (r *Repo) SnapshotVersion() (int, error) { return s.Version, nil } -func (r *Repo) targets() (*data.Targets, error) { - targetsJSON, ok := r.meta["targets.json"] +func (r *Repo) topLevelTargets() (*data.Targets, error) { + return r.targets("targets") +} + +func (r *Repo) targets(roleName string) (*data.Targets, error) { + targetsJSON, ok := r.meta[roleName+".json"] if !ok { return data.NewTargets(), nil } s := &data.Signed{} if err := json.Unmarshal(targetsJSON, s); err != nil { - return nil, err + return nil, fmt.Errorf("error unmarshalling for targets %q: %w", roleName, err) } targets := &data.Targets{} if err := json.Unmarshal(s.Signed, targets); err != nil { - return nil, err + return nil, fmt.Errorf("error unmarshalling signed data for targets %q: %w", roleName, err) } return targets, nil } @@ -310,10 +292,14 @@ func (r *Repo) timestamp() (*data.Timestamp, error) { } func (r *Repo) GenKey(role string) ([]string, error) { + // Not compatible with delegated roles. + return r.GenKeyWithExpires(role, data.DefaultExpires("root")) } func (r *Repo) GenKeyWithExpires(keyRole string, expires time.Time) (keyids []string, err error) { + // Not compatible with delegated roles. + signer, err := keys.GenerateEd25519Key() if err != nil { return []string{}, err @@ -327,11 +313,14 @@ func (r *Repo) GenKeyWithExpires(keyRole string, expires time.Time) (keyids []st } func (r *Repo) AddPrivateKey(role string, signer keys.Signer) error { + // Not compatible with delegated roles. + return r.AddPrivateKeyWithExpires(role, signer, data.DefaultExpires(role)) } func (r *Repo) AddPrivateKeyWithExpires(keyRole string, signer keys.Signer, expires time.Time) error { - if !verify.ValidRole(keyRole) { + // Not compatible with delegated roles. + if !roles.IsTopLevelRole(keyRole) { return ErrInvalidRole{keyRole} } @@ -351,10 +340,13 @@ func (r *Repo) AddPrivateKeyWithExpires(keyRole string, signer keys.Signer, expi } func (r *Repo) AddVerificationKey(keyRole string, pk *data.PublicKey) error { + // Not compatible with delegated roles. + return r.AddVerificationKeyWithExpiration(keyRole, pk, data.DefaultExpires(keyRole)) } func (r *Repo) AddVerificationKeyWithExpiration(keyRole string, pk *data.PublicKey, expires time.Time) error { + // Not compatible with delegated roles. root, err := r.root() if err != nil { return err @@ -426,11 +418,15 @@ func (r *Repo) RootKeys() ([]*data.PublicKey, error) { } func (r *Repo) RevokeKey(role, id string) error { + // Not compatible with delegated roles. + return r.RevokeKeyWithExpires(role, id, data.DefaultExpires("root")) } func (r *Repo) RevokeKeyWithExpires(keyRole, id string, expires time.Time) error { - if !verify.ValidRole(keyRole) { + // Not compatible with delegated roles. + + if !roles.IsTopLevelRole(keyRole) { return ErrInvalidRole{keyRole} } @@ -493,6 +489,106 @@ func (r *Repo) RevokeKeyWithExpires(keyRole, id string, expires time.Time) error return r.setMeta("root.json", root) } +// AddTargetsDelegation is equivalent to AddTargetsDelegationWithExpires, but +// with a default expiration time. +func (r *Repo) AddTargetsDelegation(delegator string, role data.DelegatedRole, keys []*data.PublicKey) error { + return r.AddTargetsDelegationWithExpires(delegator, role, keys, data.DefaultExpires("targets")) +} + +// AddTargetsDelegationWithExpires adds a delegation from the delegator to the +// role specified in the role argument. Key IDs referenced in role.KeyIDs +// should have corresponding Key entries in the keys argument. New metadata is +// written with the given expiration time. +func (r *Repo) AddTargetsDelegationWithExpires(delegator string, role data.DelegatedRole, keys []*data.PublicKey, expires time.Time) error { + t, err := r.targets(delegator) + if err != nil { + return fmt.Errorf("error getting delegator (%q) metadata: %w", delegator, err) + } + + if t.Delegations == nil { + t.Delegations = &data.Delegations{} + } + + t.Delegations.Keys = make(map[string]*data.PublicKey) + for _, keyID := range role.KeyIDs { + keyLoop: + for _, key := range keys { + if key.ContainsID(keyID) { + t.Delegations.Keys[keyID] = key + break keyLoop + } + } + } + + t.Delegations.Roles = append(t.Delegations.Roles, role) + t.Expires = expires.Round(time.Second) + + delegatorFile := delegator + ".json" + if _, ok := r.versionUpdated[delegatorFile]; !ok { + t.Version++ + r.versionUpdated[delegatorFile] = struct{}{} + } + + err = r.setMeta(delegatorFile, t) + if err != nil { + return fmt.Errorf("error setting metadata for %q: %w", delegatorFile, err) + } + + delegatee := role.Name + dt, err := r.targets(delegatee) + if err != nil { + return fmt.Errorf("error getting delegatee (%q) metadata: %w", delegatee, err) + } + + delegateeFile := delegatee + ".json" + if _, ok := r.versionUpdated[delegateeFile]; !ok { + dt.Version++ + r.versionUpdated[delegateeFile] = struct{}{} + } + err = r.setMeta(delegateeFile, dt) + if err != nil { + return fmt.Errorf("error setting metadata for %q: %w", delegateeFile, err) + } + + return nil +} + +// AddTargetsDelegationsForPathHashBins is equivalent to +// AddTargetsDelegationsForPathHashBinsWithExpires, but with a default +// expiration time. +func (r *Repo) AddTargetsDelegationsForPathHashBins(delegator string, binRolePrefix string, log2NumBins uint8, keys []*data.PublicKey, threshold int) error { + return r.AddTargetsDelegationsForPathHashBinsWithExpires(delegator, binRolePrefix, log2NumBins, keys, threshold, data.DefaultExpires("targets")) +} + +// AddTargetsDelegationsForPathHashBinsWithExpires adds 2^(log2NumBins) +// delegations to the delegator role, which partition the target path hash +// space into bins using the PathHashPrefixes delegation mechanism. New +// metadata is written with the given expiration time. +func (r *Repo) AddTargetsDelegationsForPathHashBinsWithExpires(delegator string, binRolePrefix string, log2NumBins uint8, keys []*data.PublicKey, threshold int, expires time.Time) error { + bins := targets.GenerateHashBins(log2NumBins) + padWidth := targets.HashPrefixLength(log2NumBins) + + keyIDs := []string{} + for _, key := range keys { + keyIDs = append(keyIDs, key.IDs()...) + } + + for _, bin := range bins { + name := bin.Name(binRolePrefix, padWidth) + err := r.AddTargetsDelegationWithExpires(delegator, data.DelegatedRole{ + Name: name, + KeyIDs: keyIDs, + PathHashPrefixes: bin.Enumerate(padWidth), + Threshold: threshold, + }, keys, expires) + if err != nil { + return fmt.Errorf("error adding delegation from %v to %v: %w", delegator, name, err) + } + } + + return nil +} + func (r *Repo) jsonMarshal(v interface{}) ([]byte, error) { b, err := cjson.Marshal(v) if err != nil { @@ -512,11 +608,20 @@ func (r *Repo) jsonMarshal(v interface{}) ([]byte, error) { } func (r *Repo) setMeta(roleFilename string, meta interface{}) error { - keys, err := r.getSortedSigningKeys(strings.TrimSuffix(roleFilename, ".json")) + db, err := r.topLevelKeysDB() if err != nil { return err } - s, err := sign.Marshal(meta, keys...) + signers, err := r.getSignersInDB(strings.TrimSuffix(roleFilename, ".json"), db) + if err != nil { + return err + } + return r.setMetaWithSigners(roleFilename, meta, signers) +} + +func (r *Repo) setMetaWithSigners(roleFilename string, meta interface{}, signers []keys.Signer) error { + fmt.Println("signing", roleFilename, signers) + s, err := sign.Marshal(meta, signers...) if err != nil { return err } @@ -530,7 +635,7 @@ func (r *Repo) setMeta(roleFilename string, meta interface{}) error { func (r *Repo) Sign(roleFilename string) error { role := strings.TrimSuffix(roleFilename, ".json") - if !verify.ValidRole(role) { + if !roles.IsTopLevelRole(role) { return ErrInvalidRole{role} } @@ -539,7 +644,11 @@ func (r *Repo) Sign(roleFilename string) error { return err } - keys, err := r.getSortedSigningKeys(role) + db, err := r.topLevelKeysDB() + if err != nil { + return err + } + keys, err := r.getSignersInDB(role, db) if err != nil { return err } @@ -562,12 +671,12 @@ func (r *Repo) Sign(roleFilename string) error { // The name must be a valid metadata file name, like root.json. func (r *Repo) AddOrUpdateSignature(roleFilename string, signature data.Signature) error { role := strings.TrimSuffix(roleFilename, ".json") - if !verify.ValidRole(role) { + if !roles.IsTopLevelRole(role) { return ErrInvalidRole{role} } // Check key ID is in valid for the role. - db, err := r.db() + db, err := r.topLevelKeysDB() if err != nil { return err } @@ -611,46 +720,46 @@ func (r *Repo) AddOrUpdateSignature(roleFilename string, signature data.Signatur return r.local.SetMeta(roleFilename, b) } -// getSortedSigningKeys returns available signing keys, sorted by key ID. +// getSignersInDB returns available signing interfaces, sorted by key ID. // // Only keys contained in the keys db are returned (i.e. local keys which have // been revoked are omitted), except for the root role in which case all local // keys are returned (revoked root keys still need to sign new root metadata so // clients can verify the new root.json and update their keys db accordingly). -func (r *Repo) getSortedSigningKeys(name string) ([]keys.Signer, error) { - signingKeys, err := r.local.GetSigners(name) +func (r *Repo) getSignersInDB(roleName string, db *verify.DB) ([]keys.Signer, error) { + fmt.Println("getting signing keys for", roleName) + signers, err := r.local.GetSigners(roleName) if err != nil { return nil, err } - if name == "root" { - sorted := make([]keys.Signer, len(signingKeys)) - copy(sorted, signingKeys) + + if roleName == "root" { + sorted := make([]keys.Signer, len(signers)) + copy(sorted, signers) sort.Sort(signer.ByIDs(sorted)) return sorted, nil } - db, err := r.db() - if err != nil { - return nil, err - } - role := db.GetRole(name) + + role := db.GetRole(roleName) if role == nil { return nil, nil } if len(role.KeyIDs) == 0 { return nil, nil } - keys := make([]keys.Signer, 0, len(role.KeyIDs)) - for _, key := range signingKeys { - for _, id := range key.PublicData().IDs() { + + signersInDB := make([]keys.Signer, 0, len(role.KeyIDs)) + for _, s := range signers { + for _, id := range s.PublicData().IDs() { if _, ok := role.KeyIDs[id]; ok { - keys = append(keys, key) + signersInDB = append(signersInDB, s) } } } - sort.Sort(signer.ByIDs(keys)) + sort.Sort(signer.ByIDs(signersInDB)) - return keys, nil + return signersInDB, nil } // Used to retrieve the signable portion of the metadata when using an external signing tool. @@ -666,13 +775,40 @@ func (r *Repo) SignedMeta(roleFilename string) (*data.Signed, error) { return s, nil } -func validMetadata(roleFilename string) bool { - for _, m := range topLevelMetadata { - if m == roleFilename { - return true +func (r *Repo) targetDelegationForPath(path string) (*data.Targets, *targets.Delegation, error) { + topLevelKeysDB, err := r.topLevelKeysDB() + if err != nil { + return nil, nil, err + } + + iterator := targets.NewDelegationsIterator(path, topLevelKeysDB) + for i := 0; i < defaultMaxDelegations; i++ { + d, ok := iterator.Next() + if !ok { + return nil, nil, ErrNoDelegatedTarget{Path: path} } + + targetsMeta, err := r.targets(d.Delegatee.Name) + if err != nil { + return nil, nil, err + } + fmt.Printf("role: %+v\n", targetsMeta) + fmt.Printf("role.Delegations: %+v\n", targetsMeta.Delegations) + fmt.Printf("d: %+v\n", d) + fmt.Println() + + if targetsMeta.Delegations == nil || len(targetsMeta.Delegations.Roles) == 0 { + return targetsMeta, &d, nil + } + + db, err := verify.NewDBFromDelegations(targetsMeta.Delegations) + if err != nil { + return nil, nil, err + } + iterator.Add(targetsMeta.Delegations.Roles, d.Delegatee.Name, db) } - return false + + return nil, nil, ErrNoDelegatedTarget{Path: path} } func (r *Repo) AddTarget(path string, custom json.RawMessage) error { @@ -687,20 +823,57 @@ func (r *Repo) AddTargetWithExpires(path string, custom json.RawMessage, expires return r.AddTargetsWithExpires([]string{path}, custom, expires) } +type targetsMetaWithKeyDB struct { + meta *data.Targets + db *verify.DB +} + func (r *Repo) AddTargetsWithExpires(paths []string, custom json.RawMessage, expires time.Time) error { if !validExpires(expires) { return ErrInvalidExpires{expires} } - t, err := r.targets() - if err != nil { - return err - } normalizedPaths := make([]string, len(paths)) for i, path := range paths { normalizedPaths[i] = util.NormalizeTarget(path) } + + targetsMetaToWrite := map[string]*targetsMetaWithKeyDB{} + if err := r.local.WalkStagedTargets(normalizedPaths, func(path string, target io.Reader) (err error) { + targetsMeta, delegation, err := r.targetDelegationForPath(path) + if err != nil { + return err + } + targetsRoleName := delegation.Delegatee.Name + + twk := &targetsMetaWithKeyDB{ + meta: targetsMeta, + db: delegation.DB, + } + + // We accumulate changes in the targets manifests staged in + // targetsMetaToWrite. If we've already visited a roleName in the + // WalkStagedTargets iteration, use the staged metadata instead of the + // fresh metadata from targetRoleForPath. + if seenMetaWithKeys, ok := targetsMetaToWrite[targetsRoleName]; ok { + // Merge the seen keys with the keys for the new target. If all + // delegations to role.Name use the same keys (probably the most common + // case with TUF) the merge is a no-op. + // seenKeys := sets.StringSliceToSet(seenMetaWithKeys.keyIDs) + // mergedKeys := seenMetaWithKeys.keyIDs + + // for _, keyID := range twk.keyIDs { + // if _, ok := seenKeys[keyID]; !ok { + // mergedKeys = append(mergedKeys, keyID) + // seenKeys[keyID] = struct{}{} + // } + // } + + // seenMetaWithKeys.keyIDs = mergedKeys + twk = seenMetaWithKeys + } + meta, err := util.GenerateTargetFileMeta(target, r.hashAlgorithms...) if err != nil { return err @@ -711,23 +884,79 @@ func (r *Repo) AddTargetsWithExpires(paths []string, custom json.RawMessage, exp // existing metadata if present if len(custom) > 0 { meta.Custom = &custom - } else if t, ok := t.Targets[path]; ok { - meta.Custom = t.Custom + } else if tf, ok := twk.meta.Targets[path]; ok { + meta.Custom = tf.Custom } // G2 -> we no longer desire any readers to ever observe non-prefix targets. - delete(t.Targets, "/"+path) - t.Targets[path] = meta + delete(twk.meta.Targets, "/"+path) + twk.meta.Targets[path] = meta + + targetsMetaToWrite[targetsRoleName] = twk + return nil }); err != nil { return err } - t.Expires = expires.Round(time.Second) - if _, ok := r.versionUpdated["targets.json"]; !ok { - t.Version++ - r.versionUpdated["targets.json"] = struct{}{} + + if len(targetsMetaToWrite) == 0 { + t, err := r.topLevelTargets() + if err != nil { + return err + } + + db, err := r.topLevelKeysDB() + if err != nil { + return err + } + + targetsMetaToWrite["targets"] = &targetsMetaWithKeyDB{ + meta: t, + db: db, + } } - return r.setMeta("targets.json", t) + + exp := expires.Round(time.Second) + for roleName, twk := range targetsMetaToWrite { + twk.meta.Expires = exp + + manifestName := roleName + ".json" + if _, ok := r.versionUpdated[manifestName]; !ok { + twk.meta.Version++ + r.versionUpdated[manifestName] = struct{}{} + } + + // signers := r.local.SignersForKeyIDs(twk.keyIDs) + // signers, err := r.local.SignersForRole(roleName) + // if err != nil { + // return err + // } + + // db, err := verify.NewDBFromDelegations(hh) + // if err != nil { + // return err + // } + signers, err := r.getSignersInDB(roleName, twk.db) + if err != nil { + return err + } + + fmt.Println("signers for", roleName, "are", signers) + fmt.Println("writing to manifest", manifestName) + + err = r.setMetaWithSigners(manifestName, twk.meta, signers) + if err != nil { + return err + } + + // var err error + // err = r.setMeta(manifestName, twk.meta) + // if err != nil { + // return err + // } + } + + return nil } func (r *Repo) RemoveTarget(path string) error { @@ -748,7 +977,7 @@ func (r *Repo) RemoveTargetsWithExpires(paths []string, expires time.Time) error return ErrInvalidExpires{expires} } - t, err := r.targets() + t, err := r.topLevelTargets() if err != nil { return err } @@ -782,6 +1011,19 @@ func (r *Repo) Snapshot() error { return r.SnapshotWithExpires(data.DefaultExpires("snapshot")) } +func (r *Repo) snapshotManifests() []string { + ret := []string{"root.json", "targets.json"} + + for name := range r.meta { + if !roles.IsVersionedManifest(name) && + roles.IsDelegatedTargetsManifest(name) { + ret = append(ret, name) + } + } + + return ret +} + func (r *Repo) SnapshotWithExpires(expires time.Time) error { if !validExpires(expires) { return ErrInvalidExpires{expires} @@ -791,15 +1033,16 @@ func (r *Repo) SnapshotWithExpires(expires time.Time) error { if err != nil { return err } - db, err := r.db() - if err != nil { - return err - } + // db, err := r.topLevelKeysDB() + // if err != nil { + // return err + // } - for _, name := range snapshotMetadata { - if err := r.verifySignature(name, db); err != nil { - return err - } + for _, name := range r.snapshotManifests() { + fmt.Println("snapshotting", name) + // if err := r.verifySignature(name, db); err != nil { + // return err + // } var err error snapshot.Meta[name], err = r.snapshotFileMeta(name) if err != nil { @@ -823,7 +1066,7 @@ func (r *Repo) TimestampWithExpires(expires time.Time) error { return ErrInvalidExpires{expires} } - db, err := r.db() + db, err := r.topLevelKeysDB() if err != nil { return err } @@ -847,51 +1090,96 @@ func (r *Repo) TimestampWithExpires(expires time.Time) error { } func (r *Repo) fileVersions() (map[string]int, error) { - root, err := r.root() - if err != nil { - return nil, err - } - targets, err := r.targets() - if err != nil { - return nil, err - } - snapshot, err := r.snapshot() - if err != nil { - return nil, err - } versions := make(map[string]int) - versions["root.json"] = root.Version - versions["targets.json"] = targets.Version - versions["snapshot.json"] = snapshot.Version + + for fileName := range r.meta { + if roles.IsVersionedManifest(fileName) { + continue + } + + roleName := strings.TrimSuffix(fileName, ".json") + + var version int + + switch roleName { + case "root": + root, err := r.root() + if err != nil { + return nil, err + } + version = root.Version + case "snapshot": + snapshot, err := r.snapshot() + if err != nil { + return nil, err + } + version = snapshot.Version + case "timestamp": + continue + default: + // Targets or delegated targets manifest. + targets, err := r.targets(roleName) + if err != nil { + return nil, err + } + + version = targets.Version + } + + versions[fileName] = version + } + return versions, nil } func (r *Repo) fileHashes() (map[string]data.Hashes, error) { hashes := make(map[string]data.Hashes) - timestamp, err := r.timestamp() - if err != nil { - return nil, err - } - snapshot, err := r.snapshot() - if err != nil { - return nil, err - } - if m, ok := snapshot.Meta["root.json"]; ok { - hashes["root.json"] = m.Hashes - } - if m, ok := snapshot.Meta["targets.json"]; ok { - hashes["targets.json"] = m.Hashes - } - if m, ok := timestamp.Meta["snapshot.json"]; ok { - hashes["snapshot.json"] = m.Hashes - } - t, err := r.targets() - if err != nil { - return nil, err - } - for name, meta := range t.Targets { - hashes[path.Join("targets", name)] = meta.Hashes + + for fileName := range r.meta { + if roles.IsVersionedManifest(fileName) { + continue + } + + roleName := strings.TrimSuffix(fileName, ".json") + + switch roleName { + case "snapshot": + timestamp, err := r.timestamp() + if err != nil { + return nil, err + } + + if m, ok := timestamp.Meta[fileName]; ok { + hashes[fileName] = m.Hashes + } + case "timestamp": + continue + default: + snapshot, err := r.snapshot() + if err != nil { + return nil, err + } + if m, ok := snapshot.Meta[fileName]; ok { + hashes[fileName] = m.Hashes + } + + // FIXME: Loading all targets into memory is not scalable if + // there are many targets. This is used to Commit, so we should + // only need new targets here. + if roleName != "root" { + t, err := r.targets(roleName) + if err != nil { + return nil, err + } + for name, m := range t.Targets { + hashes[path.Join("targets", name)] = m.Hashes + } + } + + } + } + return hashes, nil } @@ -919,7 +1207,7 @@ func (r *Repo) Commit() error { if err != nil { return err } - for _, name := range snapshotMetadata { + for _, name := range r.snapshotManifests() { expected, ok := snapshot.Meta[name] if !ok { return fmt.Errorf("tuf: snapshot.json missing hash for %s", name) @@ -947,7 +1235,7 @@ func (r *Repo) Commit() error { } // verify all signatures are correct - db, err := r.db() + db, err := r.topLevelKeysDB() if err != nil { return err } @@ -986,6 +1274,7 @@ func (r *Repo) verifySignature(roleFilename string, db *verify.DB) error { if err != nil { return err } + role := strings.TrimSuffix(roleFilename, ".json") if err := db.Verify(s, role, 0); err != nil { return ErrInsufficientSignatures{roleFilename, err} diff --git a/repo_test.go b/repo_test.go index ab4877ac..52b71447 100644 --- a/repo_test.go +++ b/repo_test.go @@ -18,6 +18,7 @@ import ( cjson "github.com/tent/canonical-json-go" "github.com/theupdateframework/go-tuf/data" "github.com/theupdateframework/go-tuf/encrypted" + "github.com/theupdateframework/go-tuf/internal/sets" "github.com/theupdateframework/go-tuf/pkg/keys" "github.com/theupdateframework/go-tuf/util" "github.com/theupdateframework/go-tuf/verify" @@ -126,7 +127,7 @@ func testNewRepo(c *C, newRepo func(local LocalStore, hashAlgorithms ...string) c.Assert(root.Keys, NotNil) c.Assert(root.Keys, HasLen, 0) - targets, err := r.targets() + targets, err := r.topLevelTargets() c.Assert(err, IsNil) c.Assert(targets.Type, Equals, "targets") c.Assert(targets.Version, Equals, 1) @@ -212,14 +213,14 @@ func (rs *RepoSuite) TestGenKey(c *C) { } // check root key + role are in db - db, err := r.db() + db, err := r.topLevelKeysDB() c.Assert(err, IsNil) for _, keyID := range ids { rootKey, err := db.GetVerifier(keyID) c.Assert(err, IsNil) c.Assert(rootKey.MarshalPublicKey().IDs(), DeepEquals, ids) role := db.GetRole("root") - c.Assert(role.KeyIDs, DeepEquals, util.StringSliceToSet(ids)) + c.Assert(role.KeyIDs, DeepEquals, sets.StringSliceToSet(ids)) // check the key was saved correctly localKeys, err := local.GetSigners("root") @@ -256,7 +257,7 @@ func (rs *RepoSuite) TestGenKey(c *C) { } c.Assert(targetsRole.KeyIDs, HasLen, 2) targetKeyIDs := make(map[string]struct{}, 2) - db, err = r.db() + db, err = r.topLevelKeysDB() c.Assert(err, IsNil) for _, id := range targetsRole.KeyIDs { targetKeyIDs[id] = struct{}{} @@ -375,14 +376,14 @@ func (rs *RepoSuite) TestAddPrivateKey(c *C) { } // check root key + role are in db - db, err := r.db() + db, err := r.topLevelKeysDB() c.Assert(err, IsNil) for _, keyID := range ids { rootKey, err := db.GetVerifier(keyID) c.Assert(err, IsNil) c.Assert(rootKey.MarshalPublicKey().IDs(), DeepEquals, ids) role := db.GetRole("root") - c.Assert(role.KeyIDs, DeepEquals, util.StringSliceToSet(ids)) + c.Assert(role.KeyIDs, DeepEquals, sets.StringSliceToSet(ids)) // check the key was saved correctly localKeys, err := local.GetSigners("root") @@ -419,7 +420,7 @@ func (rs *RepoSuite) TestAddPrivateKey(c *C) { } c.Assert(targetsRole.KeyIDs, HasLen, 2) targetKeyIDs := make(map[string]struct{}, 2) - db, err = r.db() + db, err = r.topLevelKeysDB() c.Assert(err, IsNil) for _, id := range targetsRole.KeyIDs { targetKeyIDs[id] = struct{}{} @@ -934,7 +935,7 @@ func (rs *RepoSuite) TestCommitFileSystem(c *C) { c.Assert(r.AddTarget("foo.txt", nil), IsNil) tmp.assertExists("staged/targets.json") tmp.assertEmpty("repository") - t, err := r.targets() + t, err := r.topLevelTargets() c.Assert(err, IsNil) c.Assert(t.Targets, HasLen, 1) if _, ok := t.Targets["foo.txt"]; !ok { @@ -1085,7 +1086,7 @@ func (rs *RepoSuite) TestConsistentSnapshot(c *C) { // targets should be returned by new repo newRepo, err := NewRepo(local, "sha512", "sha256") c.Assert(err, IsNil) - t, err := newRepo.targets() + t, err := newRepo.topLevelTargets() c.Assert(err, IsNil) c.Assert(t.Targets, HasLen, 1) if _, ok := t.Targets["dir/bar.txt"]; !ok { @@ -1156,7 +1157,7 @@ func (rs *RepoSuite) TestExpiresAndVersion(c *C) { c.Assert(r.Snapshot(), IsNil) c.Assert(r.Timestamp(), IsNil) c.Assert(r.Commit(), IsNil) - targets, err := r.targets() + targets, err := r.topLevelTargets() c.Assert(err, IsNil) c.Assert(targets.Expires.Unix(), Equals, expires.Round(time.Second).Unix()) c.Assert(targets.Version, Equals, 2) @@ -1166,7 +1167,7 @@ func (rs *RepoSuite) TestExpiresAndVersion(c *C) { c.Assert(r.Snapshot(), IsNil) c.Assert(r.Timestamp(), IsNil) c.Assert(r.Commit(), IsNil) - targets, err = r.targets() + targets, err = r.topLevelTargets() c.Assert(err, IsNil) c.Assert(targets.Expires.Unix(), Equals, expires.Round(time.Second).Unix()) c.Assert(targets.Version, Equals, 3) @@ -1234,7 +1235,7 @@ func (rs *RepoSuite) TestHashAlgorithm(c *C) { if test.expected == nil { test.expected = test.args } - targets, err := r.targets() + targets, err := r.topLevelTargets() c.Assert(err, IsNil) snapshot, err := r.snapshot() c.Assert(err, IsNil) @@ -1379,7 +1380,7 @@ func (rs *RepoSuite) TestManageMultipleTargets(c *C) { genKey(c, r, "timestamp") assertRepoTargets := func(paths ...string) { - t, err := r.targets() + t, err := r.topLevelTargets() c.Assert(err, IsNil) for _, path := range paths { if _, ok := t.Targets[path]; !ok { @@ -1425,7 +1426,7 @@ func (rs *RepoSuite) TestManageMultipleTargets(c *C) { c.Assert(r.Timestamp(), IsNil) c.Assert(r.Commit(), IsNil) tmp.assertEmpty("repository/targets") - t, err := r.targets() + t, err := r.topLevelTargets() c.Assert(err, IsNil) c.Assert(t.Targets, HasLen, 0) } @@ -1442,7 +1443,7 @@ func (rs *RepoSuite) TestCustomTargetMetadata(c *C) { custom := json.RawMessage(`{"foo":"bar"}`) assertCustomMeta := func(file string, custom *json.RawMessage) { - t, err := r.targets() + t, err := r.topLevelTargets() c.Assert(err, IsNil) target, ok := t.Targets[file] if !ok { @@ -1720,7 +1721,7 @@ func (rs *RepoSuite) TestBadAddOrUpdateSignatures(c *C) { checkSigIDs := func(role string) { s, err := r.SignedMeta(role) c.Assert(err, IsNil) - db, err := r.db() + db, err := r.topLevelKeysDB() c.Assert(err, IsNil) // keys is a map of key IDs. keys := db.GetRole(strings.TrimSuffix(role, ".json")).KeyIDs diff --git a/util/util.go b/util/util.go index 751c677e..efd335d7 100644 --- a/util/util.go +++ b/util/util.go @@ -258,14 +258,6 @@ func HashedPaths(p string, hashes data.Hashes) []string { return paths } -func StringSliceToSet(items []string) map[string]struct{} { - s := make(map[string]struct{}) - for _, item := range items { - s[item] = struct{}{} - } - return s -} - func AtomicallyWriteFile(filename string, data []byte, perm os.FileMode) error { dir, name := filepath.Split(filename) f, err := ioutil.TempFile(dir, name) diff --git a/verify/db.go b/verify/db.go index 0684e6a8..657c2c55 100644 --- a/verify/db.go +++ b/verify/db.go @@ -2,6 +2,7 @@ package verify import ( "github.com/theupdateframework/go-tuf/data" + "github.com/theupdateframework/go-tuf/internal/roles" "github.com/theupdateframework/go-tuf/pkg/keys" ) @@ -27,37 +28,28 @@ func NewDB() *DB { } } -type DelegationsVerifier struct { - DB *DB -} - -func (d *DelegationsVerifier) Unmarshal(b []byte, v interface{}, role string, minVersion int) error { - return d.DB.Unmarshal(b, v, role, minVersion) -} - -// NewDelegationsVerifier returns a DelegationsVerifier that verifies delegations -// of a given Targets. It reuses the DB struct to leverage verified keys, roles -// unmarshals. -func NewDelegationsVerifier(d *data.Delegations) (DelegationsVerifier, error) { +// NewDBFromDelegations returns a DB that verifies delegations +// of a given Targets. +func NewDBFromDelegations(d *data.Delegations) (*DB, error) { db := &DB{ roles: make(map[string]*Role, len(d.Roles)), verifiers: make(map[string]keys.Verifier, len(d.Keys)), } for _, r := range d.Roles { - if _, ok := topLevelRoles[r.Name]; ok { - return DelegationsVerifier{}, ErrInvalidDelegatedRole + if _, ok := roles.TopLevelRoles[r.Name]; ok { + return nil, ErrInvalidDelegatedRole } role := &data.Role{Threshold: r.Threshold, KeyIDs: r.KeyIDs} - if err := db.addRole(r.Name, role); err != nil { - return DelegationsVerifier{}, err + if err := db.AddRole(r.Name, role); err != nil { + return nil, err } } for id, k := range d.Keys { if err := db.AddKey(id, k); err != nil { - return DelegationsVerifier{}, err + return nil, err } } - return DelegationsVerifier{db}, nil + return db, nil } func (db *DB) AddKey(id string, k *data.PublicKey) error { @@ -72,31 +64,7 @@ func (db *DB) AddKey(id string, k *data.PublicKey) error { return nil } -var topLevelRoles = map[string]struct{}{ - "root": {}, - "targets": {}, - "snapshot": {}, - "timestamp": {}, -} - -// ValidRole checks if a role is a top level role. -func ValidRole(name string) bool { - return isTopLevelRole(name) -} - -func isTopLevelRole(name string) bool { - _, ok := topLevelRoles[name] - return ok -} - func (db *DB) AddRole(name string, r *data.Role) error { - if !isTopLevelRole(name) { - return ErrInvalidRole - } - return db.addRole(name, r) -} - -func (db *DB) addRole(name string, r *data.Role) error { if r.Threshold < 1 { return ErrInvalidThreshold } diff --git a/verify/db_test.go b/verify/db_test.go index cb328981..e4bab0a3 100644 --- a/verify/db_test.go +++ b/verify/db_test.go @@ -7,8 +7,8 @@ import ( "github.com/theupdateframework/go-tuf/data" ) -func TestDelegationsVerifier(t *testing.T) { - var verifierTests = []struct { +func TestDelegationsDB(t *testing.T) { + var dbTests = []struct { testName string delegations *data.Delegations initErr error @@ -42,14 +42,14 @@ func TestDelegationsVerifier(t *testing.T) { }, } - for _, tt := range verifierTests { + for _, tt := range dbTests { t.Run(tt.testName, func(t *testing.T) { - verifier, err := NewDelegationsVerifier(tt.delegations) - assert.NotNil(t, verifier) + db, err := NewDBFromDelegations(tt.delegations) assert.Equal(t, tt.initErr, err) if err == nil { + assert.NotNil(t, db) var targets data.Targets - err = verifier.Unmarshal([]byte(`{"a":"b"}`), targets, "tree", 0) + err = db.Unmarshal([]byte(`{"a":"b"}`), targets, "tree", 0) assert.Equal(t, tt.unmarshalErr, err) } }) diff --git a/verify/verify.go b/verify/verify.go index c0b6a037..371a91b0 100644 --- a/verify/verify.go +++ b/verify/verify.go @@ -7,6 +7,7 @@ import ( cjson "github.com/tent/canonical-json-go" "github.com/theupdateframework/go-tuf/data" + "github.com/theupdateframework/go-tuf/internal/roles" ) type signedMeta struct { @@ -25,7 +26,7 @@ func (db *DB) VerifyIgnoreExpiredCheck(s *data.Signed, role string, minVersion i return err } - if isTopLevelRole(role) { + if roles.IsTopLevelRole(role) { // Top-level roles can only sign metadata of the same type (e.g. snapshot // metadata must be signed by the snapshot role). if strings.ToLower(sm.Type) != strings.ToLower(role) {