From 60357969c7c7bd11875416229b0a4c055d7d7e92 Mon Sep 17 00:00:00 2001 From: Matthew Jeffryes Date: Tue, 14 Nov 2023 11:35:38 -0800 Subject: [PATCH 1/4] Use zero copy for even faster json deserialization --- go.mod | 2 +- unstable/metadata/metadata.go | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index cc7150a9d..18a402700 100644 --- a/go.mod +++ b/go.mod @@ -208,7 +208,7 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect github.com/segmentio/asm v1.1.3 // indirect - github.com/segmentio/encoding v0.3.5 // indirect + github.com/segmentio/encoding v0.3.5 github.com/sergi/go-diff v1.3.1 // indirect github.com/skeema/knownhosts v1.2.0 // indirect github.com/spf13/cast v1.5.0 // indirect diff --git a/unstable/metadata/metadata.go b/unstable/metadata/metadata.go index 00b726e5d..12deaa6c2 100644 --- a/unstable/metadata/metadata.go +++ b/unstable/metadata/metadata.go @@ -15,9 +15,8 @@ package metadata import ( - "encoding/json" - "github.com/json-iterator/go" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/segmentio/encoding/json" ) // The underlying value of a metadata blob. @@ -26,12 +25,12 @@ type Data struct{ m map[string]*json.RawMessage } func New(data []byte) (*Data, error) { m := map[string]*json.RawMessage{} if len(data) > 0 { - jsoni := jsoniter.ConfigCompatibleWithStandardLibrary - err := jsoni.Unmarshal(data, &m) + _, err := json.Parse(data, &m, json.ZeroCopy) if err != nil { return nil, err } } + return &Data{m}, nil } @@ -59,8 +58,7 @@ func Set(d *Data, key string, value any) error { delete(d.m, key) return nil } - jsoni := jsoniter.ConfigCompatibleWithStandardLibrary - data, err := jsoni.Marshal(value) + data, err := json.Marshal(value) if err != nil { return err } @@ -75,8 +73,7 @@ func Get[T any](d *Data, key string) (T, bool, error) { if !ok { return t, false, nil } - jsoni := jsoniter.ConfigCompatibleWithStandardLibrary - err := jsoni.Unmarshal(*data, &t) + _, err := json.Parse(*data, &t, json.ZeroCopy) return t, true, err } From b553dca91dfa43cba1e9478e4c95eaf03da22848 Mon Sep 17 00:00:00 2001 From: Matthew Jeffryes Date: Wed, 15 Nov 2023 06:51:19 -0800 Subject: [PATCH 2/4] Generate a condensed metadata for runtime --- pkg/tfbridge/metadata.go | 9 +++++ pkg/tfbridge/tokens_test.go | 24 ++++++------ pkg/tfbridge/x/token_test.go | 16 ++++---- pkg/tfgen/generate.go | 6 ++- unstable/metadata/metadata.go | 62 +++++++++++++++++++++--------- unstable/metadata/metadata_test.go | 2 +- 6 files changed, 79 insertions(+), 40 deletions(-) diff --git a/pkg/tfbridge/metadata.go b/pkg/tfbridge/metadata.go index 8b5b45b8a..0df1e1cf6 100644 --- a/pkg/tfbridge/metadata.go +++ b/pkg/tfbridge/metadata.go @@ -61,3 +61,12 @@ func (info *MetadataInfo) assertValid() { contract.Assertf(info.Path != "", "Path must be non-empty") } + +// trim the metadata to just the keys required for the runtime phase +// in the future this method might also substitute compressed contents within some keys +func (info *MetadataInfo) ExtractRuntimeMetadata() *MetadataInfo { + data, _ := metadata.New(nil) + metadata.CloneKey("auto-aliasing", info.Data, data) + metadata.CloneKey("mux", info.Data, data) + return &MetadataInfo{"runtime-bridge-metadata.json", ProviderMetadata(data)} +} diff --git a/pkg/tfbridge/tokens_test.go b/pkg/tfbridge/tokens_test.go index 00ab80490..0e114b717 100644 --- a/pkg/tfbridge/tokens_test.go +++ b/pkg/tfbridge/tokens_test.go @@ -654,7 +654,7 @@ func TestMaxItemsOneAliasing(t *testing.T) { // Save current state into metadata autoAliasing(info, metadata) - v := string(metadata.Marshal()) + v := string(metadata.MarshalIndent()) expected := `{ "auto-aliasing": { "resources": { @@ -681,14 +681,14 @@ func TestMaxItemsOneAliasing(t *testing.T) { assert.True(t, *info.Resources["pkg_r1"].Fields["f1"].MaxItemsOne) assert.False(t, *info.Resources["pkg_r1"].Fields["f2"].MaxItemsOne) - assert.Equal(t, expected, string(metadata.Marshal())) + assert.Equal(t, expected, string(metadata.MarshalIndent())) // Apply metadata back into the provider again, making sure there isn't a diff autoAliasing(info, metadata) assert.True(t, *info.Resources["pkg_r1"].Fields["f1"].MaxItemsOne) assert.False(t, *info.Resources["pkg_r1"].Fields["f2"].MaxItemsOne) - assert.Equal(t, expected, string(metadata.Marshal())) + assert.Equal(t, expected, string(metadata.MarshalIndent())) // Validate that overrides work @@ -716,7 +716,7 @@ func TestMaxItemsOneAliasing(t *testing.T) { } } } -}`, string(metadata.Marshal())) +}`, string(metadata.MarshalIndent())) } func TestMaxItemsOneAliasingExpiring(t *testing.T) { @@ -741,7 +741,7 @@ func TestMaxItemsOneAliasingExpiring(t *testing.T) { // Save current state into metadata autoAliasing(info, metadata) - v := string(metadata.Marshal()) + v := string(metadata.MarshalIndent()) expected := `{ "auto-aliasing": { "resources": { @@ -786,7 +786,7 @@ func TestMaxItemsOneAliasingExpiring(t *testing.T) { } } } -}`, string(metadata.Marshal())) +}`, string(metadata.MarshalIndent())) } @@ -817,7 +817,7 @@ func TestMaxItemsOneAliasingNested(t *testing.T) { // Save current state into metadata autoAliasing(info, metadata) - v := string(metadata.Marshal()) + v := string(metadata.MarshalIndent()) expected := `{ "auto-aliasing": { "resources": { @@ -851,7 +851,7 @@ func TestMaxItemsOneAliasingNested(t *testing.T) { info = provider(false, true) autoAliasing(info, metadata) - assert.Equal(t, expected, string(metadata.Marshal())) + assert.Equal(t, expected, string(metadata.MarshalIndent())) assert.True(t, *info.Resources["pkg_r1"].Fields["f2"].Elem.Fields["n1"].MaxItemsOne) assert.False(t, *info.Resources["pkg_r1"].Fields["f2"].Elem.Fields["n2"].MaxItemsOne) } @@ -919,7 +919,7 @@ func TestMaxItemsOneAliasingWithAutoNaming(t *testing.T) { } } } - }`, string((*md.Data)(p.MetadataInfo.Data).Marshal())) + }`, string((*md.Data)(p.MetadataInfo.Data).MarshalIndent())) } t.Run("auto-named-then-aliased", func(t *testing.T) { @@ -1000,7 +1000,7 @@ func TestMaxItemsOneDataSourceAliasing(t *testing.T) { } } } - }`, string((*md.Data)(p.MetadataInfo.Data).Marshal())) + }`, string((*md.Data)(p.MetadataInfo.Data).MarshalIndent())) } t.Run("auto-named-then-aliased", func(t *testing.T) { @@ -1120,7 +1120,7 @@ func TestAutoAliasingChangeDataSources(t *testing.T) { return func(t *testing.T) { p := provider(t, current, name) require.JSONEq(t, expected, - string((*md.Data)(p.MetadataInfo.Data).Marshal())) + string((*md.Data)(p.MetadataInfo.Data).MarshalIndent())) // Regardless of the input and output, once we apply some name to // our state, reapplying the same name to the new state should be @@ -1128,7 +1128,7 @@ func TestAutoAliasingChangeDataSources(t *testing.T) { t.Run("idempotent", func(t *testing.T) { p := provider(t, expected, name) require.JSONEq(t, expected, - string((*md.Data)(p.MetadataInfo.Data).Marshal())) + string((*md.Data)(p.MetadataInfo.Data).MarshalIndent())) }) } } diff --git a/pkg/tfbridge/x/token_test.go b/pkg/tfbridge/x/token_test.go index 4499ed2d9..981d7c818 100644 --- a/pkg/tfbridge/x/token_test.go +++ b/pkg/tfbridge/x/token_test.go @@ -531,7 +531,7 @@ func TestMaxItemsOneAliasing(t *testing.T) { err = AutoAliasing(info, metadata) require.NoError(t, err) - v := string(metadata.Marshal()) + v := string(metadata.MarshalIndent()) expected := `{ "auto-aliasing": { "resources": { @@ -559,7 +559,7 @@ func TestMaxItemsOneAliasing(t *testing.T) { assert.True(t, *info.Resources["pkg_r1"].Fields["f1"].MaxItemsOne) assert.False(t, *info.Resources["pkg_r1"].Fields["f2"].MaxItemsOne) - assert.Equal(t, expected, string(metadata.Marshal())) + assert.Equal(t, expected, string(metadata.MarshalIndent())) // Apply metadata back into the provider again, making sure there isn't a diff err = AutoAliasing(info, metadata) @@ -567,7 +567,7 @@ func TestMaxItemsOneAliasing(t *testing.T) { assert.True(t, *info.Resources["pkg_r1"].Fields["f1"].MaxItemsOne) assert.False(t, *info.Resources["pkg_r1"].Fields["f2"].MaxItemsOne) - assert.Equal(t, expected, string(metadata.Marshal())) + assert.Equal(t, expected, string(metadata.MarshalIndent())) // Validate that overrides work @@ -596,7 +596,7 @@ func TestMaxItemsOneAliasing(t *testing.T) { } } } -}`, string(metadata.Marshal())) +}`, string(metadata.MarshalIndent())) } func TestMaxItemsOneAliasingExpiring(t *testing.T) { @@ -623,7 +623,7 @@ func TestMaxItemsOneAliasingExpiring(t *testing.T) { err = AutoAliasing(info, metadata) require.NoError(t, err) - v := string(metadata.Marshal()) + v := string(metadata.MarshalIndent()) expected := `{ "auto-aliasing": { "resources": { @@ -669,7 +669,7 @@ func TestMaxItemsOneAliasingExpiring(t *testing.T) { } } } -}`, string(metadata.Marshal())) +}`, string(metadata.MarshalIndent())) } @@ -702,7 +702,7 @@ func TestMaxItemsOneAliasingNested(t *testing.T) { err = AutoAliasing(info, metadata) require.NoError(t, err) - v := string(metadata.Marshal()) + v := string(metadata.MarshalIndent()) expected := `{ "auto-aliasing": { "resources": { @@ -737,7 +737,7 @@ func TestMaxItemsOneAliasingNested(t *testing.T) { err = AutoAliasing(info, metadata) require.NoError(t, err) - assert.Equal(t, expected, string(metadata.Marshal())) + assert.Equal(t, expected, string(metadata.MarshalIndent())) assert.True(t, *info.Resources["pkg_r1"].Fields["f2"].Elem.Fields["n1"].MaxItemsOne) assert.False(t, *info.Resources["pkg_r1"].Fields["f2"].Elem.Fields["n2"].MaxItemsOne) } diff --git a/pkg/tfgen/generate.go b/pkg/tfgen/generate.go index 73e70dec3..e15fbac99 100644 --- a/pkg/tfgen/generate.go +++ b/pkg/tfgen/generate.go @@ -946,7 +946,11 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe files = map[string][]byte{"schema.json": bytes} if info := g.info.MetadataInfo; info != nil { - files[info.Path] = (*metadata.Data)(info.Data).Marshal() + files[info.Path] = (*metadata.Data)(info.Data).MarshalIndent() + if true { + runtimeInfo := info.ExtractRuntimeMetadata() + files[runtimeInfo.Path] = (*metadata.Data)(runtimeInfo.Data).Marshal() + } } case PCL: if g.skipExamples { diff --git a/unstable/metadata/metadata.go b/unstable/metadata/metadata.go index 12deaa6c2..2a8354a43 100644 --- a/unstable/metadata/metadata.go +++ b/unstable/metadata/metadata.go @@ -20,10 +20,10 @@ import ( ) // The underlying value of a metadata blob. -type Data struct{ m map[string]*json.RawMessage } +type Data struct{ m map[string]json.RawMessage } func New(data []byte) (*Data, error) { - m := map[string]*json.RawMessage{} + m := map[string]json.RawMessage{} if len(data) > 0 { _, err := json.Parse(data, &m, json.ZeroCopy) if err != nil { @@ -35,10 +35,24 @@ func New(data []byte) (*Data, error) { } func (d *Data) Marshal() []byte { + return d.marshal(false) +} + +func (d *Data) MarshalIndent() []byte { + return d.marshal(true) +} + +func (d *Data) marshal(indent bool) []byte { if d == nil { - d = &Data{m: make(map[string]*json.RawMessage)} + d = &Data{m: make(map[string]json.RawMessage)} + } + var bytes []byte + var err error + if indent { + bytes, err = json.MarshalIndent(d.m, "", " ") + } else { + bytes, err = json.Marshal(d.m) } - bytes, err := json.MarshalIndent(d.m, "", " ") // `d.m` is a `map[string]json.RawMessage`. `json.MarshalIndent` errors only when // it is asked to serialize an unmarshalable type (complex, function or channel) // or a cyclic data structure. Because `string` and `json.RawMessage` are @@ -63,7 +77,7 @@ func Set(d *Data, key string, value any) error { return err } msg := json.RawMessage(data) - d.m[key] = &msg + d.m[key] = msg return nil } @@ -73,28 +87,40 @@ func Get[T any](d *Data, key string) (T, bool, error) { if !ok { return t, false, nil } - _, err := json.Parse(*data, &t, json.ZeroCopy) + _, err := json.Parse(data, &t, json.ZeroCopy) return t, true, err } +func CloneKey(key string, from, to *Data) { + data, ok := from.m[key] + if !ok { + delete(to.m, key) + return + } + to.m[key] = cloneRawMessage(data) +} + func Clone(data *Data) *Data { if data == nil { return nil } - m := make(map[string]*json.RawMessage, len(data.m)) + m := make(map[string]json.RawMessage, len(data.m)) for k, v := range data.m { - dst := make(json.RawMessage, len(*v)) - n := copy(dst, *v) - // According to the documentation for `copy`: - // - // Copy returns the number of elements copied, which will be the minimum - // of len(src) and len(dst). - // - // Since `len(src)` is `len(dst)`, and `copy` cannot copy more bytes the - // its source, we know that `n == len(v)`. - contract.Assertf(n == len(*v), "failed to perform full copy") - m[k] = &dst + m[k] = cloneRawMessage(v) } return &Data{m} +} +func cloneRawMessage(m json.RawMessage) json.RawMessage { + dst := make(json.RawMessage, len(m)) + n := copy(dst, m) + // According to the documentation for `copy`: + // + // Copy returns the number of elements copied, which will be the minimum + // of len(src) and len(dst). + // + // Since `len(src)` is `len(dst)`, and `copy` cannot copy more bytes the + // its source, we know that `n == len(v)`. + contract.Assertf(n == len(m), "failed to perform full copy") + return dst } diff --git a/unstable/metadata/metadata_test.go b/unstable/metadata/metadata_test.go index 60092b818..02e1ac2cf 100644 --- a/unstable/metadata/metadata_test.go +++ b/unstable/metadata/metadata_test.go @@ -33,5 +33,5 @@ func TestMarshal(t *testing.T) { "hello", "world" ] -}`, string(data.Marshal())) +}`, string(data.MarshalIndent())) } From 34fc3896e7f040639516a3f496787fbf2ad8fe93 Mon Sep 17 00:00:00 2001 From: Matthew Jeffryes Date: Wed, 15 Nov 2023 09:53:35 -0800 Subject: [PATCH 3/4] Add tests and flag --- pkg/tfbridge/info.go | 6 ++++++ pkg/tfbridge/metadata_test.go | 34 ++++++++++++++++++++++++++++++ pkg/tfgen/generate.go | 2 +- unstable/metadata/metadata_test.go | 26 ++++++++++++++++++++++- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 pkg/tfbridge/metadata_test.go diff --git a/pkg/tfbridge/info.go b/pkg/tfbridge/info.go index 037779c8f..1e99e8677 100644 --- a/pkg/tfbridge/info.go +++ b/pkg/tfbridge/info.go @@ -133,6 +133,12 @@ type ProviderInfo struct { // // See https://github.com/pulumi/pulumi-terraform-bridge/issues/1501 XSkipDetailedDiffForChanges bool + + // Enables generation of a trimmed, runtime-only metadata file + // to help reduce resource plugin start time + // + // See also pulumi/pulumi-terraform-bridge#1524 + GenerateRuntimeMetadata bool } // Send logs or status logs to the user. diff --git a/pkg/tfbridge/metadata_test.go b/pkg/tfbridge/metadata_test.go new file mode 100644 index 000000000..69284d0a4 --- /dev/null +++ b/pkg/tfbridge/metadata_test.go @@ -0,0 +1,34 @@ +package tfbridge + +import ( + "testing" + + md "github.com/pulumi/pulumi-terraform-bridge/v3/unstable/metadata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetadataInfo(t *testing.T) { + data, err := md.New(nil) + require.NoError(t, err) + + err = md.Set(data, "hi", []string{"hello", "world"}) + require.NoError(t, err) + err = md.Set(data, "auto-aliasing", []string{"1", "2"}) + require.NoError(t, err) + err = md.Set(data, "mux", []string{"a", "b"}) + require.NoError(t, err) + + marshalled := data.Marshal() + require.Equal(t, `{"auto-aliasing":["1","2"],"hi":["hello","world"],"mux":["a","b"]}`, string(marshalled)) + + info := NewProviderMetadata(marshalled) + assert.Equal(t, "bridge-metadata.json", info.Path) + marshalledInfo := (*md.Data)(info.Data).Marshal() + assert.Equal(t, `{"auto-aliasing":["1","2"],"hi":["hello","world"],"mux":["a","b"]}`, string(marshalledInfo)) + + runtimeMetadata := info.ExtractRuntimeMetadata() + assert.Equal(t, "runtime-bridge-metadata.json", runtimeMetadata.Path) + runtimeMarshalled := (*md.Data)(runtimeMetadata.Data).Marshal() + assert.Equal(t, `{"auto-aliasing":["1","2"],"mux":["a","b"]}`, string(runtimeMarshalled)) +} diff --git a/pkg/tfgen/generate.go b/pkg/tfgen/generate.go index e15fbac99..bcfb0fb8c 100644 --- a/pkg/tfgen/generate.go +++ b/pkg/tfgen/generate.go @@ -947,7 +947,7 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe if info := g.info.MetadataInfo; info != nil { files[info.Path] = (*metadata.Data)(info.Data).MarshalIndent() - if true { + if g.info.GenerateRuntimeMetadata { runtimeInfo := info.ExtractRuntimeMetadata() files[runtimeInfo.Path] = (*metadata.Data)(runtimeInfo.Data).Marshal() } diff --git a/unstable/metadata/metadata_test.go b/unstable/metadata/metadata_test.go index 02e1ac2cf..b332e67dd 100644 --- a/unstable/metadata/metadata_test.go +++ b/unstable/metadata/metadata_test.go @@ -28,10 +28,34 @@ func TestMarshal(t *testing.T) { err = Set(data, "hi", []string{"hello", "world"}) assert.NoError(t, err) + marshalled := data.MarshalIndent() assert.Equal(t, `{ "hi": [ "hello", "world" ] -}`, string(data.MarshalIndent())) +}`, string(marshalled)) + + parsed, err := New(marshalled) + assert.NoError(t, err) + read, _, err := Get[[]string](parsed, "hi") + assert.NoError(t, err) + assert.Equal(t, []string{"hello", "world"}, read) +} + +func TestMarshalIndent(t *testing.T) { + data, err := New(nil) + require.NoError(t, err) + + err = Set(data, "hi", []string{"hello", "world"}) + assert.NoError(t, err) + + marshalled := data.Marshal() + assert.Equal(t, `{"hi":["hello","world"]}`, string(marshalled)) + + parsed, err := New(marshalled) + assert.NoError(t, err) + read, _, err := Get[[]string](parsed, "hi") + assert.NoError(t, err) + assert.Equal(t, []string{"hello", "world"}, read) } From 6a98a47085040b1b3ff2b4080b1e7b7d36a96224 Mon Sep 17 00:00:00 2001 From: "Matthew (Matt) Jeffryes" Date: Wed, 15 Nov 2023 20:25:15 -0800 Subject: [PATCH 4/4] feedback Co-authored-by: Ian Wahbe --- pkg/tfbridge/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tfbridge/metadata.go b/pkg/tfbridge/metadata.go index 0df1e1cf6..5412d0a95 100644 --- a/pkg/tfbridge/metadata.go +++ b/pkg/tfbridge/metadata.go @@ -66,7 +66,7 @@ func (info *MetadataInfo) assertValid() { // in the future this method might also substitute compressed contents within some keys func (info *MetadataInfo) ExtractRuntimeMetadata() *MetadataInfo { data, _ := metadata.New(nil) - metadata.CloneKey("auto-aliasing", info.Data, data) + metadata.CloneKey(aliasMetadataKey, info.Data, data) metadata.CloneKey("mux", info.Data, data) return &MetadataInfo{"runtime-bridge-metadata.json", ProviderMetadata(data)} }