Skip to content

Commit

Permalink
GODRIVER-3123 [master] QE Range Protocol V2 (mongodb#1723) (mongodb#1763
Browse files Browse the repository at this point in the history 
)

Co-authored-by: Qingyang Hu <[email protected]>
  • Loading branch information
@blink1073 @qingyang-hu
blink1073 and qingyang-hu authored Aug 20, 2024
1 parent a7ee617 commit 15ce781
Show file tree
Hide file tree
Showing 105 changed files with 1,998 additions and 1,412 deletions.
8 changes: 4 additions & 4 deletions etc/install-libmongocrypt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# This script installs libmongocrypt into an "install" directory.
set -eux

LIBMONGOCRYPT_TAG="1.8.2"
LIBMONGOCRYPT_TAG="1.11.0"

# Install libmongocrypt based on OS.
if [ "Windows_NT" = "${OS:-}" ]; then
Expand All @@ -13,11 +13,11 @@ if [ "Windows_NT" = "${OS:-}" ]; then
mkdir libmongocrypt-all
cd libmongocrypt-all
# The following URL is published from the upload-all task in the libmongocrypt Evergreen project.
curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/$LIBMONGOCRYPT_TAG/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz
curl -L https://github.com/mongodb/libmongocrypt/releases/download/$LIBMONGOCRYPT_TAG/libmongocrypt-windows-x86_64-$LIBMONGOCRYPT_TAG.tar.gz -o libmongocrypt-all.tar.gz
tar -xf libmongocrypt-all.tar.gz
cd ..
cp libmongocrypt-all/windows-test/bin/mongocrypt.dll c:/libmongocrypt/bin
cp libmongocrypt-all/windows-test/include/mongocrypt/*.h c:/libmongocrypt/include
cp libmongocrypt-all/bin/mongocrypt.dll c:/libmongocrypt/bin
cp libmongocrypt-all/include/mongocrypt/*.h c:/libmongocrypt/include

rm -rf libmongocrypt-all
echo "fetching build for Windows ... end"
Expand Down
112 changes: 90 additions & 22 deletions internal/integration/client_side_encryption_prose_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2455,8 +2455,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
}
})

// GODRIVER-3123. When we implement this feature, lower the min server version to 8.0.1
qeRunOpts22 := qeRunOpts.MaxServerVersion("7.99.99")
qeRunOpts22 := qeRunOpts.MinServerVersion("8.0")
mt.RunOpts("22. range explicit encryption", qeRunOpts22, func(mt *mtest.T) {
type testcase struct {
typeStr string
Expand All @@ -2470,6 +2469,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
twoHundredOne bson.RawValue
}

trimFactor := int32(1)
sparsity := int64(1)
precision := int32(2)

d128_0, err := bson.ParseDecimal128("0")
Expand Down Expand Up @@ -2497,7 +2498,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
typeStr: "DecimalNoPrecision",
field: "encryptedDecimalNoPrecision",
typeBson: bson.TypeDecimal128,
rangeOpts: options.Range().SetSparsity(1),
rangeOpts: options.Range().SetTrimFactor(trimFactor).SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_0h, d128_0l)},
six: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_6h, d128_6l)},
thirty: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_30h, d128_30l)},
Expand All @@ -2511,7 +2512,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_0h, d128_0l)}).
SetMax(bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_200h, d128_200l)}).
SetSparsity(1).
SetTrimFactor(trimFactor).
SetSparsity(sparsity).
SetPrecision(precision),
zero: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_0h, d128_0l)},
six: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_6h, d128_6l)},
Expand All @@ -2523,7 +2525,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
typeStr: "DoubleNoPrecision",
field: "encryptedDoubleNoPrecision",
typeBson: bson.TypeDouble,
rangeOpts: options.Range().SetSparsity(1),
rangeOpts: options.Range().SetTrimFactor(trimFactor).SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 0)},
six: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 30)},
Expand All @@ -2537,7 +2539,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 200)}).
SetSparsity(1).
SetTrimFactor(trimFactor).
SetSparsity(sparsity).
SetPrecision(precision),
zero: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 0)},
six: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 6)},
Expand All @@ -2552,7 +2555,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 200)}).
SetSparsity(1),
SetTrimFactor(trimFactor).
SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 0)},
six: bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 30)},
Expand All @@ -2566,7 +2570,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 200)}).
SetSparsity(1),
SetTrimFactor(trimFactor).
SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)},
six: bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 30)},
Expand All @@ -2580,7 +2585,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 200)}).
SetSparsity(1),
SetTrimFactor(trimFactor).
SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 0)},
six: bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 30)},
Expand Down Expand Up @@ -2635,7 +2641,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
// Insert 0, 6, 30, and 200.
coll := encryptedClient.Database("db").Collection("explicit_encryption")
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand Down Expand Up @@ -2682,7 +2688,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand All @@ -2698,10 +2704,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2740,10 +2746,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2782,10 +2788,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2819,10 +2825,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2858,7 +2864,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand All @@ -2872,7 +2878,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand All @@ -2899,7 +2905,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
ro := test.rangeOpts
ro.SetPrecision(2)
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(ro)
Expand All @@ -2911,6 +2917,68 @@ func TestClientSideEncryptionProse(t *testing.T) {
})
}
})

mt.RunOpts("22. range explicit encryption applies defaults", qeRunOpts22, func(mt *mtest.T) {
err := mt.Client.Database("keyvault").Collection("datakeys").Drop(context.Background())
assert.Nil(mt, err, "error on Drop: %v", err)

testVal := bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 123)}

keyVaultClient, err := mongo.Connect(options.Client().ApplyURI(mtest.ClusterURI()))
assert.Nil(mt, err, "error on Connect: %v", err)

ceo := options.ClientEncryption().
SetKeyVaultNamespace("keyvault.datakeys").
SetKmsProviders(fullKmsProvidersMap)
clientEncryption, err := mongo.NewClientEncryption(keyVaultClient, ceo)
assert.Nil(mt, err, "error on NewClientEncryption: %v", err)

dkOpts := options.DataKey()
keyID, err := clientEncryption.CreateDataKey(context.Background(), "local", dkOpts)
assert.Nil(mt, err, "error in CreateDataKey: %v", err)

eo := options.Encrypt().
SetAlgorithm("Range").
SetKeyID(keyID).
SetContentionFactor(0).
SetRangeOptions(options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 1000)}))
payloadDefaults, err := clientEncryption.Encrypt(context.Background(), testVal, eo)
assert.Nil(mt, err, "error in Encrypt: %v", err)

mt.Run("Case 1: uses libmongocrypt defaults", func(mt *mtest.T) {
trimFactor := int32(6)
sparsity := int64(2)
eo := options.Encrypt().
SetAlgorithm("Range").
SetKeyID(keyID).
SetContentionFactor(0).
SetRangeOptions(options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 1000)}).
SetTrimFactor(trimFactor).
SetSparsity(sparsity))
payload, err := clientEncryption.Encrypt(context.Background(), testVal, eo)
assert.Nil(mt, err, "error in Encrypt: %v", err)
assert.Equalf(mt, len(payload.Data), len(payloadDefaults.Data), "the returned payload size is expected to be %d", len(payloadDefaults.Data))
})

mt.Run("Case 2: accepts trimFactor 0", func(mt *mtest.T) {
trimFactor := int32(0)
eo := options.Encrypt().
SetAlgorithm("Range").
SetKeyID(keyID).
SetContentionFactor(0).
SetRangeOptions(options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 1000)}).
SetTrimFactor(trimFactor))
payload, err := clientEncryption.Encrypt(context.Background(), testVal, eo)
assert.Nil(mt, err, "error in Encrypt: %v", err)
assert.Greater(t, len(payload.Data), len(payloadDefaults.Data), "the returned payload size is expected to be greater than %d", len(payloadDefaults.Data))
})
})
}

func getWatcher(mt *mtest.T, streamType mongo.StreamType, cpt *cseProseTest) watcher {
Expand Down
17 changes: 8 additions & 9 deletions mongo/client_encryption.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,12 @@ func transformExplicitEncryptionOptions(opts ...options.Lister[options.EncryptOp
if rangeArgs.Precision != nil {
transformedRange.Precision = rangeArgs.Precision
}
transformedRange.Sparsity = rangeArgs.Sparsity
if rangeArgs.Sparsity != nil {
transformedRange.Sparsity = rangeArgs.Sparsity
}
if rangeArgs.TrimFactor != nil {
transformedRange.TrimFactor = rangeArgs.TrimFactor
}
transformed.SetRangeOptions(transformedRange)
}
return transformed
Expand Down Expand Up @@ -250,14 +255,8 @@ func (ce *ClientEncryption) Encrypt(
// 2. An Aggregate Expression of this form:
// {$and: [{$gt: [<fieldpath>, <value1>]}, {$lt: [<fieldpath>, <value2>]}]
// $gt may also be $gte. $lt may also be $lte.
// Only supported for queryType "rangePreview"
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ce *ClientEncryption) EncryptExpression(
ctx context.Context,
expr interface{},
result interface{},
opts ...options.Lister[options.EncryptOptions],
) error {
// Only supported for queryType "range"
func (ce *ClientEncryption) EncryptExpression(ctx context.Context, expr interface{}, result interface{}, opts ...options.Lister[options.EncryptOptions]) error {
transformed := transformExplicitEncryptionOptions(opts...)

exprDoc, err := marshal(expr, nil, nil)
Expand Down
34 changes: 20 additions & 14 deletions mongo/options/encryptoptions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,13 @@ const (
QueryTypeEquality string = "equality"
)

// RangeOptions specifies index options for a Queryable Encryption field
// supporting "rangePreview" queries. Beta: The Range algorithm is experimental
// only. It is not intended for public use. It is subject to breaking changes.
// RangeOptions specifies index options for a Queryable Encryption field supporting "range" queries.
type RangeOptions struct {
Min *bson.RawValue
Max *bson.RawValue
Sparsity int64
Precision *int32
Min *bson.RawValue
Max *bson.RawValue
Sparsity *int64
TrimFactor *int32
Precision *int32
}

// RangeOptionsBuilder contains options to configure Rangeopts for queryeable
Expand Down Expand Up @@ -96,6 +95,7 @@ func (e *EncryptOptionsBuilder) SetKeyAltName(keyAltName string) *EncryptOptions
// - AEAD_AES_256_CBC_HMAC_SHA_512-Random
// - Indexed
// - Unindexed
// - Range
// This is required.
// Indexed and Unindexed are used for Queryable Encryption.
func (e *EncryptOptionsBuilder) SetAlgorithm(algorithm string) *EncryptOptionsBuilder {
Expand Down Expand Up @@ -134,8 +134,7 @@ func (e *EncryptOptionsBuilder) SetContentionFactor(contentionFactor int64) *Enc
return e
}

// SetRangeOptions specifies the options to use for explicit encryption with range. It is only valid to set if algorithm is "rangePreview".
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
// SetRangeOptions specifies the options to use for explicit encryption with range. It is only valid to set if algorithm is "range".
func (e *EncryptOptionsBuilder) SetRangeOptions(ro *RangeOptionsBuilder) *EncryptOptionsBuilder {
e.Opts = append(e.Opts, func(opts *EncryptOptions) error {
opts.RangeOptions = ro
Expand All @@ -147,7 +146,6 @@ func (e *EncryptOptionsBuilder) SetRangeOptions(ro *RangeOptionsBuilder) *Encryp
}

// SetMin sets the range index minimum value.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetMin(min bson.RawValue) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Min = &min
Expand All @@ -159,7 +157,6 @@ func (ro *RangeOptionsBuilder) SetMin(min bson.RawValue) *RangeOptionsBuilder {
}

// SetMax sets the range index maximum value.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetMax(max bson.RawValue) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Max = &max
Expand All @@ -171,10 +168,20 @@ func (ro *RangeOptionsBuilder) SetMax(max bson.RawValue) *RangeOptionsBuilder {
}

// SetSparsity sets the range index sparsity.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetSparsity(sparsity int64) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Sparsity = sparsity
opts.Sparsity = &sparsity

return nil
})

return ro
}

// SetTrimFactor sets the range index trim factor.
func (ro *RangeOptionsBuilder) SetTrimFactor(trimFactor int32) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.TrimFactor = &trimFactor

return nil
})
Expand All @@ -183,7 +190,6 @@ func (ro *RangeOptionsBuilder) SetSparsity(sparsity int64) *RangeOptionsBuilder
}

// SetPrecision sets the range index precision.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetPrecision(precision int32) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Precision = &precision
Expand Down
Loading

0 comments on commit 15ce781

Please sign in to comment.