From 01e89e6db7e963b31ad27ecc332c9cbb562d4261 Mon Sep 17 00:00:00 2001 From: Manish Sahni Date: Fri, 14 Apr 2023 13:10:34 +1000 Subject: [PATCH 1/5] Added new class for manual_encryption/answers.go --- manual_encryption/answer.go | 227 ++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 manual_encryption/answer.go diff --git a/manual_encryption/answer.go b/manual_encryption/answer.go new file mode 100644 index 0000000..481cc89 --- /dev/null +++ b/manual_encryption/answer.go @@ -0,0 +1,227 @@ +package main + +import ( + "C" + "context" + "fmt" + "os" + "time" + "crypto/tls" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +// Function to create MognoDB client instance +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +// Function to create the MognoDB ClientEncryption instance +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +// Function to perform the manual encryption +func encryptManual(ce *mongo.ClientEncryption, dek primitive.Binary, alg string, data interface{}) (primitive.Binary, error) { + var out primitive.Binary + rawValueType, rawValueData, err := bson.MarshalValue(data) + if err != nil { + return primitive.Binary{}, err + } + + rawValue := bson.RawValue{Type: rawValueType, Value: rawValueData} + + encryptionOpts := options.Encrypt(). + SetAlgorithm(alg). + SetKeyID(dek) + + out, err =ce.Encrypt(context.TODO(), rawValue, encryptionOpts) // PUT CODE HERE TO MANUALLY ENCRYPT + if err != nil { + return primitive.Binary{}, err + } + + return out, nil +} + +func main() { + var ( + keyVaultDB = "__encryption" + keyVaultColl = "__keyVault" + keySpace = keyVaultDB + "." + keyVaultColl + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + clientEncryption *mongo.ClientEncryption + client *mongo.Client + exitCode = 0 + kmipTLSConfig *tls.Config + result *mongo.InsertOneResult + dekFindResult bson.M + dek primitive.Binary + err error + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + coll := client.Database("__encryption").Collection("__keyVault") + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + clientEncryption, err = createManualEncryptionClient(client, kmsProvider, keySpace, kmsTLSOptions) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload := bson.M{ + "name": bson.M{ + "firstName": "Manish", + "lastName": "Engineer", + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "1 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1980, 10, 10, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SD20NN001", + "role": []string{"CTO"}, + } + + // Retrieve our DEK + opts := options.FindOne().SetProjection(bson.D{{Key: "_id", Value: 1}}) + err = coll.FindOne(context.TODO(), bson.D{{Key: "keyAltNames", Value: "dataKey1"}}, opts).Decode(&dekFindResult) + if err != nil || len(dekFindResult) == 0 { + fmt.Printf("DEK find error: %s\n", err) + exitCode = 1 + return + } + dek = dekFindResult["_id"].(primitive.Binary) + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } else { + name["otherNames"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", name["otherNames"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + } + + name["firstName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["firstName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + name["lastName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["lastName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + payload["name"] = name + + payload["address"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["address"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["dob"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["dob"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["phoneNumber"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["phoneNumber"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["salary"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["salary"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["taxIdentifier"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["taxIdentifier"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + coll = client.Database("companyData").Collection("employee") + + result, err = coll.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + exitCode = 0 +} \ No newline at end of file From c53a11a9a218c9b3773338659f0d85f325ce6856 Mon Sep 17 00:00:00 2001 From: Manish Sahni Date: Fri, 14 Apr 2023 13:57:54 +1000 Subject: [PATCH 2/5] Added decryption/answers.go and improved main.go in manual_decryption --- manual_decryption/answers.go | 306 +++++++++++++++++++++++++++++++++++ manual_decryption/main.go | 3 +- 2 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 manual_decryption/answers.go diff --git a/manual_decryption/answers.go b/manual_decryption/answers.go new file mode 100644 index 0000000..8af3060 --- /dev/null +++ b/manual_decryption/answers.go @@ -0,0 +1,306 @@ +package main + +import ( + "C" + "context" + "crypto/tls" + "fmt" + "os" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +// Function to create MognoDB client instance +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +// Function to create the MognoDB ClientEncryption instance +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +// Function to perform the manual encryption +func encryptManual(ce *mongo.ClientEncryption, dek primitive.Binary, alg string, data interface{}) (primitive.Binary, error) { + var out primitive.Binary + rawValueType, rawValueData, err := bson.MarshalValue(data) + if err != nil { + return primitive.Binary{}, err + } + + rawValue := bson.RawValue{Type: rawValueType, Value: rawValueData} + + encryptionOpts := options.Encrypt(). + SetAlgorithm(alg). + SetKeyID(dek) + + out, err = ce.Encrypt(context.TODO(), rawValue, encryptionOpts) + if err != nil { + return primitive.Binary{}, err + } + + return out, nil +} + +// Function to decrypt +func decryptManual(c *mongo.ClientEncryption, d primitive.Binary)(bson.RawValue, error) { + out, err := c.Decrypt(context.TODO(), d) // PERFORM THE DECRYPTION HERE + if err != nil { + return bson.RawValue{}, err + } + + return out, nil +} + + +// Function that traverses a BSON object and determines if the type is a primitive, +// if so, we check if this is a binary subtype 6 and then call the manual decrypt function +// to decrypt the value. We call the same function if arrays or subdocuments are found +func traverseBson(c *mongo.ClientEncryption, d bson.M) (bson.M, error) { + for k, v := range d { + a, ok := v.(primitive.M) + if ok { + data, err := traverseBson(c, a) + if err != nil { + return bson.M{}, err + } + d[k] = data + } else { + // Check if binary Subtype 6 data, e.g. encrypted. Skip if it is not + i, ok := v.(primitive.Binary) + if !ok { + // not binary data + continue + } + if i.Subtype == 6 { + data, err := decryptManual(c, i) + if err != nil { + return bson.M{}, err + } + d[k] = data + } + } + } + return d, nil +} + +func main() { + var ( + keyVaultDB = "__encryption" + keyVaultColl = "__keyVault" + keySpace = keyVaultDB + "." + keyVaultColl + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + clientEncryption *mongo.ClientEncryption + client *mongo.Client + exitCode = 0 + result *mongo.InsertOneResult + dekFindResult bson.M + findResult bson.M + outputData bson.M + dek primitive.Binary + encryptedName primitive.Binary + kmipTLSConfig *tls.Config + err error + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + coll := client.Database("__encryption").Collection("__keyVault") + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + + clientEncryption, err = createManualEncryptionClient(client, kmsProvider, keySpace, kmsTLSOptions) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload := bson.M{ + "name": bson.M{ + "firstName": "Kuber", + "lastName": "Engineer", + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "12 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1981, 11, 11, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SDSSNN001", + "role": []string{"DEV"}, + } + + // Retrieve our DEK + opts := options.FindOne().SetProjection(bson.D{{Key: "_id", Value: 1}}) + err = coll.FindOne(context.TODO(), bson.D{{Key: "keyAltNames", Value: "dataKey1"}}, opts).Decode(&dekFindResult) + if err != nil || len(dekFindResult) == 0 { + fmt.Printf("DEK find error: %s\n", err) + exitCode = 1 + return + } + dek = dekFindResult["_id"].(primitive.Binary) + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } else { + name["otherNames"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", name["otherNames"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + } + + name["firstName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["firstName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + name["lastName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["lastName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + payload["name"] = name + + payload["address"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["address"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["dob"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["dob"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["phoneNumber"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["phoneNumber"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["salary"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["salary"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["taxIdentifier"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["taxIdentifier"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + coll = client.Database("companyData").Collection("employee") + + result, err = coll.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + // WRITE CODE AS REQUIRED TO QUERY FOR THE name.firstName + encryptedName, err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", "Kuber")// put required variables here) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + err = coll.FindOne(context.TODO(), bson.M{"name.firstName": encryptedName}).Decode(&findResult) + if err != nil { + fmt.Printf("MongoDB find error: %s\n", err) + exitCode = 1 + return + } + if len(findResult) == 0 { + fmt.Println("Cannot find document") + exitCode = 1 + return + } + fmt.Printf("%+v\n", findResult) + + // WRITE CODE TO ENCRYPT THE NAME WE ARE GOING TO QUERY FOR + err = coll.FindOne(context.TODO(), bson.M{"name.firstName": encryptedName}).Decode(&findResult) + // GO TO THE decryptManual FUNCTION AND WRITE THE CODE + outputData, err = traverseBson(clientEncryption, findResult) + if err != nil { + fmt.Printf("Encryption error: %s\n", err) + exitCode = 1 + return + } + fmt.Printf("%+v\n", outputData) + + exitCode = 0 +} \ No newline at end of file diff --git a/manual_decryption/main.go b/manual_decryption/main.go index fa52ff0..68b0dda 100644 --- a/manual_decryption/main.go +++ b/manual_decryption/main.go @@ -31,7 +31,7 @@ func createClient(c string) (*mongo.Client, error) { } // Function to create the MognoDB ClientEncryption instance -func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns stringi, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) client, err := mongo.NewClientEncryption(c, o) if err != nil { @@ -292,7 +292,6 @@ func main() { fmt.Printf("%+v\n", findResult) // WRITE CODE TO ENCRYPT THE NAME WE ARE GOING TO QUERY FOR - encryptedName = err = coll.FindOne(context.TODO(), bson.M{"name.firstName": encryptedName}).Decode(&findResult) // GO TO THE decryptManual FUNCTION AND WRITE THE CODE outputData, err = traverseBson(clientEncryption, findResult) From cdceba64681e1aeb51431f71aaa6426d1c2a3576 Mon Sep 17 00:00:00 2001 From: Manish Sahni Date: Fri, 14 Apr 2023 15:07:34 +1000 Subject: [PATCH 3/5] Added answers to man_enc_auto_decr in go --- man_encryption_auto_decryption/answers.go | 280 ++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 man_encryption_auto_decryption/answers.go diff --git a/man_encryption_auto_decryption/answers.go b/man_encryption_auto_decryption/answers.go new file mode 100644 index 0000000..d9c8716 --- /dev/null +++ b/man_encryption_auto_decryption/answers.go @@ -0,0 +1,280 @@ +package main + +import ( + "C" + "context" + "crypto/tls" + "fmt" + "os" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +func createAutoEncryptionClient(c string, ns string, kms map[string]map[string]interface{}, tlsOps map[string]*tls.Config, s bson.M) (*mongo.Client, error) { + autoEncryptionOpts := options.AutoEncryption(). + SetKeyVaultNamespace(ns). + SetKmsProviders(kms). + SetSchemaMap(s). + SetBypassAutoEncryption(true). + SetTLSConfig(tlsOps) + + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c).SetAutoEncryptionOptions(autoEncryptionOpts)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func encryptManual(ce *mongo.ClientEncryption, dek primitive.Binary, alg string, data interface{}) (primitive.Binary, error) { + var out primitive.Binary + rawValueType, rawValueData, err := bson.MarshalValue(data) + if err != nil { + return primitive.Binary{}, err + } + + rawValue := bson.RawValue{Type: rawValueType, Value: rawValueData} + + encryptionOpts := options.Encrypt(). + SetAlgorithm(alg). + SetKeyID(dek) + + out, err = ce.Encrypt(context.TODO(), rawValue, encryptionOpts) + if err != nil { + return primitive.Binary{}, err + } + + return out, nil +} + +func main() { + var ( + keyVaultDB = "__encryption" + keyVaultColl = "__keyVault" + keySpace = keyVaultDB + "." + keyVaultColl + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + clientEncryption *mongo.ClientEncryption + encryptedClient *mongo.Client + client *mongo.Client + exitCode = 0 + result *mongo.InsertOneResult + dekFindResult bson.M + findResult bson.M + dek primitive.Binary + encryptedName primitive.Binary + kmipTLSConfig *tls.Config + err error + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + coll := client.Database("__encryption").Collection("__keyVault") + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + + clientEncryption, err = createManualEncryptionClient(client, kmsProvider, keySpace, kmsTLSOptions) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + // Auto Encryption Client + encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, bson.M{}) + if err != nil { + fmt.Printf("MDB encrypted client error: %s\n", err) + exitCode = 1 + return + } + + encryptedColl := encryptedClient.Database("companyData").Collection("employee") + + payload := bson.M{ + "name": bson.M{ + "firstName": "Poorna", + "lastName": "Muggle", + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "29 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1999, 1, 12, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SDSSWN001", + "role": []string{"CE"}, + } + + // Retrieve our DEK + opts := options.FindOne().SetProjection(bson.D{{Key: "_id", Value: 1}}) + err = coll.FindOne(context.TODO(), bson.D{{Key: "keyAltNames", Value: "dataKey1"}}, opts).Decode(&dekFindResult) + if err != nil || len(dekFindResult) == 0 { + fmt.Printf("DEK find error: %s\n", err) + exitCode = 1 + return + } + dek = dekFindResult["_id"].(primitive.Binary) + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } else { + name["otherNames"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", name["otherNames"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + } + + name["firstName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["firstName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + name["lastName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["lastName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + payload["name"] = name + + payload["address"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["address"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["dob"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["dob"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["phoneNumber"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["phoneNumber"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["salary"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["salary"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + payload["taxIdentifier"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Random", payload["taxIdentifier"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + coll = client.Database("companyData").Collection("employee") + + result, err = coll.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + + encryptedName, err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", "Poorna") + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + // WRITE YOUR QUERY HERE FOR AUTODECRYPTION. REMEMBER WHICH CLIENT TO USE! + err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName":encryptedName}).Decode(&findResult) + //err = encryptedColl.FindOne(context.TODO(), bson.M{"salary":999999.99}).Decode(&findResult) + if err != nil { + fmt.Printf("MongoDB find error: %s\n", err) + exitCode = 1 + return + } + if len(findResult) == 0 { + fmt.Println("Cannot find document") + exitCode = 1 + return + } + fmt.Printf("%+v\n", findResult) + + exitCode = 0 +} \ No newline at end of file From c47fb2d0d31ab87bcaa9f291a1f3b68d2de23cc8 Mon Sep 17 00:00:00 2001 From: Manish Sahni Date: Mon, 17 Apr 2023 12:21:20 +1000 Subject: [PATCH 4/5] Added answers & few minor changes to the main code --- auto_complete/main.go | 4 +- automatic_decryption/answers.go | 259 ++++++++++++++++++++++++++++++++ automatic_encryption/answers.go | 249 ++++++++++++++++++++++++++++++ automatic_encryption/main.go | 4 +- 4 files changed, 512 insertions(+), 4 deletions(-) create mode 100644 automatic_decryption/answers.go create mode 100644 automatic_encryption/answers.go diff --git a/auto_complete/main.go b/auto_complete/main.go index 8c3348a..7ed6775 100644 --- a/auto_complete/main.go +++ b/auto_complete/main.go @@ -240,8 +240,8 @@ func main() { return } fmt.Print(result.InsertedID) - - err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName": firstname}).Decode(&findResult) + err = encryptedColl.FindOne(context.TODO(), bson.M{"salary": 999999.99}).Decode(&findResult) // As per the excercise attempt to query salary field + //err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName": firstname}).Decode(&findResult) // Uncomment this to attemt to query name.firstName field as per excercise if err != nil { fmt.Printf("MongoDB find error: %s\n", err) exitCode = 1 diff --git a/automatic_decryption/answers.go b/automatic_decryption/answers.go new file mode 100644 index 0000000..63cb2a7 --- /dev/null +++ b/automatic_decryption/answers.go @@ -0,0 +1,259 @@ +package main + +import ( + "C" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "os" + "encoding/base64" + "strings" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "github.com/goombaio/namegenerator" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +func createAutoEncryptionClient(c string, ns string, kms map[string]map[string]interface{}, tlsOps map[string]*tls.Config, s bson.M) (*mongo.Client, error) { + autoEncryptionOpts := options.AutoEncryption(). + SetKeyVaultNamespace(ns). + SetKmsProviders(kms). + SetSchemaMap(s). + SetTLSConfig(tlsOps) + + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c).SetAutoEncryptionOptions(autoEncryptionOpts)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func nameGenerator()(string, string) { + seed := time.Now().UTC().UnixNano() + nameGenerator := namegenerator.NewNameGenerator(seed) + + name := nameGenerator.Generate() + + firstName := strings.Split(name, "-")[0] + lastName := strings.Split(name, "-")[1] + + return firstName, lastName +} + +func main() { + var ( + keyVaultDB = "__encryption" + keyVaultColl = "__keyVault" + keySpace = keyVaultDB + "." + keyVaultColl + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + encryptedClient *mongo.Client + client *mongo.Client + exitCode = 0 + result *mongo.InsertOneResult + dekFindResult bson.M + findResult bson.M + dek primitive.Binary + kmipTLSConfig *tls.Config + err error + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + coll := client.Database("__encryption").Collection("__keyVault") + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + firstname, lastname := nameGenerator() + payload := bson.M{ + "name": bson.M{ + "firstName": firstname, + "lastName": lastname, + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "29 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1999, 1, 12, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SDSSWN001", + "role": []string{"Student"}, + } + + // Retrieve our DEK + opts := options.FindOne().SetProjection(bson.D{{Key: "_id", Value: 1}}) + err = coll.FindOne(context.TODO(), bson.D{{Key: "keyAltNames", Value: "dataKey1"}}, opts).Decode(&dekFindResult) + if err != nil || len(dekFindResult) == 0 { + fmt.Printf("DEK find error: %s\n", err) + exitCode = 1 + return + } + dek = dekFindResult["_id"].(primitive.Binary) + + db := "companyData" + collection := "employee" + + schemaMap := `{ + "bsonType": "object", + "encryptMetadata": { + "keyId": [` + base64.StdEncoding.EncodeToString(dek.Data) + `], + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random", + }, + "properties": { + "name": { + "bsonType": "object", + "properties": { + "firstName": { + "encrypt": { + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "lastName": { + "encrypt": { + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "otherNames": { + "encrypt": { + "bsonType": "string" + } + } + } + }, + "address": { + "encrypt": { + "bsonType": "object" + } + }, + "dob": { + "encrypt": { + "bsonType": "date" + } + }, + "phoneNumber": { + "encrypt": { + "bsonType": "string" + } + }, + "salary": { + "encrypt": { + "bsonType": "double" + } + }, + "taxIdentifier": { + "encrypt": { + "bsonType": "string" + } + } + } + }` + + // Auto Encryption Client + var testSchema bson.M + json.Unmarshal([]byte(schemaMap), &testSchema) + encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, testSchema) + if err != nil { + fmt.Printf("MDB encrypted client error: %s\n", err) + exitCode = 1 + return + } + + encryptedColl := encryptedClient.Database(db).Collection(collection) + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } + + result, err = encryptedColl.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + // WRITE YOUR QUERY HERE FOR AUTODECRYPTION. REMEMBER WHICH CLIENT TO USE! + err = encryptedColl.FindOne(context.TODO(), bson.M{"salary": 999999.99}).Decode(&findResult) // As per the excercise attempt to query salary field + //err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName": firstname}).Decode(&findResult) // Uncomment this to attemt to query name.firstName field as per excercise + if err != nil { + fmt.Printf("MongoDB find error: %s\n", err) + exitCode = 1 + return + } + if len(findResult) == 0 { + fmt.Println("Cannot find document") + exitCode = 1 + return + } + fmt.Printf("%+v\n", findResult) + exitCode = 0 +} \ No newline at end of file diff --git a/automatic_encryption/answers.go b/automatic_encryption/answers.go new file mode 100644 index 0000000..6d55a20 --- /dev/null +++ b/automatic_encryption/answers.go @@ -0,0 +1,249 @@ +package main + +import ( + "C" + "context" + "crypto/tls" + "fmt" + "os" + "strings" + "time" + "encoding/json" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "github.com/goombaio/namegenerator" + "github.com/google/uuid" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +func createAutoEncryptionClient(c string, ns string, kms map[string]map[string]interface{}, tlsOps map[string]*tls.Config, s bson.M) (*mongo.Client, error) { + autoEncryptionOpts := options.AutoEncryption(). + SetKeyVaultNamespace(ns). + SetKmsProviders(kms). + SetSchemaMap(s). + SetTLSConfig(tlsOps) + + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c).SetAutoEncryptionOptions(autoEncryptionOpts)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func nameGenerator()(string, string) { + seed := time.Now().UTC().UnixNano() + nameGenerator := namegenerator.NewNameGenerator(seed) + + name := nameGenerator.Generate() + + firstName := strings.Split(name, "-")[0] + lastName := strings.Split(name, "-")[1] + + return firstName, lastName +} + +func main(){ + var ( + keyVaultDB = "__encryption" + keyVaultColl = "__keyVault" + keySpace = keyVaultDB + "." + keyVaultColl + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + encryptedClient *mongo.Client + client *mongo.Client + exitCode = 0 + result *mongo.InsertOneResult + dekFindResult bson.M + dek primitive.Binary + kmipTLSConfig *tls.Config + err error + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + coll := client.Database("__encryption").Collection("__keyVault") + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + firstname, lastname := nameGenerator() + payload := bson.M{ + "name": bson.M{ + "firstName": firstname, + "lastName": lastname, + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "29 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1999, 1, 12, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SDSSWN001", + "role": []string{"Student"}, + } + + // Retrieve our DEK + opts := options.FindOne().SetProjection(bson.D{{Key: "_id", Value: 1}}) + err = coll.FindOne(context.TODO(), bson.D{{Key: "keyAltNames", Value: "dataKey1"}}, opts).Decode(&dekFindResult) + if err != nil || len(dekFindResult) == 0 { + fmt.Printf("DEK find error: %s\n", err) + exitCode = 1 + return + } + dek = dekFindResult["_id"].(primitive.Binary) + key, e := uuid.FromBytes(dek.Data) + if e != nil { + fmt.Printf("Error converting UUID") + exitCode = 1 + return + } + + db := "companyData" + collection := "employee" + + + schemaMap := `{ + "bsonType": "object", + "encryptMetadata": { + "keyId": [` + key.String() + `], + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random", + }, + "properties": { + "name": { + "bsonType": "object", + "properties": { + "firstName": { + "encrypt": { + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "lastName": { + "encrypt": { + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "otherNames": { + "encrypt": { + "bsonType": "string" + } + } + } + }, + "address": { + "encrypt": { + "bsonType": "object" + } + }, + "dob": { + "encrypt": { + "bsonType": "date" + } + }, + "phoneNumber": { + "encrypt": { + "bsonType": "string" + } + }, + "salary": { + "encrypt": { + "bsonType": "double" + } + }, + "taxIdentifier": { + "encrypt": { + "bsonType": "string" + } + } + } + }` + // Auto Encryption Client + var testSchema bson.M + json.Unmarshal([]byte(schemaMap), &testSchema) + encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, testSchema) + if err != nil { + fmt.Printf("MDB encrypted client error: %s\n", err) + exitCode = 1 + return + } + + encryptedColl := encryptedClient.Database(db).Collection(collection) + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } + + result, err = encryptedColl.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + exitCode = 0 +} \ No newline at end of file diff --git a/automatic_encryption/main.go b/automatic_encryption/main.go index a10293b..3818ff9 100644 --- a/automatic_encryption/main.go +++ b/automatic_encryption/main.go @@ -8,7 +8,7 @@ import ( "os" "strings" "time" - + "encoding/json" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -185,7 +185,7 @@ func main(){ // Auto Encryption Client var testSchema bson.M json.Unmarshal([]byte(schemaMap), &testSchema) - encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, &testSchema) + encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, testSchema) if err != nil { fmt.Printf("MDB encrypted client error: %s\n", err) exitCode = 1 From e03770beb3b030f9bae0e10cd39ef45879d8f24d Mon Sep 17 00:00:00 2001 From: Manish Sahni Date: Mon, 17 Apr 2023 16:07:35 +1000 Subject: [PATCH 5/5] Added more answers.go and some minor changes --- use_case_1_create/answers.go | 337 ++++++++++++++++++++++++++++++++++ use_case_1_create/main.go | 2 - use_case_2_create/main.go | 2 +- use_case_complete/main.go | 4 +- use_case_delete/answers.go | 342 +++++++++++++++++++++++++++++++++++ 5 files changed, 682 insertions(+), 5 deletions(-) create mode 100644 use_case_1_create/answers.go create mode 100644 use_case_delete/answers.go diff --git a/use_case_1_create/answers.go b/use_case_1_create/answers.go new file mode 100644 index 0000000..5eb6cd0 --- /dev/null +++ b/use_case_1_create/answers.go @@ -0,0 +1,337 @@ +package main + +import ( + "C" + "context" + "crypto/tls" + "errors" + "fmt" + "math/rand" + "os" + "strconv" + "strings" + "time" + "encoding/json" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "github.com/goombaio/namegenerator" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +func createAutoEncryptionClient(c string, ns string, kms map[string]map[string]interface{}, tlsOps map[string]*tls.Config, s bson.M) (*mongo.Client, error) { + autoEncryptionOpts := options.AutoEncryption(). + SetKeyVaultNamespace(ns). + SetKmsProviders(kms). + SetSchemaMap(s). + SetTLSConfig(tlsOps) + + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c).SetAutoEncryptionOptions(autoEncryptionOpts)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func encryptManual(ce *mongo.ClientEncryption, dek primitive.Binary, alg string, data interface{}) (primitive.Binary, error) { + var out primitive.Binary + rawValueType, rawValueData, err := bson.MarshalValue(data) + if err != nil { + return primitive.Binary{}, err + } + + rawValue := bson.RawValue{Type: rawValueType, Value: rawValueData} + + encryptionOpts := options.Encrypt(). + SetAlgorithm(alg). + SetKeyID(dek) + + out, err = ce.Encrypt(context.TODO(), rawValue, encryptionOpts) + if err != nil { + return primitive.Binary{}, err + } + + return out, nil +} + +func createDEK(c *mongo.ClientEncryption, kn string, cmk map[string]interface{}, altName string) (primitive.Binary, error) { + var ( + dek primitive.Binary + err error + ) + + cOpts := options.DataKey(). + SetMasterKey(cmk). + SetKeyAltNames([]string{altName}) + dek, err = c.CreateDataKey(context.TODO(), kn, cOpts) + if err != nil { + return primitive.Binary{}, err + } + + return dek, nil +} + +func getDEK(c *mongo.ClientEncryption, altName string) (primitive.Binary, error) { + var dekFindResult bson.M + + err := c.GetKeyByAltName(context.TODO(), altName).Decode(&dekFindResult) + if err != nil { + return primitive.Binary{}, err + } + if len(dekFindResult) == 0 { + return primitive.Binary{}, nil + } + b, ok := dekFindResult["_id"].(primitive.Binary) + if !ok { + return primitive.Binary{}, errors.New("the DEK conversion error") + } + return b, nil +} + +func nameGenerator()(string, string) { + seed := time.Now().UTC().UnixNano() + nameGenerator := namegenerator.NewNameGenerator(seed) + + name := nameGenerator.Generate() + + firstName := strings.Split(name, "-")[0] + lastName := strings.Split(name, "-")[1] + + return firstName, lastName +} + +func main() { + var ( + client *mongo.Client + clientEncryption *mongo.ClientEncryption + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + dek primitive.Binary + encryptedClient *mongo.Client + err error + exitCode = 0 + findResult bson.M + keyVaultColl = "__keyVault" + keyVaultDB = "__encryption" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + kmipTLSConfig *tls.Config + result *mongo.InsertOneResult + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + cmk := map[string]interface{}{ + "keyId": "1", // this is our CMK ID + } + keySpace := keyVaultDB + "." + keyVaultColl + + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + clientEncryption, err = createManualEncryptionClient(client, kmsProvider, keySpace, kmsTLSOptions) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + rand.Seed(time.Now().UnixNano()) + id := strconv.Itoa(int(rand.Intn(100000))) + + // get our employee DEK or create + dek, err = getDEK(clientEncryption, id) + if err != nil { + if err == mongo.ErrNoDocuments { + dek, err = createDEK(clientEncryption, provider, cmk, id) + if err != nil { + fmt.Printf("Cannot create employee DEK: %s\n", err) + exitCode = 1 + return + } + } else { + fmt.Printf("Cannot get employee DEK: %s\n", err) + exitCode = 1 + return + } + } + + firstname, lastname := nameGenerator() + payload := bson.M{ + "_id": id, + "name": bson.M{ + "firstName": firstname, + "lastName": lastname, + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "29 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1999, 1, 12, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SDSSWN001", + "role": []string{"Student"}, + } + + db := "companyData" + collection := "employee" + + schemaMap := `{ + "bsonType": "object", + "encryptMetadata": { + "keyId": /_id ,// put your JOSN pointer here, + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random", + }, + "properties": { + "name": { + "bsonType": "object", + "properties": { + "otherNames": { + "encrypt": { + "bsonType": "string" + } + } + } + }, + "address": { + "encrypt": { + "bsonType": "object" + } + }, + "dob": { + "encrypt": { + "bsonType": "date" + } + }, + "phoneNumber": { + "encrypt": { + "bsonType": "string" + } + }, + "salary": { + "encrypt": { + "bsonType": "double" + } + }, + "taxIdentifier": { + "encrypt": { + "bsonType": "string" + } + } + } + }` + + // Auto Encryption Client + var testSchema bson.M + json.Unmarshal([]byte(schemaMap), &testSchema) + encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, testSchema) + if err != nil { + fmt.Printf("MDB encrypted client error: %s\n", err) + exitCode = 1 + return + } + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } + + // manually encrypt our firstName and lastName values: + name["firstName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["firstName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + name["lastName"], err = encryptManual(clientEncryption, dek, "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", name["lastName"]) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + payload["name"] = name + + encryptedColl := encryptedClient.Database(db).Collection(collection) + + result, err = encryptedColl.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName": name["firstName"]}).Decode(&findResult) + if err != nil { + fmt.Printf("MongoDB find error: %s\n", err) + exitCode = 1 + return + } + if len(findResult) == 0 { + fmt.Println("Cannot find document") + exitCode = 1 + return + } + fmt.Printf("%+v\n", findResult) + + exitCode = 0 +} \ No newline at end of file diff --git a/use_case_1_create/main.go b/use_case_1_create/main.go index 695d181..8275b3f 100644 --- a/use_case_1_create/main.go +++ b/use_case_1_create/main.go @@ -11,13 +11,11 @@ import ( "strconv" "strings" "time" - "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "github.com/goombaio/namegenerator" - "github.com/google/uuid" ) var ( diff --git a/use_case_2_create/main.go b/use_case_2_create/main.go index 99591d0..39df759 100644 --- a/use_case_2_create/main.go +++ b/use_case_2_create/main.go @@ -12,7 +12,7 @@ import ( "strconv" "strings" "time" - + "encoding/base64" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" diff --git a/use_case_complete/main.go b/use_case_complete/main.go index 6adba86..aafddad 100644 --- a/use_case_complete/main.go +++ b/use_case_complete/main.go @@ -12,7 +12,7 @@ import ( "strconv" "strings" "time" - + "encoding/base64" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -337,7 +337,7 @@ func main() { } fmt.Printf("%+v\n", findResult) - err = trashDEK(clientEncryption, kmsProvider, keySpace, employeeDEK) + trashDEK(clientEncryption, kmsProvider, keySpace, employeeDEK) if err != nil { fmt.Printf("DEK deletion error: %s", err) } diff --git a/use_case_delete/answers.go b/use_case_delete/answers.go new file mode 100644 index 0000000..ea309a9 --- /dev/null +++ b/use_case_delete/answers.go @@ -0,0 +1,342 @@ +package main + +import ( + "C" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "math/rand" + "os" + "strconv" + "strings" + "time" + "encoding/base64" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "github.com/goombaio/namegenerator" +) + +var ( + PETNAME = "valid-rabbit" + MDB_PASSWORD = "passwordone" +) + +func createClient(c string) (*mongo.Client, error) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func createManualEncryptionClient(c *mongo.Client, kp map[string]map[string]interface{}, kns string, tlsOps map[string]*tls.Config) (*mongo.ClientEncryption, error) { + o := options.ClientEncryption().SetKeyVaultNamespace(kns).SetKmsProviders(kp).SetTLSConfig(tlsOps) + client, err := mongo.NewClientEncryption(c, o) + if err != nil { + return nil, err + } + + return client, nil +} + +func createAutoEncryptionClient(c string, ns string, kms map[string]map[string]interface{}, tlsOps map[string]*tls.Config, s bson.M) (*mongo.Client, error) { + autoEncryptionOpts := options.AutoEncryption(). + SetKeyVaultNamespace(ns). + SetKmsProviders(kms). + SetSchemaMap(s). + SetTLSConfig(tlsOps) + + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(c).SetAutoEncryptionOptions(autoEncryptionOpts)) + + if err != nil { + return nil, err + } + + return client, nil +} + +func createDEK(c *mongo.ClientEncryption, kn string, cmk map[string]interface{}, altName string) (primitive.Binary, error) { + var ( + dek primitive.Binary + err error + ) + + cOpts := options.DataKey(). + SetMasterKey(cmk). + SetKeyAltNames([]string{altName}) + dek, err = c.CreateDataKey(context.TODO(), kn, cOpts) + if err != nil { + return primitive.Binary{}, err + } + + return dek, nil +} + +func getDEK(c *mongo.ClientEncryption, altName string) (primitive.Binary, error) { + var dekFindResult bson.M + + err := c.GetKeyByAltName(context.TODO(), altName).Decode(&dekFindResult) + if err != nil { + return primitive.Binary{}, err + } + if len(dekFindResult) == 0 { + return primitive.Binary{}, nil + } + b, ok := dekFindResult["_id"].(primitive.Binary) + if !ok { + return primitive.Binary{}, errors.New("the DEK conversion error") + } + return b, nil +} + +func trashDEK(c *mongo.ClientEncryption, kp map[string]map[string]interface{}, kns string, keyID primitive.Binary) error { + var ( + delResult *mongo.DeleteResult + err error + ) + + delResult, err = c.DeleteKey(context.TODO(), keyID) + if err != nil { + return err + } + if delResult.DeletedCount == 0 { + return errors.New("no DEK deleted") + } + + return nil +} + +func nameGenerator()(string, string) { + seed := time.Now().UTC().UnixNano() + nameGenerator := namegenerator.NewNameGenerator(seed) + + name := nameGenerator.Generate() + + firstName := strings.Split(name, "-")[0] + lastName := strings.Split(name, "-")[1] + + return firstName, lastName +} + +func main() { + var ( + client *mongo.Client + clientEncryption *mongo.ClientEncryption + connectionString = "mongodb://app_user:" + MDB_PASSWORD + "@csfle-mongodb-" + PETNAME + ".mdbtraining.net/?replicaSet=rs0&tls=true&tlsCAFile=%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca.cert" + dek primitive.Binary + employeeDEK primitive.Binary + encryptedClient *mongo.Client + err error + exitCode = 0 + findResult bson.M + keyVaultColl = "__keyVault" + keyVaultDB = "__encryption" + kmipEndpoint = "csfle-kmip-" + PETNAME + ".mdbtraining.net" + kmipTLSConfig *tls.Config + result *mongo.InsertOneResult + //delResult *mongo.DeleteResult + ) + + defer func() { + os.Exit(exitCode) + }() + + provider := "kmip" + kmsProvider := map[string]map[string]interface{}{ + provider: { + "endpoint": kmipEndpoint, + }, + } + cmk := map[string]interface{}{ + "keyId": "1", // this is our CMK ID + } + keySpace := keyVaultDB + "." + keyVaultColl + + client, err = createClient(connectionString) + if err != nil { + fmt.Printf("MDB client error: %s\n", err) + exitCode = 1 + return + } + + // Set the KMIP TLS options + kmsTLSOptions := make(map[string]*tls.Config) + tlsOptions := map[string]interface{}{ + "tlsCAFile": "/etc/pki/tls/certs/ca.cert", + "tlsCertificateKeyFile": "/home/ec2-user/server.pem", + } + kmipTLSConfig, err = options.BuildTLSConfig(tlsOptions) + if err != nil { + fmt.Printf("Cannot create KMS TLS Config: %s\n", err) + exitCode = 1 + return + } + kmsTLSOptions["kmip"] = kmipTLSConfig + + clientEncryption, err = createManualEncryptionClient(client, kmsProvider, keySpace, kmsTLSOptions) + if err != nil { + fmt.Printf("ClientEncrypt error: %s\n", err) + exitCode = 1 + return + } + + rand.Seed(time.Now().UnixNano()) + id := strconv.Itoa(int(rand.Intn(100000))) + + // get our employee DEK or create + employeeDEK, err = getDEK(clientEncryption, id) + if err != nil { + if err == mongo.ErrNoDocuments { + employeeDEK, err = createDEK(clientEncryption, provider, cmk, id) + if err != nil { + fmt.Printf("Cannot create employee DEK: %s\n", err) + exitCode = 1 + return + } + } else { + fmt.Printf("Cannot get employee DEK: %s\n", err) + exitCode = 1 + return + } + } + + firstname, lastname := nameGenerator() + payload := bson.M{ + "_id": id, + "name": bson.M{ + "firstName": firstname, + "lastName": lastname, + "otherNames": nil, + }, + "address": bson.M{ + "streetAddress": "29 Bson Street", + "suburbCounty": "Mongoville", + "stateProvince": "Victoria", + "zipPostcode": "3999", + "country": "Oz", + }, + "dob": time.Date(1999, 1, 12, 0, 0, 0, 0, time.Local), + "phoneNumber": "1800MONGO", + "salary": 999999.99, + "taxIdentifier": "78SDSSWN001", + "role": []string{"Student"}, + } + + // Retrieve our DEK + dek, err = getDEK(clientEncryption, "dataKey1") + if err != nil { + fmt.Printf("DEK find error: %s\n", err) + exitCode = 1 + return + } + + db := "companyData" + collection := "employee" + + schemaMap := `{ + "bsonType": "object", + "encryptMetadata": { + "keyId": "/_id", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random", + }, + "properties": { + "name": { + "bsonType": "object", + "properties": { + "firstName": { + "encrypt": { + "keyId": [` + base64.StdEncoding.EncodeToString(dek.Data) + `] + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "lastName": { + "encrypt": { + "keyId": [` + base64.StdEncoding.EncodeToString(dek.Data) + `] + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "otherNames": { + "encrypt": { + "bsonType": "string" + } + } + } + }, + "address": { + "encrypt": { + "bsonType": "object" + } + }, + "dob": { + "encrypt": { + "bsonType": "date" + } + }, + "phoneNumber": { + "encrypt": { + "bsonType": "string" + } + }, + "salary": { + "encrypt": { + "bsonType": "double" + } + }, + "taxIdentifier": { + "encrypt": { + "bsonType": "string" + } + } + } + }` + + // Auto Encryption Client + var testSchema bson.M + json.Unmarshal([]byte(schemaMap), &testSchema) + encryptedClient, err = createAutoEncryptionClient(connectionString, keySpace, kmsProvider, kmsTLSOptions, testSchema) + if err != nil { + fmt.Printf("MDB encrypted client error: %s\n", err) + exitCode = 1 + return + } + + encryptedColl := encryptedClient.Database(db).Collection(collection) + + // remove the otherNames field if it is nil + name := payload["name"].(bson.M) + if name["otherNames"] == nil { + fmt.Println("Removing nil") + delete(name, "otherNames") + } + + result, err = encryptedColl.InsertOne(context.TODO(), payload) + if err != nil { + fmt.Printf("Insert error: %s\n", err) + exitCode = 1 + return + } + fmt.Print(result.InsertedID) + + err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName": firstname}).Decode(&findResult)// PUT CODE HERE TO RETRIEVE DOCUMENT + fmt.Printf("%+v\n", findResult) + + trashDEK(clientEncryption, kmsProvider, keySpace, employeeDEK)// PUT CODE HERE TO DELETE DEK + /*if err != nil { + return err + }*/ + // PUT SLEEP HERE FOR 60 SECONDS + time.Sleep(61 * time.Second) + + err = encryptedColl.FindOne(context.TODO(), bson.M{"name.firstName": firstname}).Decode(&findResult) // PUT CODE HERE TO RETRIEVE DOCUMENT, REMEMBER TO HANDLE ERRORS! + fmt.Printf("%+v\n", findResult) + + exitCode = 0 +} \ No newline at end of file