Feature/update to 3.9

CHANGELOG.rst

@@ -1,6 +1,56 @@
 Changelog
 =========
 
+3.9.0
+-----
+Features:
+
+* Add ``--mac-only-encrypted`` to compute MAC only over values which end up encrypted (#973)
+* Allow configuration of indentation for YAML and JSON stores (#1273, #1372)
+* Introduce a ``--pristine`` flag to ``sops exec-env`` (#912)
+* Allow to pass multiple paths to ``sops updatekeys`` (#1274)
+* Allow to override ``fileName`` with different value (#1332)
+* Sort masterkeys according to ``--decryption-order`` (#1345)
+* Add separate subcommands for encryption, decryption, rotating, editing, and setting values (#1391)
+* Add ``filestatus`` command (#545)
+* Add command ``unset`` (#1475)
+* Merge key for key groups and make keys unique (#1493)
+* Support using comments to select parts to encrypt (#974, #1392)
+
+Deprecations:
+
+* Deprecate the ``--background`` option to ``exec-env`` and ``exec-file`` (#1379)
+
+Improvements:
+
+* Warn/fail if the wrong number of arguments is provided (#1342)
+* Warn if more than one command is used (#1388)
+* Dependency updates (#1327, #1328, #1330, #1336, #1334, #1344, #1348, #1354, #1357, #1360, #1373, #1381, #1383, #1385, #1408, #1428, #1429, #1427, #1439, #1454, #1460, #1466, #1489, #1519, #1525, #1528, #1540, #1543, #1545)
+* Build with Go 1.21 (#1427)
+* Improve README.rst (#1339, #1399, #1350)
+* Fix typos (#1337, #1477, #1484)
+* Polish the ``sops help`` output a bit (#1341, #1544)
+* Improve and fix tests (#1346, #1349, #1370, #1390, #1396, #1492)
+* Create a constant for the ``sops`` metadata key (#1398)
+* Refactoring: move extraction of encryption and rotation options to separate functions (#1389)
+
+Bug fixes:
+
+* Respect ``aws_profile`` from keygroup config (#1049)
+* Fix a bug where not having a config results in a panic (#1371)
+* Consolidate Flatten/Unflatten pre/post processing (#1356)
+* INI and DotEnv stores: ``shamir_threshold`` is an integer (#1394)
+* Make check whether file contains invalid keys for encryption dependent on output store (#1393)
+* Do not panic if ``updatekeys`` is used with a config that has no creation rules defined (#1506)
+* ``exec-file``: if ``--filename`` is used, use the provided filename without random suffix (#1474)
+* Do not use DotEnv store for ``exec-env``, but specialized environment serializing code (#1436)
+* Decryption: do not fail if no matching ``creation_rule`` is present in config file (#1434)
+
+Project changes:
+
+* CI dependency updates (#1347, #1359, #1376, #1382, #1386, #1425, #1432, #1498, #1503, #1508, #1510, #1516, #1521, #1492, #1534)
+* Adjust Makefile to new goreleaser 6.0.0 release (#1526)
+
 3.8.1
 -----
 Improvements:

Makefile

@@ -2,14 +2,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-PROJECT             := github.com/MagnusTonnessen/sops
+PROJECT             := github.com/bekk/sops
 PROJECT_DIR         := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 BIN_DIR             := $(PROJECT_DIR)/bin
 
 GO                  := GOPROXY=https://proxy.golang.org go
 GO_TEST_FLAGS       ?= -race -coverprofile=profile.out -covermode=atomic
 
-GITHUB_REPOSITORY   ?= github.com/MagnusTonnessen/sops
+GITHUB_REPOSITORY   ?= github.com/bekk/sops
 
 STATICCHECK         := $(BIN_DIR)/staticcheck
 STATICCHECK_VERSION := latest
@@ -67,7 +67,7 @@ checkmd: $(MD_FILES)
 .PHONY: test
 test: vendor
 	gpg --import pgp/sops_functional_tests_key.asc 2>&1 1>/dev/null || exit 0
-	$(GO) test $(GO_TEST_FLAGS) ./...
+	LANG=en_US.UTF-8 $(GO) test $(GO_TEST_FLAGS) ./...
 
 .PHONY: showcoverage
 showcoverage: test
@@ -78,7 +78,7 @@ generate: keyservice/keyservice.pb.go
 	$(GO) generate
 
 %.pb.go: %.proto
-	protoc --go-grpc_out=. --go_opt=paths=source_relative --go_out=. $<
+	protoc --go_out=plugins=grpc:. $<
 
 .PHONY: functional-tests
 functional-tests:
@@ -94,7 +94,7 @@ functional-tests-all:
 
 .PHONY: release-snapshot
 release-snapshot: install-goreleaser install-syft
-	GITHUB_REPOSITORY=$(GITHUB_REPOSITORY) $(GORELEASER) release --clean --snapshot --skip-sign
+	GITHUB_REPOSITORY=$(GITHUB_REPOSITORY) $(GORELEASER) release --clean --snapshot --skip=sign
 
 .PHONY: clean
 clean:

README.rst

@@ -205,6 +205,7 @@ the ``--age`` option or the **SOPS_AGE_RECIPIENTS** environment variable:
 When decrypting a file with the corresponding identity, SOPS will look for a
 text file name ``keys.txt`` located in a ``sops`` subdirectory of your user
 configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/age/keys.txt``.
+If ``$XDG_CONFIG_HOME`` is not set ``$HOME/.config/sops/age/keys.txt`` is used instead.
 On macOS, this would be ``$HOME/Library/Application Support/sops/age/keys.txt``. On
 Windows, this would be ``%AppData%\sops\age\keys.txt``. You can specify the location
 of this file manually by setting the environment variable **SOPS_AGE_KEY_FILE**.
@@ -220,8 +221,12 @@ Encrypting with SSH keys via age is not yet supported by SOPS.
 
 Encrypting using GCP KMS
 ~~~~~~~~~~~~~~~~~~~~~~~~
-GCP KMS uses `Application Default Credentials
-<https://developers.google.com/identity/protocols/application-default-credentials>`_.
+GCP KMS has support for authorization with the use of `Application Default Credentials
+<https://developers.google.com/identity/protocols/application-default-credentials>`_ and using an oauth2 token.
+Application default credentials precedes the use of access token.
+
+Using Application Default Credentials you can authorize by doing this:
+
 If you already logged in using
 
 .. code:: sh
@@ -234,6 +239,19 @@ you can enable application default credentials using the sdk:
 
     $ gcloud auth application-default login
 
+Using oauth tokens you can authorize by doing this:
+
+.. code:: sh
+
+    $ export GOOGLE_OAUTH_ACCESS_TOKEN=<your access token>
+
+Or if you are logged in you can authorize by generating an access token:
+
+.. code:: sh
+
+    $ export GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)"
+
+
 Encrypting/decrypting with GCP KMS requires a KMS ResourceID. You can use the
 cloud console the get the ResourceID or you can create one using the gcloud
 sdk:
@@ -256,6 +274,7 @@ And decrypt it using::
 
      $ sops decrypt test.enc.yaml
 
+
 Encrypting using Azure Key Vault
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -668,6 +687,11 @@ of all new files. If your secrets are stored under a specific directory, like a
 ``git`` repository, you can create a ``.sops.yaml`` configuration file at the root
 directory to define which keys are used for which filename.
 
+.. note::
+
+  The file needs to be named ``.sops.yaml``. Other names (i.e. ``.sops.yml``) won't be automatically
+  discovered by SOPS. You'll need to pass the ``--config .sops.yml`` option for it to be picked up.
+
 Let's take an example:
 
 * file named **something.dev.yaml** should use one set of KMS A, PGP and age
@@ -1470,6 +1494,25 @@ The value must be formatted as json.
 
     $ sops set ~/git/svc/sops/example.yaml '["an_array"][1]' '{"uid1":null,"uid2":1000,"uid3":["bob"]}'
 
+Unset a sub-part in a document tree
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Symmetrically, SOPS can unset a specific part of a YAML or JSON document, by providing
+the path in the ``unset`` command. This is useful to unset specific values, like keys, without
+needing an editor.
+
+.. code:: sh
+
+    $ sops unset ~/git/svc/sops/example.yaml '["app2"]["key"]'
+
+The tree path syntax uses regular python dictionary syntax, without the
+variable name. Set to keys by naming them, and array elements by
+numbering them.
+
+.. code:: sh
+
+    $ sops unset ~/git/svc/sops/example.yaml '["an_array"][1]'
+
 Showing diffs in cleartext in git
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1545,9 +1588,19 @@ that match the supplied regular expression. For example, this command:
 will not encrypt the values under the ``description`` and ``metadata`` keys in a YAML file
 containing kubernetes secrets, while encrypting everything else.
 
+For YAML files, another method is to use ``--encrypted-comment-regex`` which will
+only encrypt comments and values which have a preceding comment matching the supplied
+regular expression.
+
+Conversely, you can opt in to only left certain keys without encrypting by using the
+``--unencrypted-comment-regex`` option, which will leave the values and comments
+unencrypted when they have a preeceding comment, or a trailing comment on the same line,
+that matches the supplied regular expression.
+
 You can also specify these options in the ``.sops.yaml`` config file.
 
-Note: these four options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex`` and ``--unencrypted-regex`` are
+Note: these six options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex``,
+``--unencrypted-regex``, ``--encrypted-comment-regex``, and ``--unencrypted-comment-regex`` are
 mutually exclusive and cannot all be used in the same file.
 
 Encryption Protocol

cmd/sops/common/common.go

@@ -77,13 +77,12 @@ type DecryptTreeOpts struct {
 	// IgnoreMac is whether or not to ignore the Message Authentication Code included in the SOPS tree
 	IgnoreMac bool
 	// Cipher is the cryptographic cipher to use to decrypt the values inside the tree
-	Cipher                sops.Cipher
-	DecryptionCredentials map[string]string
+	Cipher sops.Cipher
 }
 
 // DecryptTree decrypts the tree passed in through the DecryptTreeOpts and additionally returns the decrypted data key
 func DecryptTree(opts DecryptTreeOpts) (dataKey []byte, err error) {
-	dataKey, err = opts.Tree.Metadata.GetDataKeyWithKeyServices(opts.KeyServices, opts.DecryptionOrder, opts.DecryptionCredentials)
+	dataKey, err = opts.Tree.Metadata.GetDataKeyWithKeyServices(opts.KeyServices, opts.DecryptionOrder)
 	if err != nil {
 		return nil, NewExitError(err, codes.CouldNotRetrieveKey)
 	}
@@ -225,13 +224,12 @@ func GetKMSKeyWithEncryptionCtx(tree *sops.Tree) (keyGroupIndex int, keyIndex in
 
 // GenericDecryptOpts represents decryption options and config
 type GenericDecryptOpts struct {
-	Cipher                sops.Cipher
-	InputStore            sops.Store
-	InputPath             string
-	IgnoreMAC             bool
-	KeyServices           []keyservice.KeyServiceClient
-	DecryptionOrder       []string
-	DecryptionCredentials map[string]string
+	Cipher          sops.Cipher
+	InputStore      sops.Store
+	InputPath       string
+	IgnoreMAC       bool
+	KeyServices     []keyservice.KeyServiceClient
+	DecryptionOrder []string
 }
 
 // LoadEncryptedFileWithBugFixes is a wrapper around LoadEncryptedFile which includes
@@ -301,7 +299,7 @@ func FixAWSKMSEncryptionContextBug(opts GenericDecryptOpts, tree *sops.Tree) (*s
 		return nil, NewExitError(fmt.Sprintf("Failed to decrypt, meaning there is likely another problem from the encryption context bug: %s", err), codes.ErrorDecryptingTree)
 	}
 
-	errs := tree.Metadata.UpdateMasterKeysWithKeyServices(dataKey, opts.KeyServices, opts.DecryptionCredentials)
+	errs := tree.Metadata.UpdateMasterKeysWithKeyServices(dataKey, opts.KeyServices)
 	if len(errs) > 0 {
 		err = fmt.Errorf("Could not re-encrypt data key: %s", errs)
 		return nil, err

cmd/sops/decrypt.go

@@ -15,19 +15,18 @@ const notBinaryHint = ("This is likely not an encrypted binary file?" +
 	" If not, use --output-type to select the correct output type.")
 
 type decryptOpts struct {
-	Cipher                sops.Cipher
-	InputStore            sops.Store
-	OutputStore           sops.Store
-	InputPath             string
-	IgnoreMAC             bool
-	Extract               []interface{}
-	KeyServices           []keyservice.KeyServiceClient
-	DecryptionOrder       []string
-	DecryptionCredentials map[string]string
+	Cipher          sops.Cipher
+	InputStore      sops.Store
+	OutputStore     sops.Store
+	InputPath       string
+	IgnoreMAC       bool
+	Extract         []interface{}
+	KeyServices     []keyservice.KeyServiceClient
+	DecryptionOrder []string
 }
 
-func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
-	tree, err := common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{
+func decryptTree(opts decryptOpts) (tree *sops.Tree, err error) {
+	tree, err = common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{
 		Cipher:      opts.Cipher,
 		InputStore:  opts.InputStore,
 		InputPath:   opts.InputPath,
@@ -39,17 +38,25 @@ func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
 	}
 
 	_, err = common.DecryptTree(common.DecryptTreeOpts{
-		Cipher:                opts.Cipher,
-		IgnoreMac:             opts.IgnoreMAC,
-		Tree:                  tree,
-		KeyServices:           opts.KeyServices,
-		DecryptionOrder:       opts.DecryptionOrder,
-		DecryptionCredentials: opts.DecryptionCredentials,
+		Cipher:          opts.Cipher,
+		IgnoreMac:       opts.IgnoreMAC,
+		Tree:            tree,
+		KeyServices:     opts.KeyServices,
+		DecryptionOrder: opts.DecryptionOrder,
 	})
 	if err != nil {
 		return nil, err
 	}
 
+	return tree, nil
+}
+
+func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
+	tree, err := decryptTree(opts)
+	if err != nil {
+		return nil, err
+	}
+
 	if len(opts.Extract) > 0 {
 		return extract(tree, opts.Extract, opts.OutputStore)
 	}

cmd/sops/edit.go

@@ -20,15 +20,14 @@ import (
 )
 
 type editOpts struct {
-	Cipher                sops.Cipher
-	InputStore            common.Store
-	OutputStore           common.Store
-	InputPath             string
-	IgnoreMAC             bool
-	KeyServices           []keyservice.KeyServiceClient
-	DecryptionOrder       []string
-	ShowMasterKeys        bool
-	DecryptionCredentials map[string]string
+	Cipher          sops.Cipher
+	InputStore      common.Store
+	OutputStore     common.Store
+	InputPath       string
+	IgnoreMAC       bool
+	KeyServices     []keyservice.KeyServiceClient
+	DecryptionOrder []string
+	ShowMasterKeys  bool
 }
 
 type editExampleOpts struct {
@@ -61,7 +60,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 	}
 
 	// Generate a data key
-	dataKey, errs := tree.GenerateDataKeyWithKeyServices(opts.KeyServices, opts.DecryptionCredentials)
+	dataKey, errs := tree.GenerateDataKeyWithKeyServices(opts.KeyServices)
 	if len(errs) > 0 {
 		return nil, common.NewExitError(fmt.Sprintf("Error encrypting the data key with one or more master keys: %s", errs), codes.CouldNotRetrieveKey)
 	}
@@ -72,24 +71,22 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 func edit(opts editOpts) ([]byte, error) {
 	// Load the file
 	tree, err := common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{
-		Cipher:                opts.Cipher,
-		InputStore:            opts.InputStore,
-		InputPath:             opts.InputPath,
-		IgnoreMAC:             opts.IgnoreMAC,
-		KeyServices:           opts.KeyServices,
-		DecryptionCredentials: opts.DecryptionCredentials,
+		Cipher:      opts.Cipher,
+		InputStore:  opts.InputStore,
+		InputPath:   opts.InputPath,
+		IgnoreMAC:   opts.IgnoreMAC,
+		KeyServices: opts.KeyServices,
 	})
 	if err != nil {
 		return nil, err
 	}
 	// Decrypt the file
 	dataKey, err := common.DecryptTree(common.DecryptTreeOpts{
-		Cipher:                opts.Cipher,
-		IgnoreMac:             opts.IgnoreMAC,
-		Tree:                  tree,
-		KeyServices:           opts.KeyServices,
-		DecryptionOrder:       opts.DecryptionOrder,
-		DecryptionCredentials: opts.DecryptionCredentials,
+		Cipher:          opts.Cipher,
+		IgnoreMac:       opts.IgnoreMAC,
+		Tree:            tree,
+		KeyServices:     opts.KeyServices,
+		DecryptionOrder: opts.DecryptionOrder,
 	})
 	if err != nil {
 		return nil, err