Merge branch 'open-policy-agent:main' into fix_5879_1

.github/workflows/nightly.yaml

@@ -66,7 +66,7 @@ jobs:
 
       - name: Benchmark Test Golang
         run: make ci-go-perf
-        timeout-minutes: 30
+        timeout-minutes: 45
         env:
           DOCKER_RUNNING: 0
 

.github/workflows/pull-request.yaml

@@ -274,7 +274,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04, macos-latest]
-        version: ["1.18", "1.19"]
+        version: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Download generated artifacts

.go-version

@@ -1 +1 @@
-1.20.5
+1.20.6

CHANGELOG.md

@@ -5,6 +5,50 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## 0.54.0
+
+This release focuses on bug fixes, but also includes some improvements to the SDK and commandline.
+
+Note: This will be the last OPA release to support building with Golang 1.18. (Golang 1.21 is expected to be released in August. Keeping the support for 1.18 is blocking OPA from upgrading OpenTelemetry.)
+
+### Topdown and Rego
+
+- Add unwrap functionality to topdown.Error ([#5890](https://github.com/open-policy-agent/opa/issues/5890)) authored by @ajith-sub reported by @ajith-sub
+- Lazy obj performance ([#6009](https://github.com/open-policy-agent/opa/issues/6009)) authored by @johanfylling reported by @kubaj
+- ast: Only realizing `lazyObj` when compared against other object type ([6060](https://github.com/open-policy-agent/opa/pull/6060)) (authored by @johanfylling)
+- ast: Fixing issue in type-checker where partial objects couldn't have key overrides of divergent type ([#5972](https://github.com/open-policy-agent/opa/issues/5972)) authored by @johanfylling
+- planner: CallDynamic regression fix ([#5964](https://github.com/open-policy-agent/opa/issues/5964)) authored by @srenatus
+- fmt: Fix `fmt` panic in comprehension with comments ([#5798](https://github.com/open-policy-agent/opa/issues/5798)) authored by @Trolloldem reported by @Djoust
+- topdown: Format integer numbers without exponent ([#6013](https://github.com/open-policy-agent/opa/issues/6013)) authored by @kenjenkins reported by @kenjenkins
+- topdown: Fix panic in partial eval with ref head rule ([#6027](https://github.com/open-policy-agent/opa/issues/6027)) authored by @srenatus
+- Fixed a bug in `object.union_n` where nested objects were mutated ([#5975](https://github.com/open-policy-agent/opa/issues/5975)) authored by @qshu-splunk
+- Fixed the issue of the `object.subset` method failing to correctly compare array relationships ([5968](https://github.com/open-policy-agent/opa/issues/5968)) authored by @DCRUNNN
+- topdown: Fixed caching race condition issue in `http.send` ([#5997](https://github.com/open-policy-agent/opa/pull/5997)) authored by @ashutosh-narkar
+- Allow time formatting constants in rego `time.format` and `time.parse_ns` ([#5945](https://github.com/open-policy-agent/opa/issues/5945)) authored by @tjons
+
+### Runtime, Tooling, SDK
+
+- Add `--schema` flag to `opa test` ([#5923](https://github.com/open-policy-agent/opa/issues/5923)) authored by @renatosc
+- Add ability to specify namespace for optimized files ([#5933](https://github.com/open-policy-agent/opa/issues/5933)) authored by @ashutosh-narkar reported by @deezkay
+- Fix for the issue when OPA throws misleading error (storage_not_found_error) message while loading the delta bundle when persist property in config is true. ([#5959](https://github.com/open-policy-agent/opa/issues/5959)) authored by @yogisinha reported by @jnethery
+- cmd: Update storage when a file remove op is detected ([#5986](https://github.com/open-policy-agent/opa/issues/5986)) authored by @boranx
+- cmd: Add support for watch mode in opa test ([#1719](https://github.com/open-policy-agent/opa/issues/1719)) authored by @ashutosh-narkar reported by @Fox32
+- download: Pass request to docker.Authorizer ([#5902](https://github.com/open-policy-agent/opa/issues/5902)) authored by @DerGut reported by @carabasdaniel
+- plugins/discovery: Fix discovery erasing `persistence_directory` config ([#6042](https://github.com/open-policy-agent/opa/pull/6042)) authored by @blacksails
+- plugins/discovery: Fix persistence of discovery bundle ([#6048](https://github.com/open-policy-agent/opa/pull/6048)) (authored by @bdjgs)
+- Add tracing to bundle/discovery download ([#5967](https://github.com/open-policy-agent/opa/issues/5967)) authored by @mjungsbluth
+- Fallback on embedded timezone database if `tzdata` is not found on filesystem ([6038](https://github.com/open-policy-agent/opa/pull/6038)) authored by @charlieegan3
+- extensibility: Adding hooks (plugins, discovery, sdk) ([#6053](https://github.com/open-policy-agent/opa/pull/6053)) authored by @srenatus
+- sdk: allow passing in a separate `Store` implementation in SDK ([5962](https://github.com/open-policy-agent/opa/pull/5962)) authored by @srenatus
+- config: Show "extra", unknown fields in `/v1/config` API result ([6056](https://github.com/open-policy-agent/opa/pull/6056)) authored by @srenatus
+
+### Miscellaneous
+- Disable provenance attestations in buildx ([#5877](https://github.com/open-policy-agent/opa/issues/5877)) authored by @ashutosh-narkar reported by @JasonMan34
+- build: configure SELinux labels for Docker volumes ([#6054](https://github.com/open-policy-agent/opa/issues/6054)) authored by @zregvart reported by @zregvart
+- Dependency bumps, notably:
+  - golang from 1.20.4 to 1.20.5
+  - github.com/prometheus/client_golang from from 1.15.1 to v1.16.0
+
 ## 0.53.1
 
 This is a bug fix release addressing the following issues:

Makefile

@@ -62,7 +62,7 @@ TELEMETRY_URL ?= #Default empty
 
 BUILD_HOSTNAME := $(shell ./build/get-build-hostname.sh)
 
-RELEASE_BUILD_IMAGE := golang:$(GOVERSION)
+RELEASE_BUILD_IMAGE := golang:$(GOVERSION)-bullseye
 
 RELEASE_DIR ?= _release/$(VERSION)
 
@@ -142,15 +142,15 @@ wasm-sdk-e2e-test: generate
 .PHONY: check
 check:
 ifeq ($(DOCKER_RUNNING), 1)
-	docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:${GOLANGCI_LINT_VERSION} golangci-lint run -v
+	docker run --rm -v $(shell pwd):/app:ro,Z -w /app golangci/golangci-lint:${GOLANGCI_LINT_VERSION} golangci-lint run -v
 else
 	@echo "Docker not installed or running. Skipping golangci run."
 endif
 
 .PHONY: fmt
 fmt:
 ifeq ($(DOCKER_RUNNING), 1)
-	docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:${GOLANGCI_LINT_VERSION} golangci-lint run -v --fix
+	docker run --rm -v $(shell pwd):/app:Z -w /app golangci/golangci-lint:${GOLANGCI_LINT_VERSION} golangci-lint run -v --fix
 else
 	@echo "Docker not installed or running. Skipping golangci run."
 endif
@@ -249,7 +249,7 @@ CI_GOLANG_DOCKER_MAKE := $(DOCKER) run \
 	-e WASM_ENABLED=$(WASM_ENABLED) \
 	-e FUZZ_TIME=$(FUZZ_TIME) \
 	-e TELEMETRY_URL=$(TELEMETRY_URL) \
-	golang:$(GOVERSION)
+	$(RELEASE_BUILD_IMAGE)
 
 .PHONY: ci-go-%
 ci-go-%: generate
@@ -331,20 +331,20 @@ image-quick-%: ensure-executable-bin
 ifneq ($(GOARCH),arm64) # build only static images for arm64
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION) \
-		--build-arg BASE=cgr.dev/chainguard/cc-dynamic \
+		--build-arg BASE=cgr.dev/chainguard/glibc-dynamic \
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-debug \
-		--build-arg BASE=cgr.dev/chainguard/cc-dynamic:latest-dev \
+		--build-arg BASE=cgr.dev/chainguard/glibc-dynamic:latest-dev \
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-rootless \
 		--build-arg OPA_DOCKER_IMAGE_TAG=rootless \
-		--build-arg BASE=cgr.dev/chainguard/cc-dynamic:latest \
+		--build-arg BASE=cgr.dev/chainguard/glibc-dynamic:latest \
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--platform linux/$* \
 		.
@@ -370,25 +370,27 @@ endif
 push-manifest-list-%: ensure-executable-bin
 	$(DOCKER) buildx build \
 		--tag $(DOCKER_IMAGE):$* \
-		--build-arg BASE=cgr.dev/chainguard/cc-dynamic:latest \
+		--build-arg BASE=cgr.dev/chainguard/glibc-dynamic:latest \
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--platform $(DOCKER_PLATFORMS) \
+		--provenance=false \
 		--push \
 		.
-	# TODO: update busybox shell debug images to image without openssl
 	$(DOCKER) buildx build \
 		--tag $(DOCKER_IMAGE):$*-debug \
-		--build-arg BASE=cgr.dev/chainguard/cc-dynamic:latest-dev \
+		--build-arg BASE=cgr.dev/chainguard/glibc-dynamic:latest-dev \
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--platform $(DOCKER_PLATFORMS) \
+		--provenance=false \
 		--push \
 		.
 	$(DOCKER) buildx build \
 		--tag $(DOCKER_IMAGE):$*-rootless \
 		--build-arg OPA_DOCKER_IMAGE_TAG=rootless \
-		--build-arg BASE=cgr.dev/chainguard/cc-dynamic:latest \
+		--build-arg BASE=cgr.dev/chainguard/glibc-dynamic:latest \
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--platform $(DOCKER_PLATFORMS) \
+		--provenance=false \
 		--push \
 		.
 
@@ -398,6 +400,7 @@ push-manifest-list-%: ensure-executable-bin
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--build-arg BIN_SUFFIX=_static \
 		--platform $(DOCKER_PLATFORMS_STATIC) \
+		--provenance=false \
 		--push \
 		.
 
@@ -407,6 +410,7 @@ push-manifest-list-%: ensure-executable-bin
 		--build-arg BIN_DIR=$(RELEASE_DIR) \
 		--build-arg BIN_SUFFIX=_static \
 		--platform $(DOCKER_PLATFORMS_STATIC) \
+		--provenance=false \
 		--push \
 		.
 
@@ -479,10 +483,10 @@ check-go-module:
 	docker run \
 	  $(DOCKER_FLAGS) \
 	  -w /src \
-	  -v $(PWD):/src \
+	  -v $(PWD):/src:Z \
 	  -e 'GOPRIVATE=*' \
 	  --tmpfs /src/.go \
-	  golang:$(GOVERSION) \
+	  $(RELEASE_BUILD_IMAGE) \
 	  /bin/bash -c "git config --system --add safe.directory /src && go mod vendor -v"
 
 ######################################################
@@ -499,15 +503,15 @@ endif
 	@$(DOCKER) run $(DOCKER_FLAGS) \
 		-e GITHUB_TOKEN=$(GITHUB_TOKEN) \
 		-e LAST_VERSION=$(LAST_VERSION) \
-		-v $(PWD):/_src \
-		cmd.cat/make/git/go/python3/perl \
+		-v $(PWD):/_src:Z \
+		ashtalk/python-go-perl:v1 \
 		/_src/build/gen-release-patch.sh --version=$(VERSION) --source-url=/_src
 
 .PHONY: dev-patch
 dev-patch:
 	@$(DOCKER) run $(DOCKER_FLAGS) \
-		-v $(PWD):/_src \
-		cmd.cat/make/git/go/python3/perl \
+		-v $(PWD):/_src:Z \
+		ashtalk/python-go-perl:v1 \
 		/_src/build/gen-dev-patch.sh --version=$(VERSION) --source-url=/_src
 
 # Deprecated targets. To be removed.
@@ -542,16 +546,16 @@ depr-build-windows: ensure-release-dir
 
 depr-release:
 	$(DOCKER) run $(DOCKER_FLAGS) \
-		-v $(PWD)/$(RELEASE_DIR):/$(RELEASE_DIR) \
-		-v $(PWD):/_src \
+		-v $(PWD)/$(RELEASE_DIR):/$(RELEASE_DIR):Z \
+		-v $(PWD):/_src:Z \
 		-e TELEMETRY_URL=$(TELEMETRY_URL) \
 		$(RELEASE_BUILD_IMAGE) \
 		/_src/build/build-release.sh --version=$(VERSION) --output-dir=/$(RELEASE_DIR) --source-url=/_src
 
 depr-release-local:
 	$(DOCKER) run $(DOCKER_FLAGS) \
-		-v $(PWD)/$(RELEASE_DIR):/$(RELEASE_DIR) \
-		-v $(PWD):/_src \
+		-v $(PWD)/$(RELEASE_DIR):/$(RELEASE_DIR):Z \
+		-v $(PWD):/_src:Z \
 		-e TELEMETRY_URL=$(TELEMETRY_URL) \
 		$(RELEASE_BUILD_IMAGE) \
 		/_src/build/build-release.sh --output-dir=/$(RELEASE_DIR) --source-url=/_src

ast/builtins.go

@@ -211,6 +211,7 @@ var DefaultBuiltins = [...]*Builtin{
 	CryptoX509ParseCertificateRequest,
 	CryptoX509ParseRSAPrivateKey,
 	CryptoX509ParseKeyPair,
+	CryptoParsePrivateKeys,
 	CryptoHmacMd5,
 	CryptoHmacSha1,
 	CryptoHmacSha256,
@@ -2160,7 +2161,7 @@ var Format = &Builtin{
 				types.N,
 				types.NewArray([]types.Type{types.N, types.S}, nil),
 				types.NewArray([]types.Type{types.N, types.S, types.S}, nil),
-			)).Description("a number representing the nanoseconds since the epoch (UTC); or a two-element array of the nanoseconds, and a timezone string; or a three-element array of ns, timezone string and a layout string (see golang supported time formats)"),
+			)).Description("a number representing the nanoseconds since the epoch (UTC); or a two-element array of the nanoseconds, and a timezone string; or a three-element array of ns, timezone string and a layout string or golang defined formatting constant (see golang supported time formats)"),
 		),
 		types.Named("formatted timestamp", types.S).Description("the formatted timestamp represented for the nanoseconds since the epoch in the supplied timezone (or UTC)"),
 	),
@@ -2312,6 +2313,19 @@ var CryptoX509ParseRSAPrivateKey = &Builtin{
 	),
 }
 
+var CryptoParsePrivateKeys = &Builtin{
+	Name: "crypto.parse_private_keys",
+	Description: `Returns zero or more private keys from the given encoded string containing DER certificate data.
+
+If the input is empty, the function will return null. The input string should be a list of one or more concatenated PEM blocks. The whole input of concatenated PEM blocks can optionally be Base64 encoded.`,
+	Decl: types.NewFunction(
+		types.Args(
+			types.Named("keys", types.S).Description("PEM encoded data containing one or more private keys as concatenated blocks. Optionally Base64 encoded."),
+		),
+		types.Named("output", types.NewArray(nil, types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)))).Description("parsed private keys represented as objects"),
+	),
+}
+
 var CryptoMd5 = &Builtin{
 	Name:        "crypto.md5",
 	Description: "Returns a string representing the input string hashed with the MD5 function",

ast/check.go

@@ -269,7 +269,7 @@ func (tc *typeChecker) checkRule(env *TypeEnv, as *AnnotationSet, rule *Rule) {
 	}
 
 	if tpe != nil {
-		env.tree.Put(path, tpe)
+		env.tree.Insert(path, tpe)
 	}
 }
 

ast/check_test.go

@@ -351,6 +351,12 @@ func TestCheckInferenceRules(t *testing.T) {
 		{`ref_rule_single_with_number_key`, `p.q[3] { true }`},
 		{`ref_regression_array_key`,
 			`walker[[p, v]] = o { l = input; walk(l, k); [p, v] = k; o = {} }`},
+		{`overlap`, `p.q[r] = y { x = ["a", "b"]; y = x[r] }`},
+		{`overlap`, `p.q.r = false { true }`},
+		{`overlap`, `p.q.r = "false" { true }`},
+		{`overlap`, `p.q[42] = 1337 { true }`},
+		{`overlap`, `p.q.a = input.a { true }`},
+		{`overlap`, `p.q[56] = input.a { true }`},
 	}
 
 	tests := []struct {
@@ -504,6 +510,50 @@ func TestCheckInferenceRules(t *testing.T) {
 				types.NewDynamicProperty(types.NewArray([]types.Type{types.NewArray(types.A, types.A), types.A}, nil),
 					types.NewObject(nil, types.NewDynamicProperty(types.A, types.A))),
 			)},
+		{
+			note:     "ref-rules single value, full ref to known leaf",
+			rules:    ruleset2,
+			ref:      "data.overlap.p.q.r",
+			expected: types.NewAny(types.B, types.S),
+		},
+		{
+			note:     "ref-rules single value, full ref to known leaf (same key type as dynamic, different value type)",
+			rules:    ruleset2,
+			ref:      "data.overlap.p.q[42]",
+			expected: types.N,
+		},
+		{
+			note:     "ref-rules single value, full ref to known leaf (any type)",
+			rules:    ruleset2,
+			ref:      "data.overlap.p.q.a",
+			expected: types.A,
+		},
+		{
+			note:     "ref-rules single value, full ref to known leaf (same key type as dynamic, any type)",
+			rules:    ruleset2,
+			ref:      "data.overlap.p.q[56]",
+			expected: types.A,
+		},
+		{
+			note:     "ref-rules single value, full ref to dynamic leaf",
+			rules:    ruleset2,
+			ref:      "data.overlap.p.q[1]",
+			expected: types.S,
+		},
+		{
+			note:  "ref-rules single value, prefix ref to partial object root",
+			rules: ruleset2,
+			ref:   "data.overlap.p.q",
+			expected: types.NewObject(
+				[]*types.StaticProperty{
+					types.NewStaticProperty(json.Number("42"), types.N),
+					types.NewStaticProperty(json.Number("56"), types.A),
+					types.NewStaticProperty("a", types.A),
+					types.NewStaticProperty("r", types.Or(types.B, types.S)),
+				},
+				types.NewDynamicProperty(types.N, types.S),
+			),
+		},
 	}
 
 	for _, tc := range tests {

ast/compile_test.go

@@ -1844,6 +1844,41 @@ bar.baz contains "quz" if true`,
 	assertCompilerErrorStrings(t, c, expected)
 }
 
+func TestCompilerCheckRuleConflictsDefaultFunction(t *testing.T) {
+	tests := []struct {
+		note    string
+		modules []*Module
+		err     string
+	}{
+		{
+			note: "conflicting rules",
+			modules: modules(
+				`package pkg
+				default f(_) = 100
+				f(x, y) = x {
+                   x == y
+				}`),
+			err: "rego_type_error: conflicting rules data.pkg.f found",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.note, func(t *testing.T) {
+			mods := make(map[string]*Module, len(tc.modules))
+			for i, m := range tc.modules {
+				mods[fmt.Sprint(i)] = m
+			}
+			c := NewCompiler()
+			c.Modules = mods
+			compileStages(c, c.checkRuleConflicts)
+			if tc.err != "" {
+				assertCompilerErrorStrings(t, c, []string{tc.err})
+			} else {
+				assertCompilerErrorStrings(t, c, []string{})
+			}
+		})
+	}
+}
+
 func TestCompilerCheckRuleConflictsDotsInRuleHeads(t *testing.T) {
 
 	tests := []struct {