Merge branch 'main' into verification-oci-signatures

.github/workflows/pull-request.yaml

@@ -311,15 +311,30 @@ jobs:
       - name: Ensure proper formatting
         run: opa fmt --list --fail build/policy
 
-      - name: Run policy checks on changed files
+      - name: Run file policy checks on changed files
         run: |
           curl --silent --fail --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -o files.json \
             https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files
-          opa eval --bundle build/policy/ --format values --input files.json \
+          
+          opa eval -d build/policy/files.rego -d build/policy/helpers.rego  --format values --input files.json \
             --fail-defined 'data.files.deny[message]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Show input on failure
         run: opa eval --input files.json --format pretty input
         if: ${{ failure() }}
+
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@v2
+        with:
+          # keep this version in sync with the version in netlify.toml
+          hugo-version: '0.113.0'
+          extended: true
+
+      - name: Build docs site and test integrations data
+        run: |
+          cd docs
+          make dev-generate hugo-production-build
+          cd -
+          opa eval 'data.integrations.deny[message]' -i docs/website/public/index.json -d build/policy/integrations.rego --format=values --fail-defined

.go-version

@@ -1 +1 @@
-1.20.7
+1.21

ast/check.go

@@ -268,7 +268,7 @@ func (tc *typeChecker) checkRule(env *TypeEnv, as *AnnotationSet, rule *Rule) {
 	}
 
 	if tpe != nil {
-		env.tree.Insert(path, tpe)
+		env.tree.Insert(path, tpe, env)
 	}
 }
 

ast/check_test.go

@@ -355,8 +355,8 @@ func TestCheckInferenceRules(t *testing.T) {
 		{`overlap`, `p.q.r = false { true }`},
 		{`overlap`, `p.q.r = "false" { true }`},
 		{`overlap`, `p.q[42] = 1337 { true }`},
-		{`overlap`, `p.q.a = input.a { true }`},
-		{`overlap`, `p.q[56] = input.a { true }`},
+		{`overlap`, `p.q2.a = input.a { true }`},
+		{`overlap`, `p.q2[56] = input.a { true }`},
 	}
 
 	tests := []struct {
@@ -525,33 +525,28 @@ func TestCheckInferenceRules(t *testing.T) {
 		{
 			note:     "ref-rules single value, full ref to known leaf (any type)",
 			rules:    ruleset2,
-			ref:      "data.overlap.p.q.a",
+			ref:      "data.overlap.p.q2.a",
 			expected: types.A,
 		},
 		{
 			note:     "ref-rules single value, full ref to known leaf (same key type as dynamic, any type)",
 			rules:    ruleset2,
-			ref:      "data.overlap.p.q[56]",
+			ref:      "data.overlap.p.q2[56]",
 			expected: types.A,
 		},
 		{
 			note:     "ref-rules single value, full ref to dynamic leaf",
 			rules:    ruleset2,
 			ref:      "data.overlap.p.q[1]",
-			expected: types.S,
+			expected: types.Any{types.B, types.N, types.S}, // key type cannot be tied to specific dynamic value type, so we get all of them
 		},
 		{
 			note:  "ref-rules single value, prefix ref to partial object root",
 			rules: ruleset2,
 			ref:   "data.overlap.p.q",
 			expected: types.NewObject(
-				[]*types.StaticProperty{
-					types.NewStaticProperty(json.Number("42"), types.N),
-					types.NewStaticProperty(json.Number("56"), types.A),
-					types.NewStaticProperty("a", types.A),
-					types.NewStaticProperty("r", types.Or(types.B, types.S)),
-				},
-				types.NewDynamicProperty(types.N, types.S),
+				nil,
+				types.NewDynamicProperty(types.Any{types.N, types.S}, types.Any{types.B, types.N, types.S}),
 			),
 		},
 	}

ast/compile.go

@@ -698,15 +698,15 @@ func (c *Compiler) GetRulesDynamicWithOpts(ref Ref, opts RulesOptions) []*Rule {
 			// The head of the ref is always grounded.  In case another part of the
 			// ref is also grounded, we can lookup the exact child.  If it's not found
 			// we can immediately return...
-			if child := node.Child(ref[i].Value); child == nil {
-				return
-			} else if len(child.Values) > 0 {
-				// If there are any rules at this position, it's what the ref would
-				// refer to.  We can just append those and stop here.
-				insertRules(set, child.Values)
-			} else {
-				// Otherwise, we continue using the child node.
+			if child := node.Child(ref[i].Value); child != nil {
+				if len(child.Values) > 0 {
+					// Add any rules at this position
+					insertRules(set, child.Values)
+				}
+				// There might still be "sub-rules" contributing key-value "overrides" for e.g. partial object rules, continue walking
 				walk(child, i+1)
+			} else {
+				return
 			}
 
 		default:

ast/compile_test.go

@@ -7771,6 +7771,11 @@ r2 = 2`,
 r3 = 3`,
 		"hidden": `package system.hidden
 r4 = 4`,
+		"mod4": `package b.c
+r5[x] = 5 { x := "foo" }
+r5.bar = 6 { input.x }
+r5.baz = 7 { input.y }
+`,
 	})
 
 	compileStages(compiler, nil)
@@ -7780,6 +7785,9 @@ r4 = 4`,
 	rule2 := compiler.Modules["mod2"].Rules[1]
 	rule3 := compiler.Modules["mod3"].Rules[0]
 	rule4 := compiler.Modules["hidden"].Rules[0]
+	rule5 := compiler.Modules["mod4"].Rules[0]
+	rule5b := compiler.Modules["mod4"].Rules[1]
+	rule5c := compiler.Modules["mod4"].Rules[2]
 
 	tests := []struct {
 		input         string
@@ -7791,12 +7799,16 @@ r4 = 4`,
 		{input: "data.a.b[x].d", expected: []*Rule{rule1, rule3}},
 		{input: "data.a.b.c", expected: []*Rule{rule1, rule2d, rule2}},
 		{input: "data.a.b.d"},
-		{input: "data", expected: []*Rule{rule1, rule2d, rule2, rule3, rule4}},
-		{input: "data[x]", expected: []*Rule{rule1, rule2d, rule2, rule3, rule4}},
+		{input: "data", expected: []*Rule{rule1, rule2d, rule2, rule3, rule4, rule5, rule5b, rule5c}},
+		{input: "data[x]", expected: []*Rule{rule1, rule2d, rule2, rule3, rule4, rule5, rule5b, rule5c}},
 		{input: "data[data.complex_computation].b[y]", expected: []*Rule{rule1, rule2d, rule2, rule3}},
 		{input: "data[x][y].c.e", expected: []*Rule{rule2d, rule2}},
 		{input: "data[x][y].r3", expected: []*Rule{rule3}},
-		{input: "data[x][y]", expected: []*Rule{rule1, rule2d, rule2, rule3}, excludeHidden: true}, // old behaviour of GetRulesDynamic
+		{input: "data[x][y]", expected: []*Rule{rule1, rule2d, rule2, rule3, rule5, rule5b, rule5c}, excludeHidden: true}, // old behaviour of GetRulesDynamic
+		{input: "data.b.c", expected: []*Rule{rule5, rule5b, rule5c}},
+		{input: "data.b.c.r5", expected: []*Rule{rule5, rule5b, rule5c}},
+		{input: "data.b.c.r5.bar", expected: []*Rule{rule5, rule5b}}, // rule5 might still define a value for the "bar" key
+		{input: "data.b.c.r5.baz", expected: []*Rule{rule5, rule5c}},
 	}
 
 	for _, tc := range tests {

ast/env.go

@@ -6,6 +6,7 @@ package ast
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/open-policy-agent/opa/types"
 	"github.com/open-policy-agent/opa/util"
@@ -171,6 +172,11 @@ func (env *TypeEnv) getRefRec(node *typeTreeNode, ref, tail Ref) types.Type {
 	}
 
 	if node.Leaf() {
+		if node.children.Len() > 0 {
+			if child := node.Child(tail[0].Value); child != nil {
+				return env.getRefRec(child, ref, tail[1:])
+			}
+		}
 		return selectRef(node.Value(), tail)
 	}
 
@@ -305,9 +311,9 @@ func (n *typeTreeNode) Put(path Ref, tpe types.Type) {
 }
 
 // Insert inserts tpe at path in the tree, but also merges the value into any types.Object present along that path.
-// If an types.Object is inserted, any leafs already present further down the tree are merged into the inserted object.
+// If a types.Object is inserted, any leafs already present further down the tree are merged into the inserted object.
 // path must be ground.
-func (n *typeTreeNode) Insert(path Ref, tpe types.Type) {
+func (n *typeTreeNode) Insert(path Ref, tpe types.Type, env *TypeEnv) {
 	curr := n
 	for i, term := range path {
 		c, ok := curr.children.Get(term.Value)
@@ -324,7 +330,7 @@ func (n *typeTreeNode) Insert(path Ref, tpe types.Type) {
 				// If child has an object value, merge the new value into it.
 				if o, ok := child.value.(*types.Object); ok {
 					var err error
-					child.value, err = insertIntoObject(o, path[i+1:], tpe)
+					child.value, err = insertIntoObject(o, path[i+1:], tpe, env)
 					if err != nil {
 						panic(fmt.Errorf("unreachable, insertIntoObject: %w", err))
 					}
@@ -335,62 +341,131 @@ func (n *typeTreeNode) Insert(path Ref, tpe types.Type) {
 		curr = child
 	}
 
-	curr.value = tpe
+	curr.value = mergeTypes(curr.value, tpe)
 
 	if _, ok := tpe.(*types.Object); ok && curr.children.Len() > 0 {
 		// merge all leafs into the inserted object
 		leafs := curr.Leafs()
 		for p, t := range leafs {
 			var err error
-			curr.value, err = insertIntoObject(curr.value.(*types.Object), *p, t)
+			curr.value, err = insertIntoObject(curr.value.(*types.Object), *p, t, env)
 			if err != nil {
 				panic(fmt.Errorf("unreachable, insertIntoObject: %w", err))
 			}
 		}
 	}
 }
 
-func insertIntoObject(o *types.Object, path Ref, tpe types.Type) (*types.Object, error) {
-	if len(path) == 0 {
-		return o, nil
+// mergeTypes merges the types of 'a' and 'b'. If both are sets, their 'of' types are joined with an types.Or.
+// If both are objects, the key and value types of their dynamic properties are joined with types.Or:s.
+// If 'a' and 'b' are both objects, and at least one of them have static properties, they are joined
+// with an types.Or, instead of being merged.
+// If 'a' is an Any containing an Object, and 'b' is an Object (or vice versa); AND both objects have no
+// static properties, they are merged.
+// If 'a' and 'b' are different types, they are joined with an types.Or.
+func mergeTypes(a, b types.Type) types.Type {
+	if a == nil {
+		return b
 	}
 
-	key, err := JSON(path[0].Value)
-	if err != nil {
-		return nil, fmt.Errorf("invalid path term %v: %w", path[0], err)
+	if b == nil {
+		return a
 	}
 
-	if len(path) == 1 {
-		for _, prop := range o.StaticProperties() {
-			if util.Compare(prop.Key, key) == 0 {
-				prop.Value = types.Or(prop.Value, tpe)
-				return o, nil
+	switch a := a.(type) {
+	case *types.Object:
+		if bObj, ok := b.(*types.Object); ok && len(a.StaticProperties()) == 0 && len(bObj.StaticProperties()) == 0 {
+			if len(a.StaticProperties()) > 0 || len(bObj.StaticProperties()) > 0 {
+				return types.Or(a, bObj)
 			}
-		}
-		staticProps := append(o.StaticProperties(), types.NewStaticProperty(key, tpe))
-		return types.NewObject(staticProps, o.DynamicProperties()), nil
-	}
 
-	for _, prop := range o.StaticProperties() {
-		if util.Compare(prop.Key, key) == 0 {
-			if propO := prop.Value.(*types.Object); propO != nil {
-				prop.Value, err = insertIntoObject(propO, path[1:], tpe)
-				if err != nil {
-					return nil, err
+			aDynProps := a.DynamicProperties()
+			bDynProps := bObj.DynamicProperties()
+			return types.NewObject(nil, types.NewDynamicProperty(
+				types.Or(aDynProps.Key, bDynProps.Key),
+				types.Or(aDynProps.Value, bDynProps.Value)))
+		} else if bAny, ok := b.(types.Any); ok && len(a.StaticProperties()) == 0 {
+			// If a is an object type with no static components ...
+			for _, t := range bAny {
+				if tObj, ok := t.(*types.Object); ok && len(tObj.StaticProperties()) == 0 {
+					// ... and b is a types.Any containing an object with no static components, we merge them.
+					aDynProps := a.DynamicProperties()
+					tDynProps := tObj.DynamicProperties()
+					tDynProps.Key = types.Or(tDynProps.Key, aDynProps.Key)
+					tDynProps.Value = types.Or(tDynProps.Value, aDynProps.Value)
+					return bAny
 				}
-			} else {
-				return nil, fmt.Errorf("cannot insert into non-object type %v", prop.Value)
 			}
-			return o, nil
 		}
+	case *types.Set:
+		if bSet, ok := b.(*types.Set); ok {
+			return types.NewSet(types.Or(a.Of(), bSet.Of()))
+		}
+	case types.Any:
+		if _, ok := b.(types.Any); !ok {
+			return mergeTypes(b, a)
+		}
+	}
+
+	return types.Or(a, b)
+}
+
+func (n *typeTreeNode) String() string {
+	b := strings.Builder{}
+
+	if k := n.key; k != nil {
+		b.WriteString(k.String())
+	} else {
+		b.WriteString("-")
+	}
+
+	if v := n.value; v != nil {
+		b.WriteString(": ")
+		b.WriteString(v.String())
+	}
+
+	n.children.Iter(func(_, v util.T) bool {
+		if child, ok := v.(*typeTreeNode); ok {
+			b.WriteString("\n\t+ ")
+			s := child.String()
+			s = strings.ReplaceAll(s, "\n", "\n\t")
+			b.WriteString(s)
+		}
+		return false
+	})
+
+	return b.String()
+}
+
+func insertIntoObject(o *types.Object, path Ref, tpe types.Type, env *TypeEnv) (*types.Object, error) {
+	if len(path) == 0 {
+		return o, nil
 	}
 
-	child, err := insertIntoObject(types.NewObject(nil, nil), path[1:], tpe)
+	key := env.Get(path[0].Value)
+
+	if len(path) == 1 {
+		var dynamicProps *types.DynamicProperty
+		if dp := o.DynamicProperties(); dp != nil {
+			dynamicProps = types.NewDynamicProperty(types.Or(o.DynamicProperties().Key, key), types.Or(o.DynamicProperties().Value, tpe))
+		} else {
+			dynamicProps = types.NewDynamicProperty(key, tpe)
+		}
+		return types.NewObject(o.StaticProperties(), dynamicProps), nil
+	}
+
+	child, err := insertIntoObject(types.NewObject(nil, nil), path[1:], tpe, env)
 	if err != nil {
 		return nil, err
 	}
-	staticProps := append(o.StaticProperties(), types.NewStaticProperty(key, child))
-	return types.NewObject(staticProps, o.DynamicProperties()), nil
+
+	var dynamicProps *types.DynamicProperty
+	if dp := o.DynamicProperties(); dp != nil {
+		dynamicProps = types.NewDynamicProperty(types.Or(o.DynamicProperties().Key, key), types.Or(o.DynamicProperties().Value, child))
+	} else {
+		dynamicProps = types.NewDynamicProperty(key, child)
+	}
+	return types.NewObject(o.StaticProperties(), dynamicProps), nil
 }
 
 func (n *typeTreeNode) Leafs() map[*Ref]types.Type {