fix(cli): evaluate kind-less policy specs in policy develop eval (#3783)

Author: migmartri

app/cli/internal/policydevel/eval.go

@@ -69,20 +69,17 @@ type EvalSummaryDebugInfo struct {
 }
 
 func Evaluate(opts *EvalOptions, logger zerolog.Logger) (*EvalSummary, error) {
-	// 1. Create crafting schema
-	policies, err := createPolicies(opts.PolicyPath, opts.Inputs)
-	if err != nil {
-		return nil, err
-	}
-
-	// 2. Craft material with annotations
+	// 1. Craft material with annotations
 	crafted, err := craftMaterial(opts.MaterialPath, opts.MaterialKind, &logger)
 	if err != nil {
 		return nil, err
 	}
 	material := crafted.Material
 	mergeAnnotations(material, opts.Annotations, &logger)
 
+	// 2. Attach the policy to the crafted material
+	policies := createPolicies(opts.PolicyPath, opts.Inputs, material.GetId())
+
 	// 3. Verify material against policy. A crafter that transformed the artifact
 	// before storing it hands back what it stored, and that is what the policy
 	// must be evaluated against — `policy devel eval` has to reproduce what
@@ -131,7 +128,11 @@ func mergeAnnotations(material *v12.Attestation_Material, annotations map[string
 	}
 }
 
-func createPolicies(policyPath string, inputs map[string]string) (*v1.Policies, error) {
+// createPolicies attaches the policy to the material named materialName, the same
+// way a contract attaches it with selector.name. Without the selector, a policy
+// whose spec has no kind would never apply, since it has no type to match the
+// material against. A spec with a kind still only applies to materials of that kind.
+func createPolicies(policyPath string, inputs map[string]string, materialName string) *v1.Policies {
 	// Check if the policy path already has a scheme (chainloop://, http://, https://, file://)
 	ref := policyPath
 	scheme, _ := policies.RefParts(policyPath)
@@ -143,12 +144,12 @@ func createPolicies(policyPath string, inputs map[string]string) (*v1.Policies,
 	return &v1.Policies{
 		Materials: []*v1.PolicyAttachment{
 			{
-				Policy: &v1.PolicyAttachment_Ref{Ref: ref},
-				With:   inputs,
+				Policy:   &v1.PolicyAttachment_Ref{Ref: ref},
+				Selector: &v1.PolicyAttachment_MaterialSelector{Name: materialName},
+				With:     inputs,
 			},
 		},
-		Attestation: nil,
-	}, nil
+	}
 }
 
 func verifyMaterial(pol *v1.Policies, material *v12.Attestation_Material, materialPath string, content []byte, debug bool, allowedHostnames []string, attestationClient controlplanev1.AttestationServiceClient, grpcConn *grpc.ClientConn, projectName, projectVersion string, logger *zerolog.Logger) (*EvalSummary, error) {

app/cli/internal/policydevel/eval_test.go

@@ -233,6 +233,82 @@ func TestEvaluateSimplifiedPolicies(t *testing.T) {
 	})
 }
 
+// `policy develop eval` selects the policy by material name, as a contract does.
+func TestEvaluateMaterialSelection(t *testing.T) {
+	const (
+		anyKindPolicy = "testdata/any-kind-policy.yaml"
+		evidenceKind  = "EVIDENCE"
+	)
+
+	tempDir := t.TempDir()
+	logger := zerolog.New(os.Stderr)
+
+	passing := filepath.Join(tempDir, "pass.json")
+	require.NoError(t, os.WriteFile(passing, []byte(`{}`), 0600))
+	failing := filepath.Join(tempDir, "fail.json")
+	require.NoError(t, os.WriteFile(failing, []byte(`{"fail": true}`), 0600))
+
+	testCases := []struct {
+		name           string
+		policyPath     string
+		kind           string
+		materialPath   string
+		wantViolations []string
+		wantErr        string
+	}{
+		{
+			name:         "kind-less spec with explicit kind",
+			policyPath:   anyKindPolicy,
+			kind:         evidenceKind,
+			materialPath: passing,
+		},
+		{
+			name:           "kind-less spec reports violations",
+			policyPath:     anyKindPolicy,
+			kind:           evidenceKind,
+			materialPath:   failing,
+			wantViolations: []string{"material asked to fail"},
+		},
+		{
+			name:         "kind-less spec with auto-detected kind",
+			policyPath:   anyKindPolicy,
+			materialPath: "testdata/test-sbom.json",
+		},
+		{
+			name:           "spec with a matching kind",
+			policyPath:     "testdata/sbom-min-components-policy.yaml",
+			kind:           "SBOM_CYCLONEDX_JSON",
+			materialPath:   "testdata/test-sbom.json",
+			wantViolations: []string{"SBOM must have at least 2 components"},
+		},
+		{
+			name:         "spec with a different kind is not applied",
+			policyPath:   "testdata/sbom-min-components-policy.yaml",
+			kind:         evidenceKind,
+			materialPath: passing,
+			wantErr:      "no execution branch matched",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := Evaluate(&EvalOptions{
+				PolicyPath:   tc.policyPath,
+				MaterialKind: tc.kind,
+				MaterialPath: tc.materialPath,
+			}, logger)
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+				return
+			}
+
+			require.NoError(t, err)
+			assert.False(t, result.Result.Skipped)
+			assert.ElementsMatch(t, tc.wantViolations, result.Result.Violations)
+		})
+	}
+}
+
 // fixtureGitHubPAT is assembled from fragments so that this file does not itself
 // carry a credential-shaped literal for secret scanners to flag.
 const fixtureGitHubPAT = "ghp_erOZlZv0B1e3amrQ" + "ugdwZ8Ro2W4kDql9WPTf"

app/cli/internal/policydevel/testdata/any-kind-policy.yaml

@@ -0,0 +1,16 @@
+apiVersion: workflowcontract.chainloop.dev/v1
+kind: Policy
+metadata:
+  name: any-kind
+  description: Policy without kind, it applies to any material type
+spec:
+  policies:
+    - embedded: |
+        package any_kind
+
+        import rego.v1
+
+        violations contains msg if {
+            input.fail
+            msg := "material asked to fail"
+        }