diff --git a/.github/workflows/checks-codecov.yaml b/.github/workflows/checks-codecov.yaml index 4a37e6881..8b742fd3b 100644 --- a/.github/workflows/checks-codecov.yaml +++ b/.github/workflows/checks-codecov.yaml @@ -85,32 +85,15 @@ jobs: - name: Test run: make test - - name: Upload unit test coverage report - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - name: Upload test coverage artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0 with: - files: ./coverage-unit.out - disable_search: true - flags: unit - - - name: Upload generative test coverage report - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - files: ./coverage-generative.out - disable_search: true - flags: generative - - - name: Upload integration test coverage report - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - files: ./coverage-integration.out - disable_search: true - flags: integration + name: coverage-test + path: | + ./coverage-unit.out + ./coverage-generative.out + ./coverage-integration.out + retention-days: 1 Acceptance: runs-on: ubuntu-latest @@ -149,12 +132,75 @@ jobs: id: acceptance_test run: E2E_INSTRUMENTATION=true make acceptance - - name: Upload coverage report + - name: Upload acceptance coverage artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0 + with: + name: coverage-acceptance + path: ./coverage-acceptance.out + retention-days: 1 + + Upload: + name: "Upload Coverage Statistics" + runs-on: ubuntu-latest + needs: [Test, Acceptance] + steps: + - name: Harden Runner + uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 + with: + egress-policy: audit + disable-telemetry: true + + # checkout is required for codecov to map the coverage data back to files in the repo + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + + - name: Download test coverage artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: coverage-test + path: ./coverage + + - name: Download acceptance coverage artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: coverage-acceptance + path: ./coverage + + - name: Upload unit test coverage report + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: ./coverage/coverage-unit.out + disable_search: true + flags: unit + + - name: Upload generative test coverage report + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: ./coverage/coverage-generative.out + disable_search: true + flags: generative + + - name: Upload integration test coverage report + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: ./coverage/coverage-integration.out + disable_search: true + flags: integration + + - name: Upload acceptance test coverage report uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - files: ./coverage-acceptance.out + files: ./coverage/coverage-acceptance.out disable_search: true flags: acceptance diff --git a/internal/evaluator/conftest_evaluator_benchmark_test.go b/internal/evaluator/conftest_evaluator_benchmark_test.go deleted file mode 100644 index 2a39b1e6a..000000000 --- a/internal/evaluator/conftest_evaluator_benchmark_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -// This file contains benchmark tests for the Conftest Evaluator to measure -// performance characteristics. It includes benchmarks for: -// - Basic evaluation performance (BenchmarkConftestEvaluatorEvaluate) -// - Large input evaluation performance (BenchmarkConftestEvaluatorWithLargeInput) -// These benchmarks help identify performance bottlenecks and regressions -// in the evaluator's performance. - -//go:build unit - -package evaluator - -import ( - "context" - "testing" - "time" - - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/stretchr/testify/require" - - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" -) - -func BenchmarkConftestEvaluatorEvaluate(b *testing.B) { - ctx := context.Background() - - // Create policy source - policySource := &source.PolicyUrl{ - Url: "file://testdata/policies", - Kind: source.PolicyKind, - } - - // Create config provider - configProvider := &mockConfigProvider{} - configProvider.On("EffectiveTime").Return(time.Now()) - configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ - Sources: []ecc.Source{ - { - Policy: []string{"file://testdata/policies"}, - }, - }, - }) - - // Create evaluator - evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}) - require.NoError(b, err) - defer evaluator.Destroy() - - // Create test target - target := EvaluationTarget{ - Inputs: []string{"testdata/input.json"}, - Target: "benchmark", - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := evaluator.Evaluate(ctx, target) - require.NoError(b, err) - } -} - -func BenchmarkConftestEvaluatorWithLargeInput(b *testing.B) { - ctx := context.Background() - - // Create policy source - policySource := &source.PolicyUrl{ - Url: "file://testdata/policies", - Kind: source.PolicyKind, - } - - // Create config provider - configProvider := &mockConfigProvider{} - configProvider.On("EffectiveTime").Return(time.Now()) - configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ - Sources: []ecc.Source{ - { - Policy: []string{"file://testdata/policies"}, - }, - }, - }) - - // Create evaluator - evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}) - require.NoError(b, err) - defer evaluator.Destroy() - - // Create test target with multiple inputs - target := EvaluationTarget{ - Inputs: []string{"testdata/large-input.json"}, - Target: "benchmark-large", - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := evaluator.Evaluate(ctx, target) - require.NoError(b, err) - } -} diff --git a/internal/evaluator/conftest_evaluator_integration_basic_test.go b/internal/evaluator/conftest_evaluator_integration_basic_test.go deleted file mode 100644 index 21f2f539c..000000000 --- a/internal/evaluator/conftest_evaluator_integration_basic_test.go +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -// This file contains integration tests for the Conftest Evaluator that test -// the complete evaluation flow with real policy sources and file systems. -// It includes tests for: -// - Basic integration functionality (TestConftestEvaluatorIntegrationBasic) -// - Integration with test data and file systems (TestConftestEvaluatorIntegrationWithTestData) -// These tests verify that the evaluator works correctly in real-world scenarios -// with actual policy files and data sources. - -//go:build integration - -package evaluator - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "testing" - "time" - - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" -) - -func TestConftestEvaluatorIntegrationBasic(t *testing.T) { - ctx := context.Background() - - // Create a simple policy source - policySource := &source.PolicyUrl{ - Url: "file://testdata/policies", - Kind: source.PolicyKind, - } - - // Create config provider - configProvider := &mockConfigProvider{} - configProvider.On("EffectiveTime").Return(time.Now()) - configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ - Sources: []ecc.Source{ - { - Policy: []string{"file://testdata/policies"}, - }, - }, - }) - - // Create evaluator - evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}) - require.NoError(t, err) - defer evaluator.Destroy() - - // Test that evaluator is created successfully - assert.NotNil(t, evaluator) - assert.NotEmpty(t, evaluator.CapabilitiesPath()) -} - -func TestConftestEvaluatorIntegrationWithTestData(t *testing.T) { - ctx := context.Background() - - // Create a temporary directory for the test - tmpDir := t.TempDir() - policyDir := filepath.Join(tmpDir, "policy") - err := os.MkdirAll(policyDir, 0o755) - require.NoError(t, err) - - // Create a simple policy file for testing - policyContent := `package main - -import rego.v1 - -deny contains result if { - result := { - "code": "main.test", - "msg": "Test value found", - } -}` - err = os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600) - require.NoError(t, err) - - // Create policy source - policySource := &source.PolicyUrl{ - Url: "file://" + policyDir, - Kind: source.PolicyKind, - } - - // Create config provider - configProvider := &mockConfigProvider{} - configProvider.On("EffectiveTime").Return(time.Now()) - configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ - Sources: []ecc.Source{ - { - Policy: []string{"file://" + policyDir}, - }, - }, - }) - - // Create evaluator - evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}) - require.NoError(t, err) - defer evaluator.Destroy() - - // Test evaluation with simple input - target := EvaluationTarget{ - Inputs: []string{filepath.Join(tmpDir, "input.json")}, - Target: "test", - } - - // Create a simple input file - inputData := map[string]interface{}{ - "test": "value", - } - inputBytes, err := json.Marshal(inputData) - require.NoError(t, err) - err = os.WriteFile(target.Inputs[0], inputBytes, 0o600) - require.NoError(t, err) - - // Run evaluation - result, err := evaluator.Evaluate(ctx, target) - require.NoError(t, err) - assert.NotNil(t, result) -} - -func TestConftestEvaluatorIntegrationWithComponentNames(t *testing.T) { - ctx := context.Background() - - // Create a temporary directory for the test - tmpDir := t.TempDir() - policyDir := filepath.Join(tmpDir, "policy") - err := os.MkdirAll(policyDir, 0o755) - require.NoError(t, err) - - // Create policies that will be filtered by ComponentNames - policyContent := `package test - -import rego.v1 - -# METADATA -# title: Check A -# custom: -# short_name: check_a -deny contains result if { - result := { - "code": "test.check_a", - "msg": "Check A always fails" - } -} - -# METADATA -# title: Check B -# custom: -# short_name: check_b -deny contains result if { - result := { - "code": "test.check_b", - "msg": "Check B always fails" - } -} -` - err = os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600) - require.NoError(t, err) - - // Create policy source - policySource := &source.PolicyUrl{ - Url: "file://" + policyDir, - Kind: source.PolicyKind, - } - - // Create config provider with ComponentNames filter - configProvider := &mockConfigProvider{} - configProvider.On("EffectiveTime").Return(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC)) - configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ - Sources: []ecc.Source{ - { - Policy: []string{"file://" + policyDir}, - }, - }, - }) - - // Create evaluator with VolatileConfig that excludes check_a for comp1 - evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{ - VolatileConfig: &ecc.VolatileSourceConfig{ - Exclude: []ecc.VolatileCriteria{ - { - Value: "test.check_a", - ComponentNames: []ecc.ComponentName{"comp1"}, - EffectiveOn: "2024-01-01T00:00:00Z", - EffectiveUntil: "2025-01-01T00:00:00Z", - }, - }, - }, - }) - require.NoError(t, err) - defer evaluator.Destroy() - - // Debug: Check exclude criteria - conftestEval := evaluator.(conftestEvaluator) - t.Logf("Exclude componentItems: %+v", conftestEval.exclude.componentItems) - t.Logf("Exclude defaultItems: %+v", conftestEval.exclude.defaultItems) - t.Logf("Exclude digestItems: %+v", conftestEval.exclude.digestItems) - - // Create test input - inputData := map[string]interface{}{ - "test": "value", - } - inputBytes, err := json.Marshal(inputData) - require.NoError(t, err) - inputPath := filepath.Join(tmpDir, "input.json") - err = os.WriteFile(inputPath, inputBytes, 0o600) - require.NoError(t, err) - - // Test comp1 - check_a should be excluded - target1 := EvaluationTarget{ - Inputs: []string{inputPath}, - Target: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - ComponentName: "comp1", - } - - result1, err := evaluator.Evaluate(ctx, target1) - require.NoError(t, err) - require.NotNil(t, result1) - - // Debug: Print all failures - t.Logf("comp1 results: %d outcomes", len(result1)) - for i, outcome := range result1 { - t.Logf(" Outcome %d: %d failures, %d successes", i, len(outcome.Failures), len(outcome.Successes)) - for _, failure := range outcome.Failures { - t.Logf(" Failure: %s", failure.Metadata["code"]) - } - for _, success := range outcome.Successes { - t.Logf(" Success: %s", success.Metadata["code"]) - } - } - - // Verify check_a is excluded, check_b is not - hasCheckA := false - hasCheckB := false - for _, outcome := range result1 { - for _, failure := range outcome.Failures { - if codeStr, ok := failure.Metadata["code"].(string); ok { - if codeStr == "test.check_a" { - hasCheckA = true - } - if codeStr == "test.check_b" { - hasCheckB = true - } - } - } - } - assert.False(t, hasCheckA, "Expected check_a to be excluded for comp1") - assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp1") - - // Test comp2 - check_a should NOT be excluded - target2 := EvaluationTarget{ - Inputs: []string{inputPath}, - Target: "quay.io/repo/img@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", - ComponentName: "comp2", - } - - result2, err := evaluator.Evaluate(ctx, target2) - require.NoError(t, err) - require.NotNil(t, result2) - - // Verify both checks are evaluated for comp2 - hasCheckA2 := false - hasCheckB2 := false - for _, outcome := range result2 { - for _, failure := range outcome.Failures { - if codeStr, ok := failure.Metadata["code"].(string); ok { - if codeStr == "test.check_a" { - hasCheckA2 = true - } - if codeStr == "test.check_b" { - hasCheckB2 = true - } - } - } - } - assert.True(t, hasCheckA2, "Expected check_a to be evaluated for comp2") - assert.True(t, hasCheckB2, "Expected check_b to be evaluated for comp2") - - // Test same image with different components - monorepo scenario - sameImage := "quay.io/monorepo@sha256:fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321" - - target3 := EvaluationTarget{ - Inputs: []string{inputPath}, - Target: sameImage, - ComponentName: "comp1", - } - - result3, err := evaluator.Evaluate(ctx, target3) - require.NoError(t, err) - - hasCheckA3 := false - for _, outcome := range result3 { - for _, failure := range outcome.Failures { - if codeStr, ok := failure.Metadata["code"].(string); ok && codeStr == "test.check_a" { - hasCheckA3 = true - } - } - } - assert.False(t, hasCheckA3, "Expected check_a excluded for comp1 even with different image") - - target4 := EvaluationTarget{ - Inputs: []string{inputPath}, - Target: sameImage, - ComponentName: "comp2", - } - - result4, err := evaluator.Evaluate(ctx, target4) - require.NoError(t, err) - - hasCheckA4 := false - for _, outcome := range result4 { - for _, failure := range outcome.Failures { - if codeStr, ok := failure.Metadata["code"].(string); ok && codeStr == "test.check_a" { - hasCheckA4 = true - } - } - } - assert.True(t, hasCheckA4, "Expected check_a evaluated for comp2 with same image") -} diff --git a/internal/evaluator/conftest_evaluator_unit_core_test.go b/internal/evaluator/conftest_evaluator_unit_core_test.go deleted file mode 100644 index a690b4ee4..000000000 --- a/internal/evaluator/conftest_evaluator_unit_core_test.go +++ /dev/null @@ -1,736 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -// This file contains unit tests for the core evaluation logic and basic functionalities -// of the Conftest Evaluator. It includes tests for: -// - Basic evaluation functionality (TestConftestEvaluatorEvaluate) -// - Severity handling (TestConftestEvaluatorEvaluateSeverity) -// - Capabilities configuration (TestConftestEvaluatorCapabilities) -// - Success/warning/failure scenarios (TestConftestEvaluatorEvaluateNoSuccessWarningsOrFailures) -// - Unconforming rule behavior (TestUnconformingRule) -// These are fast, deterministic unit tests that focus on the core evaluation logic. - -//go:build unit - -package evaluator - -import ( - "context" - "encoding/json" - "io/fs" - "os" - "path" - "path/filepath" - "sort" - "strings" - "testing" - "time" - - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/gkampitakis/go-snaps/snaps" - "github.com/open-policy-agent/opa/v1/ast" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "k8s.io/kube-openapi/pkg/util/sets" - - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" - "github.com/conforma/cli/internal/utils" -) - -func TestConftestEvaluatorEvaluateSeverity(t *testing.T) { - results := []Outcome{ - { - Failures: []Result{ - { - Message: "missing effective date", - Metadata: map[string]any{}, - }, - { - Message: "already effective", - Metadata: map[string]any{ - "effective_on": "2021-01-01T00:00:00Z", - }, - }, - { - Message: "invalid effective date", - Metadata: map[string]any{ - "effective_on": "hangout-not-a-date", - }, - }, - { - Message: "unexpected effective date type", - Metadata: map[string]any{ - "effective_on": true, - }, - }, - { - Message: "not yet effective", - Metadata: map[string]any{ - "effective_on": "3021-01-01T00:00:00Z", - }, - }, - { - Message: "failure to warning", - Metadata: map[string]any{ - "severity": "warning", - }, - }, - { - Message: "failure to failure", - Metadata: map[string]any{ - "severity": "failure", - }, - }, - { - Message: "unexpected severity value on failure", - Metadata: map[string]any{ - "severity": "spam", - }, - }, - { - Message: "unexpected severity type on failure", - Metadata: map[string]any{ - "severity": 42, - }, - }, - }, - Warnings: []Result{ - { - Message: "existing warning", - Metadata: map[string]any{ - "effective_on": "2021-01-01T00:00:00Z", - }, - }, - { - Message: "warning to failure", - Metadata: map[string]any{ - "severity": "failure", - }, - }, - { - Message: "warning to warning", - Metadata: map[string]any{ - "severity": "warning", - }, - }, - { - Message: "unexpected severity value on warning", - Metadata: map[string]any{ - "severity": "spam", - }, - }, - { - Message: "unexpected severity type on warning", - Metadata: map[string]any{ - "severity": 42, - }, - }, - }, - }, - } - - expectedResults := []Outcome{ - { - Failures: []Result{ - { - Message: "warning to failure", - Metadata: map[string]any{ - "severity": "failure", - }, - }, - { - Message: "missing effective date", - Metadata: map[string]any{}, - }, - { - Message: "already effective", - Metadata: map[string]any{ - "effective_on": "2021-01-01T00:00:00Z", - }, - }, - { - Message: "invalid effective date", - Metadata: map[string]any{ - "effective_on": "hangout-not-a-date", - }, - }, - { - Message: "unexpected effective date type", - Metadata: map[string]any{ - "effective_on": true, - }, - }, - { - Message: "failure to failure", - Metadata: map[string]any{ - "severity": "failure", - }, - }, - { - Message: "unexpected severity value on failure", - Metadata: map[string]any{ - "severity": "spam", - }, - }, - { - Message: "unexpected severity type on failure", - Metadata: map[string]any{ - "severity": 42, - }, - }, - }, - Warnings: []Result{ - { - Message: "existing warning", - Metadata: map[string]any{ - "effective_on": "2021-01-01T00:00:00Z", - }, - }, - { - Message: "warning to warning", - Metadata: map[string]any{ - "severity": "warning", - }, - }, - { - Message: "unexpected severity value on warning", - Metadata: map[string]any{ - "severity": "spam", - }, - }, - { - Message: "unexpected severity type on warning", - Metadata: map[string]any{ - "severity": 42, - }, - }, - - { - Message: "not yet effective", - Metadata: map[string]any{ - "effective_on": "3021-01-01T00:00:00Z", - }, - }, - { - Message: "failure to warning", - Metadata: map[string]any{ - "severity": "warning", - }, - }, - }, - Skipped: []Result{}, - Exceptions: []Result{}, - }, - } - - r := mockTestRunner{} - dl := mockDownloader{} - inputs := EvaluationTarget{Inputs: []string{"inputs"}} - expectedData := Data(map[string]any{ - "a": 1, - }) - - ctx := setupTestContext(&r, &dl) - r.On("Run", ctx, inputs.Inputs).Return(results, expectedData, nil) - - pol, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - src := testPolicySource{} - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - src, - }, pol, ecc.Source{}, []string{}) - - assert.NoError(t, err) - actualResults, err := evaluator.Evaluate(ctx, inputs) - assert.NoError(t, err) - assert.Equal(t, expectedResults, actualResults) -} - -func TestConftestEvaluatorCapabilities(t *testing.T) { - ctx := setupTestContext(nil, nil) - fs := utils.FS(ctx) - - p, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - testPolicySource{}, - }, p, ecc.Source{}, []string{}) - assert.NoError(t, err) - - blob, err := afero.ReadFile(fs, evaluator.CapabilitiesPath()) - assert.NoError(t, err) - var capabilities ast.Capabilities - err = json.Unmarshal(blob, &capabilities) - assert.NoError(t, err) - - defaultBuiltins := sets.NewString() - for _, b := range ast.CapabilitiesForThisVersion().Builtins { - defaultBuiltins.Insert(b.Name) - } - - gotBuiltins := sets.NewString() - for _, b := range capabilities.Builtins { - gotBuiltins.Insert(b.Name) - } - - expectedRemoved := sets.NewString("opa.runtime", "http.send", "net.lookup_ip_addr") - - assert.Equal(t, defaultBuiltins.Difference(gotBuiltins), expectedRemoved) - assert.Equal(t, []string{""}, capabilities.AllowNet) -} - -func TestConftestEvaluatorEvaluateNoSuccessWarningsOrFailures(t *testing.T) { - tests := []struct { - name string - results []Outcome - sourceConfig *ecc.SourceConfig - }{ - { - name: "no results", - results: []Outcome{ - { - Failures: []Result{}, - Warnings: []Result{}, - Successes: []Result{}, - }, - }, - }, - { - name: "no included results", - results: []Outcome{ - { - Failures: []Result{{Metadata: map[string]any{"code": "breakfast.spam"}}}, - Warnings: []Result{{Metadata: map[string]any{"code": "lunch.spam"}}}, - Successes: []Result{{Metadata: map[string]any{"code": "dinner.spam"}}}, - }, - }, - sourceConfig: &ecc.SourceConfig{ - Include: []string{"brunch.spam"}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := mockTestRunner{} - dl := mockDownloader{} - inputs := EvaluationTarget{Inputs: []string{"inputs"}} - ctx := setupTestContext(&r, &dl) - - r.On("Run", ctx, inputs.Inputs).Return(tt.results, Data(nil), nil) - - p, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - testPolicySource{}, - }, p, ecc.Source{Config: tt.sourceConfig}, []string{}) - - assert.NoError(t, err) - actualResults, err := evaluator.Evaluate(ctx, inputs) - assert.ErrorContains(t, err, "no successes, warnings, or failures, check input") - assert.Nil(t, actualResults) - }) - } -} - -func TestConftestEvaluatorEvaluate(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(path.Join(dir, "inputs"), 0755)) - require.NoError(t, os.WriteFile(path.Join(dir, "inputs", "data.json"), []byte("{}"), 0600)) - - rego, err := fs.Sub(policies, "__testdir__/simple") - require.NoError(t, err) - - rules, err := rulesArchiveFromFS(t, rego) - require.NoError(t, err) - - ctx := withCapabilities(context.Background(), testCapabilities) - - eTime, err := time.Parse(policy.DateFormat, "2014-05-31") - require.NoError(t, err) - config := &mockConfigProvider{} - config.On("EffectiveTime").Return(eTime) - config.On("SigstoreOpts").Return(policy.SigstoreOpts{ - CertificateIdentity: "cert-identity", - CertificateIdentityRegExp: "cert-identity-regexp", - CertificateOIDCIssuer: "cert-oidc-issuer", - CertificateOIDCIssuerRegExp: "cert-oidc-issuer-regexp", - IgnoreRekor: true, - RekorURL: "https://rekor.local/", - PublicKey: utils.TestPublicKey, - }, nil) - config.On("Spec").Return(ecc.EnterpriseContractPolicySpec{}) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{ - Url: rules, - Kind: source.PolicyKind, - }, - }, config, ecc.Source{}, []string{}) - require.NoError(t, err) - - results, err := evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{path.Join(dir, "inputs")}}) - require.NoError(t, err) - - // sort the slice by code for test stability - sort.Slice(results, func(l, r int) bool { - return strings.Compare(results[l].Namespace, results[r].Namespace) < 0 - }) - - for i := range results { - // let's not fail the snapshot on different locations of $TMPDIR - results[i].FileName = filepath.ToSlash(strings.Replace(results[i].FileName, dir, "$TMPDIR", 1)) - // sort the slice by code for test stability - sort.Slice(results[i].Successes, func(l, r int) bool { - return strings.Compare(results[i].Successes[l].Metadata[metadataCode].(string), results[i].Successes[r].Metadata[metadataCode].(string)) < 0 - }) - } - - snaps.MatchSnapshot(t, results) -} - -func TestUnconformingRule(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(path.Join(dir, "inputs"), 0755)) - require.NoError(t, os.WriteFile(path.Join(dir, "inputs", "data.json"), []byte("{}"), 0600)) - - rego, err := fs.Sub(policies, "__testdir__/unconforming") - require.NoError(t, err) - - rules, err := rulesArchiveFromFS(t, rego) - require.NoError(t, err) - - ctx := context.Background() - - p, err := policy.NewInertPolicy(ctx, "") - require.NoError(t, err) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{ - Url: rules, - Kind: source.PolicyKind, - }, - }, p, ecc.Source{}, []string{}) - require.NoError(t, err) - - _, err = evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{path.Join(dir, "inputs")}}) - require.Error(t, err) - assert.EqualError(t, err, `the rule "deny = true if { true }" returns an unsupported value, at no_msg.rego:5`) -} - -// --- Reintroduced tests from the original monolith --- -// These restore coverage for mixed annotated vs non-annotated rule behavior -// and filtering across mixed packages. - -// TestAnnotatedAndNonAnnotatedRules tests the separation of annotated and non-annotated rules -func TestAnnotatedAndNonAnnotatedRules(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(path.Join(dir, "inputs"), 0755)) - require.NoError(t, os.WriteFile(path.Join(dir, "inputs", "data.json"), []byte("{}"), 0600)) - - // Create a test directory with both annotated and non-annotated rules - testDir := path.Join(dir, "test_policies") - require.NoError(t, os.MkdirAll(testDir, 0755)) - - // Annotated rule - annotatedRule := `package annotated - -import rego.v1 - -# METADATA -# title: Annotated Rule -# description: This rule has annotations -# custom: -# short_name: annotated_rule -deny contains result if { - result := { - "code": "annotated.rule", - "msg": "Annotated rule failure", - } -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "annotated.rego"), []byte(annotatedRule), 0600)) - - // Non-annotated rule - nonAnnotatedRule := `package nonannotated - -import rego.v1 - -deny contains result if { - result := { - "code": "nonannotated.rule", - "msg": "Non-annotated rule failure", - } -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "nonannotated.rego"), []byte(nonAnnotatedRule), 0600)) - - // Non-annotated rule without code in result - nonAnnotatedRuleNoCode := `package noresultcode - -import rego.v1 - -deny contains result if { - result := "No code in result" -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "noresultcode.rego"), []byte(nonAnnotatedRuleNoCode), 0600)) - - // Create rules archive - archivePath := path.Join(dir, "rules.tar.gz") - createTestArchive(t, testDir, archivePath) - - ctx := withCapabilities(context.Background(), testCapabilities) - - eTime, err := time.Parse(policy.DateFormat, "2014-05-31") - require.NoError(t, err) - config := &mockConfigProvider{} - config.On("EffectiveTime").Return(eTime) - config.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - config.On("Spec").Return(ecc.EnterpriseContractPolicySpec{}) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{Url: archivePath, Kind: source.PolicyKind}, - }, config, ecc.Source{}, []string{}) - require.NoError(t, err) - - results, err := evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{path.Join(dir, "inputs")}}) - require.NoError(t, err) - - // Verify annotated successes tracked, non-annotated successes not tracked - foundAnnotatedSuccess := false - foundNonAnnotatedSuccess := false - for _, result := range results { - for _, success := range result.Successes { - if code, ok := success.Metadata[metadataCode].(string); ok { - if code == "annotated.annotated_rule" { - foundAnnotatedSuccess = true - assert.Contains(t, success.Metadata, metadataTitle) - assert.Contains(t, success.Metadata, metadataDescription) - } - if code == "nonannotated.rule" { - foundNonAnnotatedSuccess = true - } - } - } - } - assert.True(t, foundAnnotatedSuccess, "Annotated rule should be tracked for success computation") - assert.False(t, foundNonAnnotatedSuccess, "Non-annotated rules should not be tracked for success computation") -} - -// TestRuleCollectionWithMixedRules tests rule collection logic with mixed annotated and non-annotated rules -func TestRuleCollectionWithMixedRules(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(path.Join(dir, "inputs"), 0755)) - require.NoError(t, os.WriteFile(path.Join(dir, "inputs", "data.json"), []byte("{}"), 0600)) - - // Create test directory with mixed rules - testDir := path.Join(dir, "mixed_policies") - require.NoError(t, os.MkdirAll(testDir, 0755)) - - // Annotated failing - annotatedFailingRule := `package mixed - -import rego.v1 - -# METADATA -# title: Annotated Failing Rule -# description: This annotated rule will fail -# custom: -# short_name: annotated_failing -deny contains result if { - result := { - "code": "mixed.annotated_failing", - "msg": "Annotated rule failure", - } -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "annotated_failing.rego"), []byte(annotatedFailingRule), 0600)) - - // Annotated passing - annotatedPassingRule := `package mixed - -import rego.v1 - -# METADATA -# title: Annotated Passing Rule -# description: This annotated rule will pass -# custom: -# short_name: annotated_passing -deny contains result if { - false - result := "This should not be reached" -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "annotated_passing.rego"), []byte(annotatedPassingRule), 0600)) - - // Non-annotated failing - nonAnnotatedFailingRule := `package mixed - -import rego.v1 - -deny contains result if { - result := { - "code": "mixed.nonannotated_failing", - "msg": "Non-annotated rule failure", - } -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "nonannotated_failing.rego"), []byte(nonAnnotatedFailingRule), 0600)) - - // Non-annotated passing - nonAnnotatedPassingRule := `package mixed - -import rego.v1 - -deny contains result if { - false - result := "This should not be reached" -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "nonannotated_passing.rego"), []byte(nonAnnotatedPassingRule), 0600)) - - // Create rules archive - archivePath := path.Join(dir, "rules.tar.gz") - createTestArchive(t, testDir, archivePath) - - ctx := withCapabilities(context.Background(), testCapabilities) - - eTime, err := time.Parse(policy.DateFormat, "2014-05-31") - require.NoError(t, err) - config := &mockConfigProvider{} - config.On("EffectiveTime").Return(eTime) - config.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - config.On("Spec").Return(ecc.EnterpriseContractPolicySpec{}) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{Url: archivePath, Kind: source.PolicyKind}, - }, config, ecc.Source{}, []string{}) - require.NoError(t, err) - - results, err := evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{path.Join(dir, "inputs")}}) - require.NoError(t, err) - - var annotatedFailures, annotatedSuccesses, nonAnnotatedFailures, nonAnnotatedSuccesses int - for _, result := range results { - for _, failure := range result.Failures { - if code, ok := failure.Metadata[metadataCode].(string); ok { - switch code { - case "mixed.annotated_failing": - annotatedFailures++ - case "mixed.nonannotated_failing": - nonAnnotatedFailures++ - } - } - } - for _, success := range result.Successes { - if code, ok := success.Metadata[metadataCode].(string); ok { - switch code { - case "mixed.annotated_passing": - annotatedSuccesses++ - case "mixed.nonannotated_passing": - nonAnnotatedSuccesses++ - } - } - } - } - assert.Equal(t, 1, annotatedFailures, "Should have one annotated failure") - assert.Equal(t, 1, annotatedSuccesses, "Should have one annotated success") - assert.Equal(t, 1, nonAnnotatedFailures, "Should have one non-annotated failure") - assert.Equal(t, 0, nonAnnotatedSuccesses, "Should not track non-annotated rules for success computation") -} - -// TestFilteringWithMixedRules verifies that both annotated and non-annotated rules participate in filtering -func TestFilteringWithMixedRules(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(path.Join(dir, "inputs"), 0755)) - require.NoError(t, os.WriteFile(path.Join(dir, "inputs", "data.json"), []byte("{}"), 0600)) - - // Create test directory with rules in different packages - testDir := path.Join(dir, "filtering_policies") - require.NoError(t, os.MkdirAll(testDir, 0755)) - - // Annotated rule in package 'a' - annotatedRuleA := `package a - -import rego.v1 - -# METADATA -# title: Annotated Rule A -# description: This annotated rule is in package a -# custom: -# short_name: annotated_a -deny contains result if { - result := { - "code": "a.annotated", - "msg": "Annotated rule in package a", - } -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "a_annotated.rego"), []byte(annotatedRuleA), 0600)) - - // Non-annotated rule in package 'b' - nonAnnotatedRuleB := `package b - -import rego.v1 - -deny contains result if { - result := { - "code": "b.nonannotated", - "msg": "Non-annotated rule in package b", - } -}` - require.NoError(t, os.WriteFile(path.Join(testDir, "b_nonannotated.rego"), []byte(nonAnnotatedRuleB), 0600)) - - // Create rules archive - archivePath := path.Join(dir, "rules.tar.gz") - createTestArchive(t, testDir, archivePath) - - ctx := withCapabilities(context.Background(), testCapabilities) - - eTime, err := time.Parse(policy.DateFormat, "2014-05-31") - require.NoError(t, err) - config := &mockConfigProvider{} - config.On("EffectiveTime").Return(eTime) - config.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - config.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ - Configuration: &ecc.EnterpriseContractPolicyConfiguration{ - Include: []string{"a.*", "b.*"}, // Include both packages - }, - }) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{Url: archivePath, Kind: source.PolicyKind}, - }, config, ecc.Source{}, []string{}) - require.NoError(t, err) - - results, err := evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{path.Join(dir, "inputs")}}) - require.NoError(t, err) - - foundAnnotatedFailure := false - foundNonAnnotatedFailure := false - for _, result := range results { - for _, failure := range result.Failures { - if code, ok := failure.Metadata[metadataCode].(string); ok { - if code == "a.annotated" { - foundAnnotatedFailure = true - } - if code == "b.nonannotated" { - foundNonAnnotatedFailure = true - } - } - } - } - assert.True(t, foundAnnotatedFailure, "Annotated rule should be included in filtering") - assert.True(t, foundNonAnnotatedFailure, "Non-annotated rule should be included in filtering") -} diff --git a/internal/evaluator/conftest_evaluator_unit_data_test.go b/internal/evaluator/conftest_evaluator_unit_data_test.go deleted file mode 100644 index b7abed70c..000000000 --- a/internal/evaluator/conftest_evaluator_unit_data_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -// This file contains unit tests for data directory preparation and processing. -// It includes tests for: -// - Data directory preparation (TestPrepareDataDirs) -// These tests focus on how the evaluator prepares and processes data directories -// that are passed to OPA policies during evaluation. - -//go:build unit - -package evaluator - -import ( - "context" - "path/filepath" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/conforma/cli/internal/utils" -) - -func TestPrepareDataDirs(t *testing.T) { - tests := []struct { - name string - filePaths []string // ordered list of file paths to create - expectedDirs []string // expected directories in same order - }{ - { - name: "files in subdirectories", - filePaths: []string{ - "foo/data.json", - "another/path/info.yaml", - "third/deep/path/config.yml", - "some/path/no-data.txt", - }, - expectedDirs: []string{ - "foo", - "another/path", - "third/deep/path", - }, - }, - { - name: "realistic konflux example", - filePaths: []string{ - "data/a67f0d7cc/rule_data.yml", - "data/a67f0d7cc/required_tasks.yml", - "data/a67f0d7cc/known_rpm_repositories.yml", - "data/e8a615778/data/data/trusted_tekton_tasks.yml", - "data/config/config.json", - }, - expectedDirs: []string{ - "data/a67f0d7cc", - "data/e8a615778/data/data", - "data/config", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create a temporary filesystem - fs := afero.NewMemMapFs() - ctx := utils.WithFS(context.Background(), fs) - - // Create the base data directory - dataDir := "/test/data" - require.NoError(t, fs.MkdirAll(dataDir, 0755)) - - // Create the test files with minimal content - for _, filePath := range tt.filePaths { - fullPath := filepath.Join(dataDir, filePath) - require.NoError(t, fs.MkdirAll(filepath.Dir(fullPath), 0755)) - require.NoError(t, afero.WriteFile(fs, fullPath, []byte("test"), 0644)) - } - - // Create evaluator instance - evaluator := conftestEvaluator{ - dataDir: dataDir, - fs: fs, - } - - // Call prepareDataDirs with the base data directory as data source - // In real usage, dataSourceDirs would be the directories returned by GetPolicy - actualDirs, err := evaluator.prepareDataDirs(ctx, []string{dataDir}) - require.NoError(t, err) - - // Convert expected relative paths to absolute paths - expectedAbsolute := make([]string, len(tt.expectedDirs)) - for i, dir := range tt.expectedDirs { - if dir == "." { - expectedAbsolute[i] = dataDir - } else { - expectedAbsolute[i] = filepath.Join(dataDir, dir) - } - } - - assert.ElementsMatch(t, expectedAbsolute, actualDirs) - }) - } -} diff --git a/internal/evaluator/conftest_evaluator_unit_filtering_test.go b/internal/evaluator/conftest_evaluator_unit_filtering_test.go deleted file mode 100644 index b8ef21ac7..000000000 --- a/internal/evaluator/conftest_evaluator_unit_filtering_test.go +++ /dev/null @@ -1,549 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -// This file contains unit tests related to include/exclude logic, matcher generation, -// and result filtering functionality. It includes tests for: -// - Include/exclude criteria handling (TestConftestEvaluatorIncludeExclude) -// - Matcher generation for rule matching (TestMakeMatchers) -// - Name scoring for rule prioritization (TestNameScoring) -// - Result trimming functionality (TestCheckResultsTrim) -// These tests focus on the filtering and rule matching aspects of the evaluator. - -//go:build unit - -package evaluator - -import ( - "testing" - - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/stretchr/testify/assert" - - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" -) - -func TestConftestEvaluatorIncludeExclude(t *testing.T) { - tests := []struct { - name string - results []Outcome - config *ecc.EnterpriseContractPolicyConfiguration - want []Outcome - }{ - { - name: "exclude by package name", - results: []Outcome{ - { - Failures: []Result{ - {Metadata: map[string]any{"code": "breakfast.spam"}}, - {Metadata: map[string]any{"code": "lunch.spam"}}, - }, - Warnings: []Result{ - {Metadata: map[string]any{"code": "breakfast.ham"}}, - {Metadata: map[string]any{"code": "lunch.ham"}}, - }, - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{Exclude: []string{"breakfast"}}, - want: []Outcome{ - { - Failures: []Result{ - {Metadata: map[string]any{"code": "lunch.spam"}}, - }, - Warnings: []Result{ - {Metadata: map[string]any{"code": "lunch.ham"}}, - }, - Skipped: []Result{}, - Exceptions: []Result{}, - }, - }, - }, - { - name: "include by package", - results: []Outcome{ - { - Failures: []Result{ - {Metadata: map[string]any{"code": "breakfast.spam"}}, - {Metadata: map[string]any{"code": "lunch.spam"}}, - }, - Warnings: []Result{ - {Metadata: map[string]any{"code": "breakfast.ham"}}, - {Metadata: map[string]any{"code": "lunch.ham"}}, - }, - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{Include: []string{"breakfast"}}, - want: []Outcome{ - { - Failures: []Result{ - {Metadata: map[string]any{"code": "breakfast.spam"}}, - }, - Warnings: []Result{ - {Metadata: map[string]any{"code": "breakfast.ham"}}, - }, - Skipped: []Result{}, - Exceptions: []Result{}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := mockTestRunner{} - dl := mockDownloader{} - inputs := EvaluationTarget{Inputs: []string{"inputs"}} - ctx := setupTestContext(&r, &dl) - r.On("Run", ctx, inputs.Inputs).Return(tt.results, Data(nil), nil) - - p, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - p = p.WithSpec(ecc.EnterpriseContractPolicySpec{ - Configuration: tt.config, - }) - - sourceConfig := ecc.Source{ - Config: &ecc.SourceConfig{ - Include: tt.config.Include, - Exclude: tt.config.Exclude, - }, - } - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - testPolicySource{}, - }, p, sourceConfig, []string{}) - - assert.NoError(t, err) - got, err := evaluator.Evaluate(ctx, inputs) - assert.NoError(t, err) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestMakeMatchers(t *testing.T) { - cases := []struct { - name string - code string - term any - want []string - }{ - { - name: "valid", code: "breakfast.spam", term: "eggs", - want: []string{ - "breakfast", "breakfast.*", "breakfast.spam", "breakfast:eggs", "breakfast.*:eggs", - "breakfast.spam:eggs", "*", - }, - }, - { - name: "valid without term", code: "breakfast.spam", - want: []string{"breakfast", "breakfast.*", "breakfast.spam", "*"}, - }, - {name: "incomplete code", code: "spam", want: []string{"*"}}, - {name: "empty code", code: "", want: []string{"*"}}, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - result := Result{Metadata: map[string]any{}} - if tt.code != "" { - result.Metadata["code"] = tt.code - } - if tt.term != "" { - result.Metadata["term"] = tt.term - } - assert.Equal(t, tt.want, LegacyMakeMatchers(result)) - }) - } -} - -func TestNameScoring(t *testing.T) { - cases := []struct { - name string - score int - }{ - { - name: "*", - score: 1, - }, - { - name: "pkg", - score: 10, - }, - { - name: "pkg.rule", - score: 110, - }, - { - name: "pkg:term", - score: 110, - }, - { - name: "pkg.rule:term", - score: 210, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - assert.Equal(t, c.score, LegacyScore(c.name)) - }) - } -} - -func TestCheckResultsTrim(t *testing.T) { - cases := []struct { - name string - given []Outcome - expected []Outcome - }{ - { - name: "simple dependency", - given: []Outcome{ - { - Failures: []Result{ - { - Message: "failure 1", - Metadata: map[string]interface{}{ - metadataCode: "a.failure1", - }, - }, - }, - Successes: []Result{ - { - Message: "pass", - Metadata: map[string]interface{}{ - metadataCode: "a.success1", - metadataDependsOn: []string{"a.failure1"}, - }, - }, - }, - }, - }, - expected: []Outcome{ - { - Failures: []Result{ - { - Message: "failure 1", - Metadata: map[string]interface{}{ - metadataCode: "a.failure1", - }, - }, - }, - Successes: []Result{}, - }, - }, - }, - } - - for i, c := range cases { - t.Run(c.name, func(t *testing.T) { - trim(&cases[i].given) - assert.Equal(t, c.expected, c.given) - }) - } -} - -// TestIsResultIncludedWithComponentName tests the isResultIncluded method -// with the componentName parameter to ensure component-based filtering works correctly. -// This test exercises the legacy fallback path for backward compatibility. -func TestIsResultIncludedWithComponentName(t *testing.T) { - tests := []struct { - name string - result Result - imageRef string - componentName string - include *Criteria - exclude *Criteria - expected bool - }{ - { - name: "include by component name", - result: Result{ - Metadata: map[string]any{"code": "test.check_a"}, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.check_a"}, - }, - defaultItems: []string{}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expected: true, - }, - { - name: "exclude by component name", - result: Result{ - Metadata: map[string]any{"code": "test.check_b"}, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{"*"}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.check_b"}, - }, - defaultItems: []string{}, - }, - expected: false, - }, - { - name: "component-specific include overrides global exclude", - result: Result{ - Metadata: map[string]any{"code": "test.check_c"}, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.check_c"}, - }, - defaultItems: []string{}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{"test"}, - }, - expected: true, - }, - { - name: "different component - not included", - result: Result{ - Metadata: map[string]any{"code": "test.check_d"}, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "other-component", - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.check_d"}, - }, - defaultItems: []string{}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expected: false, - }, - { - name: "empty component name - uses only global criteria", - result: Result{ - Metadata: map[string]any{"code": "test.check_e"}, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "", - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.check_e"}, - }, - defaultItems: []string{"*"}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expected: true, // Matches global "*" - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - evaluator := conftestEvaluator{ - include: tt.include, - exclude: tt.exclude, - } - missingIncludes := map[string]bool{} - got := evaluator.isResultIncluded(tt.result, tt.imageRef, tt.componentName, missingIncludes) - assert.Equal(t, tt.expected, got) - }) - } -} - -// TestComputeSuccessesLegacyFallback tests the computeSuccesses method with nil unifiedFilter -// to exercise the legacy fallback path that uses isResultIncluded directly. -// This ensures backward compatibility and provides coverage for the legacy code path. -func TestComputeSuccessesLegacyFallback(t *testing.T) { - tests := []struct { - name string - result Outcome - rules policyRules - imageRef string - componentName string - missingIncludes map[string]bool - include *Criteria - exclude *Criteria - expectedCount int - expectedCodes []string - }{ - { - name: "include success by component name - legacy path", - result: Outcome{ - Namespace: "test", - Failures: []Result{}, - Warnings: []Result{}, - Skipped: []Result{}, - }, - rules: policyRules{ - "test.success_rule": { - Code: "test.success_rule", - Package: "test", - ShortName: "success_rule", - Title: "Success Rule", - }, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - missingIncludes: map[string]bool{}, - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.success_rule"}, - }, - defaultItems: []string{}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expectedCount: 1, - expectedCodes: []string{"test.success_rule"}, - }, - { - name: "exclude success by component name - legacy path", - result: Outcome{ - Namespace: "test", - Failures: []Result{}, - Warnings: []Result{}, - Skipped: []Result{}, - }, - rules: policyRules{ - "test.excluded_rule": { - Code: "test.excluded_rule", - Package: "test", - ShortName: "excluded_rule", - Title: "Excluded Rule", - }, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - missingIncludes: map[string]bool{}, - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{"*"}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.excluded_rule"}, - }, - defaultItems: []string{}, - }, - expectedCount: 0, - expectedCodes: []string{}, - }, - { - name: "multiple rules with mixed inclusion - legacy path", - result: Outcome{ - Namespace: "test", - Failures: []Result{}, - Warnings: []Result{}, - Skipped: []Result{}, - }, - rules: policyRules{ - "test.included_rule": { - Code: "test.included_rule", - Package: "test", - ShortName: "included_rule", - Title: "Included Rule", - }, - "test.excluded_rule": { - Code: "test.excluded_rule", - Package: "test", - ShortName: "excluded_rule", - Title: "Excluded Rule", - }, - }, - imageRef: "quay.io/test/image@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - missingIncludes: map[string]bool{}, - include: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.included_rule"}, - }, - defaultItems: []string{}, - }, - exclude: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expectedCount: 1, - expectedCodes: []string{"test.included_rule"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - evaluator := conftestEvaluator{ - include: tt.include, - exclude: tt.exclude, - } - - // Call computeSuccesses with nil unifiedFilter to exercise the legacy path - successes := evaluator.computeSuccesses( - tt.result, - tt.rules, - tt.imageRef, - tt.componentName, - tt.missingIncludes, - nil, // nil unifiedFilter triggers the legacy fallback path - ) - - assert.Equal(t, tt.expectedCount, len(successes), "unexpected number of successes") - - // Verify the expected codes are present - actualCodes := make([]string, 0, len(successes)) - for _, success := range successes { - if code, ok := success.Metadata[metadataCode].(string); ok { - actualCodes = append(actualCodes, code) - } - } - assert.ElementsMatch(t, tt.expectedCodes, actualCodes, "unexpected success codes") - }) - } -} diff --git a/internal/evaluator/conftest_evaluator_unit_metadata_test.go b/internal/evaluator/conftest_evaluator_unit_metadata_test.go deleted file mode 100644 index edd9fd4ac..000000000 --- a/internal/evaluator/conftest_evaluator_unit_metadata_test.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -// This file contains unit tests for rule metadata processing and annotation handling. -// It includes tests for: -// - Annotation data collection (TestCollectAnnotationData) -// - Rule metadata processing (TestRuleMetadata) -// - Rules without metadata handling (TestRulesWithoutMetadata) -// - Warning for rules not showing up (TestWarnRuleNotShowingUp) -// These tests focus on how the evaluator processes and handles rule metadata -// and annotations from OPA policies. - -//go:build unit - -package evaluator - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - "github.com/MakeNowJust/heredoc" - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/open-policy-agent/opa/v1/ast" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/conforma/cli/internal/opa/rule" - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" -) - -func TestCollectAnnotationData(t *testing.T) { - module := ast.MustParseModuleWithOpts(heredoc.Doc(` - package a.b.c - import rego.v1 - - # METADATA - # title: Title - # description: Description - # custom: - # short_name: short - # collections: [A, B, C] - # effective_on: 2022-01-01T00:00:00Z - # depends_on: a.b.c - # pipeline_intention: [release, production] - deny contains msg if { - msg := "hi" - }`), ast.ParserOptions{ - ProcessAnnotation: true, - }) - - rules := policyRules{} - require.NoError(t, rules.collect(ast.NewAnnotationsRef(module.Annotations[0]))) - - assert.Equal(t, policyRules{ - "a.b.c.short": { - Code: "a.b.c.short", - Collections: []string{"A", "B", "C"}, - DependsOn: []string{"a.b.c"}, - Description: "Description", - EffectiveOn: "2022-01-01T00:00:00Z", - Kind: rule.Deny, - Package: "a.b.c", - PipelineIntention: []string{"release", "production"}, - ShortName: "short", - Title: "Title", - DocumentationUrl: "https://conforma.dev/docs/policy/packages/release_c.html#c__short", - }, - }, rules) -} - -func TestRuleMetadata(t *testing.T) { - effectiveOnTest := time.Now().Format(effectiveOnFormat) - - effectiveTimeTest := time.Now().Add(-24 * time.Hour) - ctx := context.TODO() - ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTimeTest) - - rules := policyRules{ - "warning1": rule.Info{ - Title: "Warning1", - }, - "failure2": rule.Info{ - Title: "Failure2", - Description: "Failure 2 description", - }, - "warning2": rule.Info{ - Title: "Warning2", - Description: "Warning 2 description", - EffectiveOn: "2022-01-01T00:00:00Z", - }, - "warning3": rule.Info{ - Title: "Warning3", - Description: "Warning 3 description", - EffectiveOn: effectiveOnTest, - }, - "pipelineIntentionRule": rule.Info{ - Title: "Pipeline Intention Rule", - Description: "Rule with pipeline intention", - PipelineIntention: []string{"release", "production"}, - }, - } - cases := []struct { - name string - result Result - rules policyRules - want Result - }{ - { - name: "update title", - result: Result{ - Metadata: map[string]any{ - "code": "warning1", - "collections": []any{"A"}, - }, - }, - rules: rules, - want: Result{ - Metadata: map[string]any{ - "code": "warning1", - "collections": []string{"A"}, - "title": "Warning1", - }, - }, - }, - { - name: "update title and description", - result: Result{ - Metadata: map[string]any{ - "code": "failure2", - "collections": []any{"A"}, - }, - }, - rules: rules, - want: Result{ - Metadata: map[string]any{ - "code": "failure2", - "collections": []string{"A"}, - "description": "Failure 2 description", - "title": "Failure2", - }, - }, - }, - { - name: "drop stale effectiveOn", - result: Result{ - Metadata: map[string]any{ - "code": "warning2", - "collections": []any{"A"}, - }, - }, - rules: rules, - want: Result{ - Metadata: map[string]any{ - "code": "warning2", - "collections": []string{"A"}, - "description": "Warning 2 description", - "title": "Warning2", - }, - }, - }, - { - name: "add relevant effectiveOn", - result: Result{ - Metadata: map[string]any{ - "code": "warning3", - "collections": []any{"A"}, - }, - }, - rules: rules, - want: Result{ - Metadata: map[string]any{ - "code": "warning3", - "collections": []string{"A"}, - "description": "Warning 3 description", - "title": "Warning3", - "effective_on": effectiveOnTest, - }, - }, - }, - { - name: "rule not found", - result: Result{ - Metadata: map[string]any{ - "collections": []any{"A"}, - }, - }, - rules: rules, - want: Result{ - Metadata: map[string]any{ - "collections": []any{"A"}, - }, - }, - }, - { - name: "add pipeline intention metadata", - result: Result{ - Metadata: map[string]any{ - "code": "pipelineIntentionRule", - "collections": []any{"B"}, - }, - }, - rules: rules, - want: Result{ - Metadata: map[string]any{ - "code": "pipelineIntentionRule", - "collections": []string{"B"}, - "title": "Pipeline Intention Rule", - "description": "Rule with pipeline intention", - }, - }, - }, - } - for i, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - addRuleMetadata(ctx, &cases[i].result, tt.rules) - assert.Equal(t, tt.result, tt.want) - }) - } -} - -func TestRulesWithoutMetadata(t *testing.T) { - // Create a temporary directory for the test - tempDir := t.TempDir() - - // Create a simple policy file without metadata - policyContent := `package main - -import rego.v1 - -deny contains result if { - result := { - "msg": "Simple deny rule", - "severity": "failure" - } -} - -warn contains result if { - result := { - "msg": "Simple warn rule", - "severity": "warning" - } -}` - - policyFile := filepath.Join(tempDir, "simple.rego") - err := os.WriteFile(policyFile, []byte(policyContent), 0600) - require.NoError(t, err) - - // Create input directory structure - inputDir := filepath.Join(tempDir, "inputs") - require.NoError(t, os.MkdirAll(inputDir, 0755)) - inputFile := filepath.Join(inputDir, "data.json") - err = os.WriteFile(inputFile, []byte("{}"), 0600) - require.NoError(t, err) - - // Create evaluator using the proper constructor - ctx := context.Background() - config := &mockConfigProvider{} - config.On("EffectiveTime").Return(time.Now()) - config.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - config.On("Spec").Return(ecc.EnterpriseContractPolicySpec{}) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{ - Url: tempDir, - Kind: source.PolicyKind, - }, - }, config, ecc.Source{}, []string{}) - require.NoError(t, err) - - // Evaluate the policy - results, err := evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{inputDir}}) - - // The evaluation should succeed - require.NoError(t, err) - require.NotNil(t, results) - require.Len(t, results, 1, "Expected one result set") - - result := results[0] - - // Check that we have results (this is what the acceptance test expects) - // The rules should always evaluate to true since they have no conditions - totalResults := len(result.Failures) + len(result.Warnings) + len(result.Successes) - require.Greater(t, totalResults, 0, "Expected to find at least one result from the simple.rego rules") - - // Check that we have the expected results - require.Len(t, result.Failures, 1, "Expected 1 deny rule") - require.Len(t, result.Warnings, 1, "Expected 1 warn rule") - - // Verify the content of the results - expectedMessages := []string{ - "Simple deny rule", - "Simple warn rule", - } - - allResults := append(result.Failures, result.Warnings...) - require.Len(t, allResults, 2, "Expected 2 total results") - - for _, expectedMsg := range expectedMessages { - found := false - for _, result := range allResults { - if result.Message == expectedMsg { - found = true - break - } - } - require.True(t, found, "Expected to find result with message: %s", expectedMsg) - } -} - -func TestWarnRuleNotShowingUp(t *testing.T) { - // Create a temporary directory for the test - tempDir := t.TempDir() - - // Create the warn.rego file (exact content from acceptance test) - policyContent := `# Simplest always-warning policy -package main - -import rego.v1 - -warn contains result if { - result := "Has a warning" -}` - - policyFile := filepath.Join(tempDir, "warn.rego") - err := os.WriteFile(policyFile, []byte(policyContent), 0600) - require.NoError(t, err) - - // Create input directory structure - inputDir := filepath.Join(tempDir, "inputs") - require.NoError(t, os.MkdirAll(inputDir, 0755)) - inputFile := filepath.Join(inputDir, "data.json") - err = os.WriteFile(inputFile, []byte("{}"), 0600) - require.NoError(t, err) - - // Create evaluator using the proper constructor - ctx := context.Background() - config := &mockConfigProvider{} - config.On("EffectiveTime").Return(time.Now()) - config.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) - config.On("Spec").Return(ecc.EnterpriseContractPolicySpec{}) - - evaluator, err := NewConftestEvaluatorWithNamespace(ctx, []source.PolicySource{ - &source.PolicyUrl{ - Url: tempDir, - Kind: source.PolicyKind, - }, - }, config, ecc.Source{}, []string{}) - require.NoError(t, err) - - // Evaluate the policy - results, err := evaluator.Evaluate(ctx, EvaluationTarget{Inputs: []string{inputDir}}) - - // The evaluation should succeed - require.NoError(t, err) - require.NotNil(t, results) - require.Len(t, results, 1, "Expected one result set") - - result := results[0] - - // Check that we have the warning result - require.Len(t, result.Warnings, 1, "Expected 1 warn rule from warn.rego") - require.Equal(t, "Has a warning", result.Warnings[0].Message, "Expected warning message to match") - - // The warning should be included in the output - totalResults := len(result.Failures) + len(result.Warnings) + len(result.Successes) - require.Greater(t, totalResults, 0, "Expected to find at least one result from the warn.rego rules") -} diff --git a/internal/evaluator/criteria_test.go b/internal/evaluator/criteria_test.go deleted file mode 100644 index 446ea417f..000000000 --- a/internal/evaluator/criteria_test.go +++ /dev/null @@ -1,988 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -//go:build unit - -package evaluator - -import ( - "testing" - "time" - - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/conforma/cli/internal/policy" -) - -func TestLen(t *testing.T) { - tests := []struct { - name string - criteria *Criteria - expectedLen int - }{ - { - name: "Empty Criteria", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expectedLen: 0, - }, - { - name: "Only Default Items", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{}, - defaultItems: []string{"default1", "default2"}, - }, - expectedLen: 2, - }, - { - name: "Only Digest Items", - criteria: &Criteria{ - digestItems: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - componentItems: map[string][]string{}, - defaultItems: []string{}, - }, - expectedLen: 3, - }, - { - name: "Only Component Items", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "comp1": {"value1", "value2"}, - "comp2": {"value3"}, - }, - defaultItems: []string{}, - }, - expectedLen: 3, - }, - { - name: "Both Default and Digest Items", - criteria: &Criteria{ - digestItems: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - componentItems: map[string][]string{}, - defaultItems: []string{"default1", "default2"}, - }, - expectedLen: 5, - }, - { - name: "Default, Digest, and Component Items", - criteria: &Criteria{ - digestItems: map[string][]string{ - "key1": {"value1", "value2"}, - }, - componentItems: map[string][]string{ - "comp1": {"value3"}, - "comp2": {"value4", "value5"}, - }, - defaultItems: []string{"default1"}, - }, - expectedLen: 6, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.criteria.len(); got != tt.expectedLen { - t.Errorf("Criteria.len() = %d, want %d", got, tt.expectedLen) - } - }) - } -} - -func TestAddItem(t *testing.T) { - tests := []struct { - name string - key string - value string - initial *Criteria - expected *Criteria - }{ - { - name: "Add to defaultItems", - key: "", - value: "defaultValue", - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{"defaultValue"}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - }, - { - name: "Add to digestItems", - key: "key1", - value: "digestValue1", - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: map[string][]string{ - "key1": {"digestValue1"}, - }, - componentItems: make(map[string][]string), - }, - }, - { - name: "Add to existing digestItems", - key: "key1", - value: "digestValue2", - initial: &Criteria{ - defaultItems: []string{}, - digestItems: map[string][]string{ - "key1": {"digestValue1"}, - }, - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: map[string][]string{ - "key1": {"digestValue1", "digestValue2"}, - }, - componentItems: make(map[string][]string), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.initial.addItem(tt.key, tt.value) - require.Equal(t, tt.initial, tt.expected) - }) - } -} - -func TestAddArray(t *testing.T) { - tests := []struct { - name string - key string - values []string - initial *Criteria - expected *Criteria - }{ - { - name: "Add to defaultItems", - key: "", - values: []string{"defaultValue1", "defaultValue2"}, - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{"defaultValue1", "defaultValue2"}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - }, - { - name: "Add to digestItems", - key: "key1", - values: []string{"digestValue1", "digestValue2"}, - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: map[string][]string{ - "key1": {"digestValue1", "digestValue2"}, - }, - componentItems: make(map[string][]string), - }, - }, - { - name: "Add to existing digestItems", - key: "key1", - values: []string{"digestValue2", "digestValue3"}, - initial: &Criteria{ - defaultItems: []string{}, - digestItems: map[string][]string{ - "key1": {"digestValue1"}, - }, - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: map[string][]string{ - "key1": {"digestValue1", "digestValue2", "digestValue3"}, - }, - componentItems: make(map[string][]string), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.initial.addArray(tt.key, tt.values) - require.Equal(t, tt.initial, tt.expected) - }) - } -} - -func TestAddComponentItem(t *testing.T) { - tests := []struct { - name string - componentName string - value string - initial *Criteria - expected *Criteria - }{ - { - name: "Add to componentItems", - componentName: "comp1", - value: "componentValue1", - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"componentValue1"}, - }, - }, - }, - { - name: "Add to existing componentItems", - componentName: "comp1", - value: "componentValue2", - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"componentValue1"}, - }, - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"componentValue1", "componentValue2"}, - }, - }, - }, - { - name: "Add to different components", - componentName: "comp2", - value: "componentValue3", - initial: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"componentValue1"}, - }, - }, - expected: &Criteria{ - defaultItems: []string{}, - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"componentValue1"}, - "comp2": {"componentValue3"}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.initial.addComponentItem(tt.componentName, tt.value) - require.Equal(t, tt.expected, tt.initial) - }) - } -} - -func TestGet(t *testing.T) { - c := &Criteria{ - digestItems: map[string][]string{ - "quay.io/test/ec-test": {"item"}, - "sha256:2c5e3b2f1e2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c": {"item-digest"}, - }, - componentItems: map[string][]string{}, - defaultItems: []string{"default1", "default2"}, - } - tests := []struct { - name string - key string - expected []string - }{ - { - name: "test with image ref", - key: "quay.io/test/ec-test", - expected: []string{"item", "default1", "default2"}, - }, - { - name: "test with image ref and tag", - key: "quay.io/test/ec-test:latest", - expected: []string{"item", "default1", "default2"}, - }, - { - name: "test with image digest", - key: "quay.io/test/ec@sha256:2c5e3b2f1e2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c", - expected: []string{"item-digest", "default1", "default2"}, - }, - { - name: "test key doesn't exist", - key: "unknown", - expected: []string{"default1", "default2"}, - }, - { - name: "test with image and bad digest", - key: "quay.io/test/ec-test@sha256:2c5e3b2f1e2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d", - expected: []string{"default1", "default2"}, - }, - { - name: "test with image not set", - expected: []string{"default1", "default2"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, c.get(tt.key, "")) - }) - } -} - -// MockConfigProvider implements ConfigProvider interface for testing -type MockConfigProvider struct { - effectiveTime time.Time -} - -func (m *MockConfigProvider) EffectiveTime() time.Time { - return m.effectiveTime -} - -func (m *MockConfigProvider) SigstoreOpts() (policy.SigstoreOpts, error) { - return policy.SigstoreOpts{}, nil -} - -func (m *MockConfigProvider) Spec() ecc.EnterpriseContractPolicySpec { - return ecc.EnterpriseContractPolicySpec{} -} - -func TestCollectVolatileConfigItems(t *testing.T) { - // Create a fixed time for testing - fixedTime := time.Date(2025, 8, 18, 12, 0, 0, 0, time.UTC) - - tests := []struct { - name string - items *Criteria - volatileCriteria []ecc.VolatileCriteria - configProvider ConfigProvider - expectedItems *Criteria - expectedSuccess bool - }{ - { - name: "Successful scenario - criteria within time range", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "volatile-item-1", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ImageRef: "quay.io/test/image:latest", - }, - { - Value: "volatile-item-2", - EffectiveOn: "2025-08-10T00:00:00Z", - EffectiveUntil: "2025-08-20T23:59:59Z", - ImageDigest: "sha256:abc123", - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedItems: &Criteria{ - digestItems: map[string][]string{ - "quay.io/test/image:latest": {"volatile-item-1"}, - "sha256:abc123": {"volatile-item-2"}, - }, - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - expectedSuccess: true, - }, - { - name: "Failed scenario - criteria outside time range", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "expired-item", - EffectiveOn: "2025-07-01T00:00:00Z", - EffectiveUntil: "2025-07-31T23:59:59Z", - ImageUrl: "quay.io/test/expired", - }, - { - Value: "future-item", - EffectiveOn: "2025-09-01T00:00:00Z", - EffectiveUntil: "2025-09-30T23:59:59Z", - ImageRef: "quay.io/test/future", - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedItems: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - expectedSuccess: true, // Function doesn't fail, just doesn't add items - }, - { - name: "Warning scenario - invalid time formats", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "partial-invalid-item", - EffectiveOn: "2025-08-01T00:00:00Z", // Valid format - EffectiveUntil: "not-a-date", // Invalid format - ImageDigest: "sha256:def456", - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedItems: &Criteria{ - digestItems: map[string][]string{ - "sha256:def456": {"partial-invalid-item"}, - }, - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - expectedSuccess: true, // Function handles invalid times gracefully - }, - { - name: "Component names with volatile criteria", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "cve.scanning", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1", "comp2", "comp3"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedItems: &Criteria{ - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"cve.scanning"}, - "comp2": {"cve.scanning"}, - "comp3": {"cve.scanning"}, - }, - defaultItems: []string{"existing-item"}, - }, - expectedSuccess: true, - }, - { - name: "Component names with multiple values", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing-item"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "cve.scanning", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1", "comp2"}, - }, - { - Value: "slsa.provenance", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedItems: &Criteria{ - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"cve.scanning", "slsa.provenance"}, - "comp2": {"cve.scanning"}, - }, - defaultItems: []string{"existing-item"}, - }, - expectedSuccess: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create a copy of the initial items to avoid modifying the test data - initialItems := &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: make([]string, len(tt.items.defaultItems)), - } - copy(initialItems.defaultItems, tt.items.defaultItems) - for k, v := range tt.items.digestItems { - initialItems.digestItems[k] = make([]string, len(v)) - copy(initialItems.digestItems[k], v) - } - for k, v := range tt.items.componentItems { - initialItems.componentItems[k] = make([]string, len(v)) - copy(initialItems.componentItems[k], v) - } - - // Call the function - result := collectVolatileConfigItems(initialItems, tt.volatileCriteria, tt.configProvider) - - // Verify the result - if tt.expectedSuccess { - require.Equal(t, tt.expectedItems.defaultItems, result.defaultItems, "defaultItems mismatch") - require.Equal(t, len(tt.expectedItems.digestItems), len(result.digestItems), "digestItems count mismatch") - - for expectedKey, expectedValues := range tt.expectedItems.digestItems { - actualValues, exists := result.digestItems[expectedKey] - require.True(t, exists, "Expected key %s not found in result", expectedKey) - require.Equal(t, expectedValues, actualValues, "Values mismatch for key %s", expectedKey) - } - } - }) - } -} - -func TestCollectVolatileConfigItemsWithComponentNames(t *testing.T) { - // Create a fixed time for testing - fixedTime := time.Date(2025, 8, 18, 12, 0, 0, 0, time.UTC) - - tests := []struct { - name string - items *Criteria - volatileCriteria []ecc.VolatileCriteria - configProvider ConfigProvider - expectedComponentItems map[string][]string - expectedDefaultItems []string - }{ - { - name: "ComponentNames only - single component", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_a", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.check_a"}, - }, - expectedDefaultItems: []string{}, - }, - { - name: "ComponentNames - multiple components", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_b", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1", "comp2"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.check_b"}, - "comp2": {"test.check_b"}, - }, - expectedDefaultItems: []string{}, - }, - { - name: "ComponentNames outside time window - effectiveUntil passed", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_c", - EffectiveOn: "2025-07-01T00:00:00Z", - EffectiveUntil: "2025-07-31T23:59:59Z", // Before fixedTime - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{}, - expectedDefaultItems: []string{}, - }, - { - name: "ComponentNames with future effectiveOn", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_d", - EffectiveOn: "2025-09-01T00:00:00Z", // After fixedTime - EffectiveUntil: "2025-09-30T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{}, - expectedDefaultItems: []string{}, - }, - { - name: "ComponentNames within time window", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_e", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.check_e"}, - }, - expectedDefaultItems: []string{}, - }, - { - name: "Multiple criteria - different components", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_f", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - { - Value: "test.check_g", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp2"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.check_f"}, - "comp2": {"test.check_g"}, - }, - expectedDefaultItems: []string{}, - }, - { - name: "Multiple criteria - same component accumulates", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_h", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - { - Value: "test.check_i", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.check_h", "test.check_i"}, - }, - expectedDefaultItems: []string{}, - }, - { - name: "ComponentNames with existing default items", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{"existing.default"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.check_j", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.check_j"}, - }, - expectedDefaultItems: []string{"existing.default"}, - }, - { - name: "Mix of ComponentNames and global criteria", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: make(map[string][]string), - defaultItems: []string{}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.component_check", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - { - Value: "test.global_check", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - // No ComponentNames, ImageRef, ImageUrl, or ImageDigest - global - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"test.component_check"}, - }, - expectedDefaultItems: []string{"test.global_check"}, - }, - { - name: "ComponentNames with existing default items and component items", - items: &Criteria{ - digestItems: make(map[string][]string), - componentItems: map[string][]string{ - "comp1": {"existing.comp_check"}, - }, - defaultItems: []string{"existing.default"}, - }, - volatileCriteria: []ecc.VolatileCriteria{ - { - Value: "test.new_comp_check", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - ComponentNames: []ecc.ComponentName{"comp1"}, - }, - { - Value: "test.new_global", - EffectiveOn: "2025-08-01T00:00:00Z", - EffectiveUntil: "2025-08-31T23:59:59Z", - // Global - }, - }, - configProvider: &MockConfigProvider{effectiveTime: fixedTime}, - expectedComponentItems: map[string][]string{ - "comp1": {"existing.comp_check", "test.new_comp_check"}, - }, - expectedDefaultItems: []string{"existing.default", "test.new_global"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := collectVolatileConfigItems(tt.items, tt.volatileCriteria, tt.configProvider) - - // Verify componentItems - require.Equal(t, len(tt.expectedComponentItems), len(result.componentItems), "componentItems count mismatch") - for expectedKey, expectedValues := range tt.expectedComponentItems { - actualValues, exists := result.componentItems[expectedKey] - require.True(t, exists, "Expected component key %s not found in result", expectedKey) - require.Equal(t, expectedValues, actualValues, "Values mismatch for component %s", expectedKey) - } - - // Verify defaultItems - require.Equal(t, tt.expectedDefaultItems, result.defaultItems, "defaultItems mismatch") - }) - } -} - -func TestCriteriaGetWithComponentName(t *testing.T) { - tests := []struct { - name string - criteria *Criteria - imageRef string - componentName string - expected []string - }{ - { - name: "Component match - returns component-specific + global", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"@minimal", "test.some_policy"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - expected: []string{"@minimal", "test.some_policy", "*"}, - }, - { - name: "Component no match - returns only global", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"@minimal", "test.some_policy"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "other-component", - expected: []string{"*"}, - }, - { - name: "Empty component name - returns only image + global", - criteria: &Criteria{ - digestItems: map[string][]string{ - "quay.io/repo/img": {"test.image_check"}, - }, - componentItems: map[string][]string{ - "my-component": {"@minimal"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "", - expected: []string{"test.image_check", "*"}, - }, - { - name: "Image + Component both match - returns all merged", - criteria: &Criteria{ - digestItems: map[string][]string{ - "quay.io/repo/img": {"test.image_check"}, - "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef": {"test.digest_check"}, - }, - componentItems: map[string][]string{ - "my-component": {"test.component_check"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - expected: []string{"test.image_check", "test.digest_check", "test.component_check", "*"}, - }, - { - name: "Invalid image ref - returns only global (error fallback)", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"test.component_check"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "::::invalid:::::", - componentName: "my-component", - expected: []string{"*"}, - }, - { - name: "No matches at all - returns only global", - criteria: &Criteria{ - digestItems: map[string][]string{ - "quay.io/other/img": {"test.other_check"}, - }, - componentItems: map[string][]string{ - "other-component": {"test.other_component"}, - }, - defaultItems: []string{"default1", "default2"}, - }, - imageRef: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - expected: []string{"default1", "default2"}, - }, - { - name: "Multiple component items", - criteria: &Criteria{ - digestItems: map[string][]string{}, - componentItems: map[string][]string{ - "my-component": {"check1", "check2", "check3"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - componentName: "my-component", - expected: []string{"check1", "check2", "check3", "*"}, - }, - { - name: "Image without digest - returns only repo + component + global", - criteria: &Criteria{ - digestItems: map[string][]string{ - "quay.io/repo/img": {"test.image_check"}, - }, - componentItems: map[string][]string{ - "my-component": {"test.component_check"}, - }, - defaultItems: []string{"*"}, - }, - imageRef: "quay.io/repo/img:latest", - componentName: "my-component", - expected: []string{"test.image_check", "test.component_check", "*"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := tt.criteria.get(tt.imageRef, tt.componentName) - require.Equal(t, tt.expected, result) - }) - } -} diff --git a/internal/evaluator/filters_test.go b/internal/evaluator/filters_test.go deleted file mode 100644 index 293b2eb59..000000000 --- a/internal/evaluator/filters_test.go +++ /dev/null @@ -1,1798 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -//go:build unit - -package evaluator - -import ( - "encoding/json" - "strings" - "testing" - "time" - - ecc "github.com/conforma/crds/api/v1alpha1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - - "github.com/conforma/cli/internal/opa/rule" - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" -) - -////////////////////////////////////////////////////////////////////////////// -// test scaffolding -////////////////////////////////////////////////////////////////////////////// - -func makeSource(ruleData string, includes []string) ecc.Source { - s := ecc.Source{} - if ruleData != "" { - s.RuleData = &extv1.JSON{Raw: json.RawMessage(ruleData)} - } - if len(includes) > 0 { - s.Config = &ecc.SourceConfig{Include: includes} - } - return s -} - -////////////////////////////////////////////////////////////////////////////// -// FilterFactory tests -////////////////////////////////////////////////////////////////////////////// - -func TestDefaultFilterFactory(t *testing.T) { - tests := []struct { - name string - source ecc.Source - wantFilters int - }{ - { - name: "no config", - source: ecc.Source{}, - wantFilters: 1, // Always adds PipelineIntentionFilter - }, - { - name: "pipeline intention only", - source: makeSource(`{"pipeline_intention":"release"}`, nil), - wantFilters: 1, - }, - { - name: "include list only", - source: makeSource("", []string{"@redhat", "cve"}), - wantFilters: 2, // PipelineIntentionFilter + IncludeListFilter - }, - { - name: "both pipeline_intention and include list", - source: makeSource(`{"pipeline_intention":"release"}`, []string{"@redhat", "cve"}), - wantFilters: 2, - }, - { - name: "no includes and no pipeline_intention - PipelineIntentionFilter still added", - source: makeSource("", nil), - wantFilters: 1, // PipelineIntentionFilter is always added - }, - } - - for _, tc := range tests { - got := NewDefaultFilterFactory().CreateFilters(tc.source) - assert.Len(t, got, tc.wantFilters, tc.name) - } -} - -////////////////////////////////////////////////////////////////////////////// -// IncludeListFilter – core behaviour -////////////////////////////////////////////////////////////////////////////// - -func TestIncludeListFilter(t *testing.T) { - rules := policyRules{ - "pkg.rule": {Collections: []string{"redhat"}}, - "cve.rule": {Collections: []string{"security"}}, - "other.rule": {}, - "labels.rule": {Collections: []string{"security"}}, - "foo.bar": {}, - } - - tests := []struct { - name string - entries []string - wantPkgs []string - }{ - { - name: "@redhat collection", - entries: []string{"@redhat"}, - wantPkgs: []string{"pkg"}, - }, - { - name: "explicit package", - entries: []string{"cve"}, - wantPkgs: []string{"cve"}, - }, - { - name: "package.rule entry", - entries: []string{"labels.rule"}, - wantPkgs: []string{"labels"}, - }, - { - name: "OR across entries", - entries: []string{"@redhat", "cve"}, - wantPkgs: []string{"pkg", "cve"}, - }, - { - name: "non‑existent entry", - entries: []string{"@none"}, - wantPkgs: []string{}, - }, - } - - for _, tc := range tests { - got := filterNamespaces(rules, NewIncludeListFilter(tc.entries)) - assert.ElementsMatch(t, tc.wantPkgs, got, tc.name) - } -} - -////////////////////////////////////////////////////////////////////////////// -// PipelineIntentionFilter -////////////////////////////////////////////////////////////////////////////// - -func TestPipelineIntentionFilter(t *testing.T) { - rules := policyRules{ - "a.r": {PipelineIntention: []string{"release"}}, - "b.r": {PipelineIntention: []string{"dev"}}, - "c.r": {}, - } - - tests := []struct { - name string - intentions []string - wantPkgs []string - }{ - { - name: "no intentions ⇒ only packages with no pipeline_intention metadata", - intentions: nil, - wantPkgs: []string{"c"}, // Only c has no pipeline_intention metadata - }, - { - name: "pipeline_intention set - include packages with matching pipeline_intention metadata", - intentions: []string{"release"}, - wantPkgs: []string{"a"}, // Only a has matching pipeline_intention metadata - }, - { - name: "pipeline_intention set with multiple values - include packages with any matching pipeline_intention metadata", - intentions: []string{"dev", "release"}, - wantPkgs: []string{"a", "b"}, // Both a and b have matching pipeline_intention metadata - }, - } - - for _, tc := range tests { - got := filterNamespaces(rules, NewPipelineIntentionFilter(tc.intentions)) - assert.ElementsMatch(t, tc.wantPkgs, got, tc.name) - } -} - -////////////////////////////////////////////////////////////////////////////// -// Complete filtering behavior tests -////////////////////////////////////////////////////////////////////////////// - -func TestCompleteFilteringBehavior(t *testing.T) { - rules := policyRules{ - "release.rule1": {PipelineIntention: []string{"release"}}, - "release.rule2": {PipelineIntention: []string{"release", "production"}}, - "dev.rule1": {PipelineIntention: []string{"dev"}}, - "general.rule1": {}, // No pipeline_intention metadata - "general.rule2": {}, // No pipeline_intention metadata - } - - tests := []struct { - name string - source ecc.Source - expectedPkg []string - }{ - { - name: "no includes and no pipeline_intention - only packages with no pipeline_intention metadata", - source: makeSource("", nil), - expectedPkg: []string{"general"}, // Only general has no pipeline_intention metadata - }, - { - name: "pipeline_intention set - only packages with matching pipeline_intention metadata", - source: makeSource(`{"pipeline_intention":"release"}`, nil), - expectedPkg: []string{"release"}, // Only release has matching pipeline_intention metadata - }, - { - name: "includes set - only matching packages with no pipeline_intention metadata", - source: makeSource("", []string{"release", "general"}), - expectedPkg: []string{"general"}, // Only general has no pipeline_intention metadata and matches includes - }, - { - name: "both pipeline_intention and includes - AND logic", - source: makeSource(`{"pipeline_intention":"release"}`, []string{"release"}), - expectedPkg: []string{"release"}, // Only release matches both conditions - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - filterFactory := NewDefaultFilterFactory() - filters := filterFactory.CreateFilters(tc.source) - got := filterNamespaces(rules, filters...) - assert.ElementsMatch(t, tc.expectedPkg, got, tc.name) - }) - } -} - -////////////////////////////////////////////////////////////////////////////// -// Test filtering with rules that don't have metadata -////////////////////////////////////////////////////////////////////////////// - -func TestFilteringWithRulesWithoutMetadata(t *testing.T) { - // This test demonstrates how filtering works with rules that don't have - // pipeline_intention metadata, like the example fail_with_data.rego rule. - rules := policyRules{ - "main.fail_with_data": {}, // Rule without any metadata (like fail_with_data.rego) - "release.security": {PipelineIntention: []string{"release"}}, - "dev.validation": {PipelineIntention: []string{"dev"}}, - "general.basic": {}, // Another rule without metadata - } - - tests := []struct { - name string - source ecc.Source - expectedPkg []string - description string - }{ - { - name: "no pipeline_intention - only rules without metadata", - source: makeSource("", nil), - expectedPkg: []string{"main", "general"}, // Only packages with rules that have no pipeline_intention metadata - description: "When no pipeline_intention is configured, only rules without pipeline_intention metadata are evaluated", - }, - { - name: "pipeline_intention set - only rules with matching metadata", - source: makeSource(`{"pipeline_intention":"release"}`, nil), - expectedPkg: []string{"release"}, // Only package with matching pipeline_intention metadata - description: "When pipeline_intention is set, only rules with matching pipeline_intention metadata are evaluated", - }, - { - name: "includes with no pipeline_intention - only matching rules without metadata", - source: makeSource("", []string{"main", "release"}), - expectedPkg: []string{"main"}, // Only main has no pipeline_intention metadata and matches includes - description: "When includes are set but no pipeline_intention, only rules without metadata that match includes are evaluated", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - filterFactory := NewDefaultFilterFactory() - filters := filterFactory.CreateFilters(tc.source) - got := filterNamespaces(rules, filters...) - assert.ElementsMatch(t, tc.expectedPkg, got, tc.description) - }) - } -} - -func TestECPolicyResolver(t *testing.T) { - // Create a mock source with policy configuration - source := ecc.Source{ - Config: &ecc.SourceConfig{ - Include: []string{"cve", "@redhat"}, - Exclude: []string{"slsa3", "test.test_data_found"}, - }, - } - - // Create a simple config provider for testing - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - // Create policy resolver - resolver := NewECPolicyResolver(source, configProvider) - - // Create mock rules - rules := policyRules{ - "cve.high_severity": rule.Info{ - Package: "cve", - Code: "cve.high_severity", - Collections: []string{"redhat"}, - }, - "cve.medium_severity": rule.Info{ - Package: "cve", - Code: "cve.medium_severity", - Collections: []string{"redhat"}, - }, - "slsa3.provenance": rule.Info{ - Package: "slsa3", - Code: "slsa3.provenance", - }, - "test.test_data_found": rule.Info{ - Package: "test", - Code: "test.test_data_found", - }, - "tasks.required_tasks_found": rule.Info{ - Package: "tasks", - Code: "tasks.required_tasks_found", - Collections: []string{"redhat"}, - }, - } - - // Resolve policy - result := resolver.ResolvePolicy(rules, "test-target") - - // Verify included rules - assert.True(t, result.IncludedRules["cve.high_severity"], "cve.high_severity should be included") - assert.True(t, result.IncludedRules["cve.medium_severity"], "cve.medium_severity should be included") - assert.True(t, result.IncludedRules["tasks.required_tasks_found"], "tasks.required_tasks_found should be included") - - // Verify excluded rules - assert.True(t, result.ExcludedRules["slsa3.provenance"], "slsa3.provenance should be excluded") - assert.True(t, result.ExcludedRules["test.test_data_found"], "test.test_data_found should be excluded") - - // Verify included packages - assert.True(t, result.IncludedPackages["cve"], "cve package should be included") - assert.True(t, result.IncludedPackages["tasks"], "tasks package should be included") - assert.False(t, result.IncludedPackages["slsa3"], "slsa3 package should not be included (contains only excluded rules)") - assert.False(t, result.IncludedPackages["test"], "test package should not be included (contains only excluded rules)") - - // Verify that packages with only excluded rules are not included - // (they should not appear in IncludedPackages) - - // Verify explanations - assert.Contains(t, result.Explanations["cve.high_severity"], "included") - assert.Contains(t, result.Explanations["slsa3.provenance"], "excluded") -} - -func TestECPolicyResolver_DefaultBehavior(t *testing.T) { - // Create a source with no explicit includes (should default to "*") - source := ecc.Source{ - Config: &ecc.SourceConfig{ - Exclude: []string{"test.test_data_found"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - rules := policyRules{ - "cve.high_severity": rule.Info{ - Package: "cve", - Code: "cve.high_severity", - }, - "test.test_data_found": rule.Info{ - Package: "test", - Code: "test.test_data_found", - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // Should include everything by default except explicitly excluded - assert.True(t, result.IncludedRules["cve.high_severity"], "cve.high_severity should be included by default") - assert.True(t, result.ExcludedRules["test.test_data_found"], "test.test_data_found should be excluded") -} - -func TestECPolicyResolver_PipelineIntention_RuleLevel(t *testing.T) { - // Create a source with pipeline intention - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - rules := policyRules{ - "tasks.build_task": rule.Info{ - Package: "tasks", - Code: "tasks.build_task", - PipelineIntention: []string{"build"}, - }, - "tasks.deploy_task": rule.Info{ - Package: "tasks", - Code: "tasks.deploy_task", - PipelineIntention: []string{"deploy"}, - }, - "general.security_check": rule.Info{ - Package: "general", - Code: "general.security_check", - // No pipeline intention - should not be included - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // Debug output - t.Logf("Pipeline intentions: %v", resolver.(*ECPolicyResolver).pipelineIntentions) - t.Logf("Included rules: %v", result.IncludedRules) - t.Logf("Excluded rules: %v", result.ExcludedRules) - t.Logf("Explanations: %v", result.Explanations) - - // Pipeline intention filtering now works at rule level - // Only rules that match the pipeline intention should be included - assert.True(t, result.IncludedRules["tasks.build_task"], "tasks.build_task should be included (matches build intention)") - assert.False(t, result.IncludedRules["tasks.deploy_task"], "tasks.deploy_task should not be included (doesn't match build intention)") - assert.False(t, result.IncludedRules["general.security_check"], "general.security_check should not be included (no pipeline intention)") - - // Check package inclusion - tasks package should be included because it has at least one included rule - assert.True(t, result.IncludedPackages["tasks"], "tasks package should be included (has included rules)") - assert.False(t, result.IncludedPackages["general"], "general package should not be included (no included rules)") -} - -func TestECPolicyResolver_PipelineIntention_NoIntentionSpecified(t *testing.T) { - // Create a source with no pipeline intention - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - rules := policyRules{ - "tasks.build_task": rule.Info{ - Package: "tasks", - Code: "tasks.build_task", - PipelineIntention: []string{"build"}, - }, - "general.security_check": rule.Info{ - Package: "general", - Code: "general.security_check", - // No pipeline intention - should be included - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // When no pipeline intention is specified, only rules with no pipeline intention should be included - assert.False(t, result.IncludedRules["tasks.build_task"], "tasks.build_task should not be included (has pipeline intention)") - assert.True(t, result.IncludedRules["general.security_check"], "general.security_check should be included (no pipeline intention)") - - // Check package inclusion - assert.False(t, result.IncludedPackages["tasks"], "tasks package should not be included (no included rules)") - assert.True(t, result.IncludedPackages["general"], "general package should be included (has included rules)") -} - -func TestECPolicyResolver_Example(t *testing.T) { - // Example: Using the comprehensive policy resolver with the policy config from the user's example - - // Create a source with the policy configuration from the user's example - source := ecc.Source{ - Config: &ecc.SourceConfig{ - Include: []string{ - "cve", // package example - "@redhat", // collection example - }, - Exclude: []string{ - "slsa3", // exclude package example - "test.test_data_found", // exclude a rule - "tasks.required_tasks_found:clamav-scan", // exclude a rule with a term - }, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - // Create mock rules that would be found in the policy - rules := policyRules{ - "cve.high_severity": rule.Info{ - Package: "cve", - Code: "cve.high_severity", - Collections: []string{"redhat"}, - }, - "cve.medium_severity": rule.Info{ - Package: "cve", - Code: "cve.medium_severity", - Collections: []string{"redhat"}, - }, - "slsa3.provenance": rule.Info{ - Package: "slsa3", - Code: "slsa3.provenance", - }, - "test.test_data_found": rule.Info{ - Package: "test", - Code: "test.test_data_found", - }, - "tasks.required_tasks_found": rule.Info{ - Package: "tasks", - Code: "tasks.required_tasks_found", - Collections: []string{"redhat"}, - }, - "tasks.build_task": rule.Info{ - Package: "tasks", - Code: "tasks.build_task", - Collections: []string{"redhat"}, - }, - } - - // Use the convenience function to get comprehensive policy resolution - result := GetECPolicyResolution(source, configProvider, rules, "test-target") - - // Verify the results - t.Logf("=== Comprehensive Policy Resolution Results ===") - t.Logf("Included Rules: %v", result.IncludedRules) - t.Logf("Excluded Rules: %v", result.ExcludedRules) - t.Logf("Included Packages: %v", result.IncludedPackages) - t.Logf("Missing Includes: %v", result.MissingIncludes) - t.Logf("Explanations: %v", result.Explanations) - - // Expected behavior based on the policy configuration: - // - cve.high_severity: included (matches "cve" package and "@redhat" collection) - // - cve.medium_severity: included (matches "cve" package and "@redhat" collection) - // - slsa3.provenance: excluded (matches "slsa3" package exclusion) - // - test.test_data_found: excluded (matches "test.test_data_found" rule exclusion) - // - tasks.required_tasks_found: included (matches "@redhat" collection) - // - tasks.build_task: included (matches "@redhat" collection) - - assert.True(t, result.IncludedRules["cve.high_severity"], "cve.high_severity should be included") - assert.True(t, result.IncludedRules["cve.medium_severity"], "cve.medium_severity should be included") - assert.True(t, result.ExcludedRules["slsa3.provenance"], "slsa3.provenance should be excluded") - assert.True(t, result.ExcludedRules["test.test_data_found"], "test.test_data_found should be excluded") - assert.True(t, result.IncludedRules["tasks.required_tasks_found"], "tasks.required_tasks_found should be included") - assert.True(t, result.IncludedRules["tasks.build_task"], "tasks.build_task should be included") - - // Check package inclusion - assert.True(t, result.IncludedPackages["cve"], "cve package should be included") - assert.True(t, result.IncludedPackages["tasks"], "tasks package should be included") - assert.False(t, result.IncludedPackages["slsa3"], "slsa3 package should not be included (contains only excluded rules)") - assert.False(t, result.IncludedPackages["test"], "test package should not be included (contains only excluded rules)") - - // Verify that packages with only excluded rules are not included - // (they should not appear in IncludedPackages) -} - -func TestUnifiedPostEvaluationFilter(t *testing.T) { - // Test basic filtering functionality - t.Run("Basic Filtering", func(t *testing.T) { - source := ecc.Source{ - Config: &ecc.SourceConfig{ - Include: []string{"cve", "@redhat"}, - Exclude: []string{"test.test_data_found"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(source, configProvider)) - - // Create test results - results := []Result{ - { - Message: "High severity CVE found", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - { - Message: "Test data found", - Metadata: map[string]interface{}{ - metadataCode: "test.test_data_found", - }, - }, - { - Message: "Redhat collection rule", - Metadata: map[string]interface{}{ - metadataCode: "tasks.build_task", - metadataCollections: []string{"redhat"}, - }, - }, - } - - rules := policyRules{ - "cve.high_severity": rule.Info{ - Package: "cve", - Code: "cve.high_severity", - }, - "test.test_data_found": rule.Info{ - Package: "test", - Code: "test.test_data_found", - }, - "tasks.build_task": rule.Info{ - Package: "tasks", - Code: "tasks.build_task", - Collections: []string{"redhat"}, - }, - } - - missingIncludes := map[string]bool{ - "cve": true, - "@redhat": true, - } - - filteredResults, updatedMissingIncludes := filter.FilterResults( - results, rules, "test-target", "", missingIncludes, time.Now()) - - // Should include cve.high_severity and tasks.build_task, exclude test.test_data_found - assert.Len(t, filteredResults, 2) - - // Check that the correct results are included - codes := make([]string, 0, len(filteredResults)) - for _, result := range filteredResults { - if code, ok := result.Metadata[metadataCode].(string); ok { - codes = append(codes, code) - } - } - assert.Contains(t, codes, "cve.high_severity") - assert.Contains(t, codes, "tasks.build_task") - assert.NotContains(t, codes, "test.test_data_found") - - // Check that missing includes were updated - assert.Len(t, updatedMissingIncludes, 0) // All includes should be matched - }) - - // Test pipeline intention filtering - t.Run("Pipeline Intention Filtering", func(t *testing.T) { - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["release"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(source, configProvider)) - - // Create test results with different pipeline intentions - results := []Result{ - { - Message: "Release security check", - Metadata: map[string]interface{}{ - metadataCode: "release.security_check", - }, - }, - { - Message: "Build task", - Metadata: map[string]interface{}{ - metadataCode: "build.build_task", - }, - }, - } - - rules := policyRules{ - "release.security_check": rule.Info{ - Package: "release", - Code: "release.security_check", - PipelineIntention: []string{"release"}, - }, - "build.build_task": rule.Info{ - Package: "build", - Code: "build.build_task", - PipelineIntention: []string{"build"}, - }, - } - - missingIncludes := map[string]bool{ - "*": true, - } - - filteredResults, updatedMissingIncludes := filter.FilterResults( - results, rules, "test-target", "", missingIncludes, time.Now()) - - // Should only include release.security_check (matches pipeline intention) - assert.Len(t, filteredResults, 1) - - // Check that the correct result is included - if len(filteredResults) > 0 { - code := filteredResults[0].Metadata[metadataCode].(string) - assert.Equal(t, "release.security_check", code) - } - - // Check that missing includes were updated - assert.Len(t, updatedMissingIncludes, 0) // Wildcard should be matched - }) - - // Test missing includes handling - t.Run("Missing Includes Handling", func(t *testing.T) { - source := ecc.Source{ - Config: &ecc.SourceConfig{ - Include: []string{"nonexistent.package", "cve"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(source, configProvider)) - - results := []Result{ - { - Message: "CVE found", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - } - - rules := policyRules{ - "cve.high_severity": rule.Info{ - Package: "cve", - Code: "cve.high_severity", - }, - } - - missingIncludes := map[string]bool{ - "nonexistent.package": true, - "cve": true, - } - - filteredResults, updatedMissingIncludes := filter.FilterResults( - results, rules, "test-target", "", missingIncludes, time.Now()) - - // Should include the CVE result - assert.Len(t, filteredResults, 1) - - // Should still have the unmatched include - assert.Len(t, updatedMissingIncludes, 1) - assert.True(t, updatedMissingIncludes["nonexistent.package"]) - assert.False(t, updatedMissingIncludes["cve"]) // Should be removed as it was matched - }) -} - -func TestUnifiedPostEvaluationFilterVsLegacy(t *testing.T) { - // Test that the new comprehensive post-evaluation filter produces - // the same results as the legacy filtering approach - - t.Run("Compare Filtering Results", func(t *testing.T) { - // Create a policy configuration that exercises various filtering scenarios - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["release"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"cve", "@redhat", "security.*"}, - Exclude: []string{"test.test_data_found", "slsa3.provenance"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - // Create test results that cover different scenarios - results := []Result{ - // Included by package include - { - Message: "High severity CVE found", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - // Included by collection include - { - Message: "Redhat collection rule", - Metadata: map[string]interface{}{ - metadataCode: "tasks.build_task", - metadataCollections: []string{"redhat"}, - }, - }, - // Included by wildcard include - { - Message: "Security signature check", - Metadata: map[string]interface{}{ - metadataCode: "security.signature_check", - }, - }, - // Excluded by explicit exclude - { - Message: "Test data found", - Metadata: map[string]interface{}{ - metadataCode: "test.test_data_found", - }, - }, - // Excluded by package exclude - { - Message: "SLSA provenance", - Metadata: map[string]interface{}{ - metadataCode: "slsa3.provenance", - }, - }, - // Excluded by pipeline intention (doesn't match release) - { - Message: "Build task", - Metadata: map[string]interface{}{ - metadataCode: "build.build_task", - }, - }, - // Included by pipeline intention (matches release) - { - Message: "Release security check", - Metadata: map[string]interface{}{ - metadataCode: "release.security_check", - }, - }, - } - - rules := policyRules{ - "cve.high_severity": rule.Info{ - Package: "cve", - Code: "high_severity", - }, - "tasks.build_task": rule.Info{ - Package: "tasks", - Code: "build_task", - Collections: []string{"redhat"}, - }, - "security.signature_check": rule.Info{ - Package: "security", - Code: "signature_check", - PipelineIntention: []string{"release"}, - }, - "test.test_data_found": rule.Info{ - Package: "test", - Code: "test_data_found", - }, - "slsa3.provenance": rule.Info{ - Package: "slsa3", - Code: "provenance", - }, - "build.build_task": rule.Info{ - Package: "build", - Code: "build_task", - PipelineIntention: []string{"build"}, - }, - "release.security_check": rule.Info{ - Package: "release", - Code: "security_check", - PipelineIntention: []string{"release"}, - }, - } - - // Test the new comprehensive filter - newFilter := NewLegacyPostEvaluationFilter(source, configProvider) - newMissingIncludes := map[string]bool{ - "cve": true, - "@redhat": true, - "security.*": true, - } - newFilteredResults, newUpdatedMissingIncludes := newFilter.FilterResults( - results, rules, "test-target", "", newMissingIncludes, time.Now()) - - // Test the legacy approach using the standalone functions - legacyMissingIncludes := map[string]bool{ - "cve": true, - "@redhat": true, - "security.*": true, - } - var legacyFilteredResults []Result - for _, result := range results { - code := ExtractStringFromMetadata(result, metadataCode) - if code == "" { - continue - } - - // Use the legacy IsResultIncluded function - include := &Criteria{ - defaultItems: []string{"cve", "@redhat", "security.*"}, - } - exclude := &Criteria{ - defaultItems: []string{"test.test_data_found", "slsa3.provenance"}, - } - - if LegacyIsResultIncluded(result, "test-target", "", legacyMissingIncludes, include, exclude) { - legacyFilteredResults = append(legacyFilteredResults, result) - } - } - - // Compare the results - t.Logf("New filter results: %d items", len(newFilteredResults)) - t.Logf("Legacy filter results: %d items", len(legacyFilteredResults)) - - // Extract codes for comparison - newCodes := make([]string, 0, len(newFilteredResults)) - for _, result := range newFilteredResults { - if code, ok := result.Metadata[metadataCode].(string); ok { - newCodes = append(newCodes, code) - } - } - - legacyCodes := make([]string, 0, len(legacyFilteredResults)) - for _, result := range legacyFilteredResults { - if code, ok := result.Metadata[metadataCode].(string); ok { - legacyCodes = append(legacyCodes, code) - } - } - - t.Logf("New filter codes: %v", newCodes) - t.Logf("Legacy filter codes: %v", legacyCodes) - - // The results should be the same - assert.ElementsMatch(t, newCodes, legacyCodes, "New and legacy filters should produce the same results") - - // Check missing includes - t.Logf("New missing includes: %v", newUpdatedMissingIncludes) - t.Logf("Legacy missing includes: %v", legacyMissingIncludes) - assert.Equal(t, len(newUpdatedMissingIncludes), len(legacyMissingIncludes), - "Missing includes should be the same") - }) -} - -func TestIncludeExcludePolicyResolver(t *testing.T) { - // Create a config provider - config := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - // Create a source with pipeline intention - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - // Create rules with pipeline intention metadata - rules := policyRules{ - "build.rule1": rule.Info{ - Code: "build.rule1", - Package: "build", - ShortName: "rule1", - PipelineIntention: []string{"build"}, - }, - "deploy.rule2": rule.Info{ - Code: "deploy.rule2", - Package: "deploy", - ShortName: "rule2", - PipelineIntention: []string{"deploy"}, // Different intention - }, - "general.rule3": rule.Info{ - Code: "general.rule3", - Package: "general", - ShortName: "rule3", - // No pipeline intention - }, - } - - // Test ECPolicyResolver (should filter by pipeline intention) - ecResolver := NewECPolicyResolver(source, config) - ecResult := ecResolver.ResolvePolicy(rules, "test") - - // Test IncludeExcludePolicyResolver (should ignore pipeline intention) - includeExcludeResolver := NewIncludeExcludePolicyResolver(source, config) - includeExcludeResult := includeExcludeResolver.ResolvePolicy(rules, "test") - - // Verify that the EC resolver excludes rules with non-matching pipeline intentions - require.False(t, ecResult.IncludedRules["deploy.rule2"], "EC resolver should exclude rule with non-matching pipeline intention") - require.True(t, ecResult.IncludedRules["build.rule1"], "EC resolver should include rule with matching pipeline intention") - require.False(t, ecResult.IncludedRules["general.rule3"], "EC resolver should exclude rule with no pipeline intention when pipeline intentions are specified") - - // Verify that the include-exclude resolver includes all rules regardless of pipeline intention - require.True(t, includeExcludeResult.IncludedRules["build.rule1"], "Include-exclude resolver should include rule with matching pipeline intention") - require.True(t, includeExcludeResult.IncludedRules["deploy.rule2"], "Include-exclude resolver should include rule with non-matching pipeline intention") - require.True(t, includeExcludeResult.IncludedRules["general.rule3"], "Include-exclude resolver should include rule with no pipeline intention") - - // Verify that both resolvers include the same packages - require.True(t, ecResult.IncludedPackages["build"], "EC resolver should include build package") - require.False(t, ecResult.IncludedPackages["general"], "EC resolver should exclude general package (no matching pipeline intention)") - require.True(t, includeExcludeResult.IncludedPackages["build"], "Include-exclude resolver should include build package") - require.True(t, includeExcludeResult.IncludedPackages["deploy"], "Include-exclude resolver should include deploy package") - require.True(t, includeExcludeResult.IncludedPackages["general"], "Include-exclude resolver should include general package") -} - -func TestPackageInclusionLogic(t *testing.T) { - tests := []struct { - name string - rules policyRules - config *ecc.EnterpriseContractPolicyConfiguration - expectedIncluded map[string]bool - description string - }{ - { - name: "Package with only included rules", - rules: policyRules{ - "security.signature_check": rule.Info{ - Code: "security.signature_check", - Package: "security", - }, - "security.vulnerability_scan": rule.Info{ - Code: "security.vulnerability_scan", - Package: "security", - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{ - Include: []string{"security"}, - }, - expectedIncluded: map[string]bool{ - "security": true, - }, - description: "Package with only included rules should be included", - }, - { - name: "Package with only excluded rules", - rules: policyRules{ - "test.test_data_found": rule.Info{ - Code: "test.test_data_found", - Package: "test", - }, - "test.debug_info": rule.Info{ - Code: "test.debug_info", - Package: "test", - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{ - Exclude: []string{"test"}, - }, - expectedIncluded: map[string]bool{}, - description: "Package with only excluded rules should not be included", - }, - { - name: "Package with mixed included and excluded rules", - rules: policyRules{ - "security.signature_check": rule.Info{ - Code: "security.signature_check", - Package: "security", - }, - "security.debug_info": rule.Info{ - Code: "security.debug_info", - Package: "security", - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{ - Include: []string{"security.signature_check"}, - Exclude: []string{"security.debug_info"}, - }, - expectedIncluded: map[string]bool{ - "security": true, - }, - description: "Package with mixed rules should be included (has at least one included rule)", - }, - { - name: "Package with no explicit include/exclude", - rules: policyRules{ - "general.validation": rule.Info{ - Code: "general.validation", - Package: "general", - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{ - Include: []string{"*"}, - }, - expectedIncluded: map[string]bool{ - "general": true, - }, - description: "Package with no explicit criteria but wildcard include should be included", - }, - { - name: "Multiple packages with different scenarios", - rules: policyRules{ - "security.signature_check": rule.Info{ - Code: "security.signature_check", - Package: "security", - }, - "test.debug_info": rule.Info{ - Code: "test.debug_info", - Package: "test", - }, - "cve.high_severity": rule.Info{ - Code: "cve.high_severity", - Package: "cve", - }, - }, - config: &ecc.EnterpriseContractPolicyConfiguration{ - Include: []string{"security", "cve"}, - Exclude: []string{"test"}, - }, - expectedIncluded: map[string]bool{ - "security": true, - "cve": true, - }, - description: "Multiple packages: included packages should be included, excluded-only packages should not", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - config := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - source := ecc.Source{ - Config: &ecc.SourceConfig{ - Include: tc.config.Include, - Exclude: tc.config.Exclude, - }, - } - - resolver := NewIncludeExcludePolicyResolver(source, config) - result := resolver.ResolvePolicy(tc.rules, "test") - - // Verify included packages - for pkg, expected := range tc.expectedIncluded { - if expected { - assert.True(t, result.IncludedPackages[pkg], - "Package %s should be included: %s", pkg, tc.description) - } else { - assert.False(t, result.IncludedPackages[pkg], - "Package %s should not be included: %s", pkg, tc.description) - } - } - - // Verify that packages not in expected lists are not included - for pkg := range result.IncludedPackages { - if _, expected := tc.expectedIncluded[pkg]; !expected { - t.Errorf("Unexpected included package: %s", pkg) - } - } - }) - } -} - -func TestMissingIncludesFilterUpdate(t *testing.T) { - tests := []struct { - name string - initialMissing map[string]bool - filteredResults []Result - expectedMissing map[string]bool - description string - }{ - { - name: "All includes matched", - initialMissing: map[string]bool{ - "cve": true, - "@redhat": true, - "security.*": true, - }, - filteredResults: []Result{ - { - Message: "CVE found", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - { - Message: "Redhat collection rule", - Metadata: map[string]interface{}{ - metadataCode: "tasks.build_task", - metadataCollections: []string{"redhat"}, - }, - }, - { - Message: "Security check", - Metadata: map[string]interface{}{ - metadataCode: "security.signature_check", - }, - }, - }, - expectedMissing: map[string]bool{}, - description: "Tests that all include criteria are removed when matched by results", - }, - { - name: "Partial includes matched", - initialMissing: map[string]bool{ - "cve": true, - "@redhat": true, - "nonexistent.*": true, - }, - filteredResults: []Result{ - { - Message: "CVE found", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - }, - expectedMissing: map[string]bool{ - "@redhat": true, - "nonexistent.*": true, - }, - description: "Tests that only matched include criteria are removed", - }, - { - name: "No includes matched", - initialMissing: map[string]bool{ - "@security": true, - "release.*": true, - }, - filteredResults: []Result{ - { - Message: "Unrelated rule", - Metadata: map[string]interface{}{ - metadataCode: "test.unrelated", - }, - }, - }, - expectedMissing: map[string]bool{ - "@security": true, - "release.*": true, - }, - description: "Tests that no include criteria are removed when none match", - }, - { - name: "Wildcard matching", - initialMissing: map[string]bool{ - "*": true, - }, - filteredResults: []Result{ - { - Message: "Any rule", - Metadata: map[string]interface{}{ - metadataCode: "any.package.rule", - }, - }, - }, - expectedMissing: map[string]bool{}, - description: "Tests that wildcard includes are matched by any result", - }, - { - name: "Collection matching", - initialMissing: map[string]bool{ - "@redhat": true, - }, - filteredResults: []Result{ - { - Message: "Redhat collection rule", - Metadata: map[string]interface{}{ - metadataCode: "tasks.build_task", - metadataCollections: []string{"redhat"}, - }, - }, - }, - expectedMissing: map[string]bool{}, - description: "Tests that collection includes are matched by results with matching collections", - }, - { - name: "Package matching", - initialMissing: map[string]bool{ - "cve": true, - }, - filteredResults: []Result{ - { - Message: "CVE rule", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - }, - expectedMissing: map[string]bool{}, - description: "Tests that package includes are matched by results from that package", - }, - { - name: "Rule-specific matching", - initialMissing: map[string]bool{ - "cve.high_severity": true, - }, - filteredResults: []Result{ - { - Message: "Specific CVE rule", - Metadata: map[string]interface{}{ - metadataCode: "cve.high_severity", - }, - }, - }, - expectedMissing: map[string]bool{}, - description: "Tests that rule-specific includes are matched by the exact rule", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create a copy of the initial missing includes - missingIncludes := make(map[string]bool) - for k, v := range tt.initialMissing { - missingIncludes[k] = v - } - - // Simulate the filter update logic - for include := range missingIncludes { - matched := false - for _, result := range tt.filteredResults { - matchers := LegacyMakeMatchers(result) - for _, matcher := range matchers { - if matcher == include { - matched = true - break - } - } - if matched { - break - } - } - if matched { - delete(missingIncludes, include) - } - } - - // Verify the expected missing includes - for expectedItem := range tt.expectedMissing { - assert.True(t, missingIncludes[expectedItem], - "Expected item '%s' should remain in missingIncludes", expectedItem) - } - - // Verify no unexpected items remain - for actualItem := range missingIncludes { - assert.True(t, tt.expectedMissing[actualItem], - "Unexpected item '%s' remains in missingIncludes", actualItem) - } - - t.Logf("Test case: %s", tt.description) - t.Logf("Initial missingIncludes: %v", tt.initialMissing) - t.Logf("Final missingIncludes: %v", missingIncludes) - }) - } -} - -func TestConftestEvaluator_FilterType_ECPolicy(t *testing.T) { - // Test that ec-policy filter type uses ECPolicyResolver - sourceConfig := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - ctx := setupTestContext(nil, nil) - configProvider, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - policySources := []source.PolicySource{testPolicySource{}} - t.Logf("About to call NewConftestEvaluatorWithFilterType") - evaluator, err := NewConftestEvaluatorWithFilterType(ctx, policySources, configProvider, sourceConfig, "ec-policy") - t.Logf("NewConftestEvaluatorWithFilterType returned: evaluator=%v, err=%v", evaluator, err) - if err != nil { - t.Logf("Error creating evaluator: %v", err) - } - assert.NoError(t, err) - assert.NotNil(t, evaluator, "evaluator should not be nil") - - t.Logf("Evaluator type: %T", evaluator) - conftestEval, ok := evaluator.(conftestEvaluator) - t.Logf("Type assertion result: ok=%v, conftestEval=%v", ok, conftestEval) - assert.True(t, ok, "evaluator should be conftestEvaluator") - - // Should use ECPolicyResolver which supports pipeline intentions - _, isECPolicyResolver := conftestEval.policyResolver.(*ECPolicyResolver) - assert.True(t, isECPolicyResolver, "should use ECPolicyResolver for ec-policy filter type") -} - -func TestConftestEvaluator_FilterType_IncludeExclude(t *testing.T) { - // Test that include-exclude filter type uses IncludeExcludePolicyResolver - sourceConfig := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - ctx := setupTestContext(nil, nil) - configProvider, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - policySources := []source.PolicySource{testPolicySource{}} - evaluator, err := NewConftestEvaluatorWithFilterType(ctx, policySources, configProvider, sourceConfig, "include-exclude") - assert.NoError(t, err) - - conftestEval, ok := evaluator.(conftestEvaluator) - assert.True(t, ok, "evaluator should be conftestEvaluator") - - // Should use IncludeExcludePolicyResolver which ignores pipeline intentions - _, isIncludeExcludeResolver := conftestEval.policyResolver.(*IncludeExcludePolicyResolver) - assert.True(t, isIncludeExcludeResolver, "should use IncludeExcludePolicyResolver for include-exclude filter type") -} - -func TestConftestEvaluator_FilterType_Default(t *testing.T) { - // Test that default filter type uses IncludeExcludePolicyResolver - sourceConfig := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - ctx := setupTestContext(nil, nil) - configProvider, err := policy.NewOfflinePolicy(ctx, policy.Now) - assert.NoError(t, err) - - policySources := []source.PolicySource{testPolicySource{}} - evaluator, err := NewConftestEvaluatorWithFilterType(ctx, policySources, configProvider, sourceConfig, "unknown-type") - assert.NoError(t, err) - - conftestEval, ok := evaluator.(conftestEvaluator) - assert.True(t, ok, "evaluator should be conftestEvaluator") - - // Should default to IncludeExcludePolicyResolver - _, isIncludeExcludeResolver := conftestEval.policyResolver.(*IncludeExcludePolicyResolver) - assert.True(t, isIncludeExcludeResolver, "should default to IncludeExcludePolicyResolver for unknown filter type") -} - -////////////////////////////////////////////////////////////////////////////// -// Tests for Rego package with multiple rules - some with pipeline_intention, some without -////////////////////////////////////////////////////////////////////////////// - -func TestRegoPackageWithMixedPipelineIntentions(t *testing.T) { - // Test case: A Rego package with multiple rules where some have pipeline_intention - // defined and some don't. Only rules with the defined pipeline_intention should be included. - - t.Run("ECPolicyResolver with build intention", func(t *testing.T) { - // Create a source with build pipeline intention - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - // Create rules representing a Rego package with mixed pipeline intentions - rules := policyRules{ - // Rules with build pipeline intention - should be included - "security.build_security_check": rule.Info{ - Package: "security", - Code: "build_security_check", - PipelineIntention: []string{"build"}, - }, - "security.build_vulnerability_scan": rule.Info{ - Package: "security", - Code: "build_vulnerability_scan", - PipelineIntention: []string{"build"}, - }, - // Rules with different pipeline intention - should be excluded - "security.deploy_security_check": rule.Info{ - Package: "security", - Code: "deploy_security_check", - PipelineIntention: []string{"deploy"}, - }, - "security.release_security_check": rule.Info{ - Package: "security", - Code: "release_security_check", - PipelineIntention: []string{"release"}, - }, - // Rules with no pipeline intention - should be excluded when pipeline intention is specified - "security.general_security_check": rule.Info{ - Package: "security", - Code: "general_security_check", - // No pipeline intention - }, - "security.basic_validation": rule.Info{ - Package: "security", - Code: "basic_validation", - // No pipeline intention - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // Verify that only rules with build pipeline intention are included - assert.True(t, result.IncludedRules["build_security_check"], - "build_security_check should be included (matches build intention)") - assert.True(t, result.IncludedRules["build_vulnerability_scan"], - "build_vulnerability_scan should be included (matches build intention)") - - // Verify that rules with different pipeline intentions are excluded - assert.False(t, result.IncludedRules["deploy_security_check"], - "deploy_security_check should be excluded (doesn't match build intention)") - assert.False(t, result.IncludedRules["release_security_check"], - "release_security_check should be excluded (doesn't match build intention)") - - // Verify that rules with no pipeline intention are excluded - assert.False(t, result.IncludedRules["general_security_check"], - "general_security_check should be excluded (no pipeline intention)") - assert.False(t, result.IncludedRules["basic_validation"], - "basic_validation should be excluded (no pipeline intention)") - - // Verify that the security package is included (has at least one included rule) - assert.True(t, result.IncludedPackages["security"], - "security package should be included (has included rules)") - - // Verify explanations - assert.Contains(t, result.Explanations["build_security_check"], "included") - assert.Contains(t, result.Explanations["deploy_security_check"], "pipeline intention") - assert.Contains(t, result.Explanations["general_security_check"], "pipeline intention") - }) - - t.Run("ECPolicyResolver with no pipeline intention specified", func(t *testing.T) { - // Create a source with no pipeline intention - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - // Same rules as above - rules := policyRules{ - "security.build_security_check": rule.Info{ - Package: "security", - Code: "build_security_check", - PipelineIntention: []string{"build"}, - }, - "security.deploy_security_check": rule.Info{ - Package: "security", - Code: "deploy_security_check", - PipelineIntention: []string{"deploy"}, - }, - "security.general_security_check": rule.Info{ - Package: "security", - Code: "general_security_check", - // No pipeline intention - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // When no pipeline intention is specified, only rules with no pipeline intention should be included - assert.False(t, result.IncludedRules["build_security_check"], - "build_security_check should be excluded (has pipeline intention)") - assert.False(t, result.IncludedRules["deploy_security_check"], - "deploy_security_check should be excluded (has pipeline intention)") - assert.True(t, result.IncludedRules["general_security_check"], - "general_security_check should be included (no pipeline intention)") - - // Verify that the security package is included (has at least one included rule) - assert.True(t, result.IncludedPackages["security"], - "security package should be included (has included rules)") - }) - - t.Run("IncludeExcludePolicyResolver ignores pipeline intentions", func(t *testing.T) { - // Create a source with build pipeline intention - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewIncludeExcludePolicyResolver(source, configProvider) - - // Same rules as above - rules := policyRules{ - "security.build_security_check": rule.Info{ - Package: "security", - Code: "build_security_check", - PipelineIntention: []string{"build"}, - }, - "security.deploy_security_check": rule.Info{ - Package: "security", - Code: "deploy_security_check", - PipelineIntention: []string{"deploy"}, - }, - "security.general_security_check": rule.Info{ - Package: "security", - Code: "general_security_check", - // No pipeline intention - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // IncludeExcludePolicyResolver should include all rules regardless of pipeline intention - assert.True(t, result.IncludedRules["build_security_check"], - "build_security_check should be included (include-exclude ignores pipeline intention)") - assert.True(t, result.IncludedRules["deploy_security_check"], - "deploy_security_check should be included (include-exclude ignores pipeline intention)") - assert.True(t, result.IncludedRules["general_security_check"], - "general_security_check should be included (include-exclude ignores pipeline intention)") - - // Verify that the security package is included - assert.True(t, result.IncludedPackages["security"], - "security package should be included") - }) -} - -func TestMultiplePackagesWithMixedPipelineIntentions(t *testing.T) { - // Test case: Multiple Rego packages with mixed pipeline intentions - // This demonstrates how the filtering works across different packages - - t.Run("Multiple packages with build intention", func(t *testing.T) { - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - // Create rules across multiple packages - rules := policyRules{ - // Security package - some rules with build intention - "security.build_security_check": rule.Info{ - Package: "security", - Code: "build_security_check", - PipelineIntention: []string{"build"}, - }, - "security.deploy_security_check": rule.Info{ - Package: "security", - Code: "deploy_security_check", - PipelineIntention: []string{"deploy"}, - }, - "security.general_check": rule.Info{ - Package: "security", - Code: "general_check", - // No pipeline intention - }, - - // CVE package - some rules with build intention - "cve.build_vulnerability_scan": rule.Info{ - Package: "cve", - Code: "build_vulnerability_scan", - PipelineIntention: []string{"build"}, - }, - "cve.release_vulnerability_scan": rule.Info{ - Package: "cve", - Code: "release_vulnerability_scan", - PipelineIntention: []string{"release"}, - }, - "cve.general_scan": rule.Info{ - Package: "cve", - Code: "general_scan", - // No pipeline intention - }, - - // Tasks package - all rules with build intention - "tasks.build_task": rule.Info{ - Package: "tasks", - Code: "build_task", - PipelineIntention: []string{"build"}, - }, - "tasks.build_validation": rule.Info{ - Package: "tasks", - Code: "build_validation", - PipelineIntention: []string{"build"}, - }, - - // General package - no rules with pipeline intention - "general.basic_check": rule.Info{ - Package: "general", - Code: "basic_check", - // No pipeline intention - }, - "general.validation": rule.Info{ - Package: "general", - Code: "validation", - // No pipeline intention - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // Verify included rules (only those with build intention) - assert.True(t, result.IncludedRules["build_security_check"], - "build_security_check should be included") - assert.True(t, result.IncludedRules["build_vulnerability_scan"], - "build_vulnerability_scan should be included") - assert.True(t, result.IncludedRules["build_task"], - "build_task should be included") - assert.True(t, result.IncludedRules["build_validation"], - "build_validation should be included") - - // Verify excluded rules (different or no pipeline intention) - assert.False(t, result.IncludedRules["deploy_security_check"], - "deploy_security_check should be excluded") - assert.False(t, result.IncludedRules["general_check"], - "general_check should be excluded") - assert.False(t, result.IncludedRules["release_vulnerability_scan"], - "release_vulnerability_scan should be excluded") - assert.False(t, result.IncludedRules["general_scan"], - "general_scan should be excluded") - assert.False(t, result.IncludedRules["basic_check"], - "basic_check should be excluded") - assert.False(t, result.IncludedRules["validation"], - "validation should be excluded") - - // Verify package inclusion - assert.True(t, result.IncludedPackages["security"], - "security package should be included (has included rules)") - assert.True(t, result.IncludedPackages["cve"], - "cve package should be included (has included rules)") - assert.True(t, result.IncludedPackages["tasks"], - "tasks package should be included (has included rules)") - assert.False(t, result.IncludedPackages["general"], - "general package should be excluded (no included rules)") - - // Verify explanations contain appropriate information - // For included rules, explanations should mention "included" - // For excluded rules, explanations should mention "pipeline intention" - for ruleName, explanation := range result.Explanations { - if result.IncludedRules[ruleName] { - assert.Contains(t, explanation, "included", - "Explanation for included rule %s should mention 'included'", ruleName) - } else if strings.Contains(explanation, "pipeline intention") { - assert.Contains(t, explanation, "pipeline intention", - "Explanation for excluded rule %s should mention 'pipeline intention'", ruleName) - } - } - }) -} - -func TestPipelineIntentionWithMultipleValues(t *testing.T) { - // Test case: Rules with multiple pipeline intention values - // This demonstrates how rules with multiple intentions are handled - - t.Run("Rule with multiple pipeline intentions", func(t *testing.T) { - source := ecc.Source{ - RuleData: &extv1.JSON{Raw: json.RawMessage(`{"pipeline_intention":["build"]}`)}, - Config: &ecc.SourceConfig{ - Include: []string{"*"}, - }, - } - - configProvider := &simpleConfigProvider{ - effectiveTime: time.Now(), - } - - resolver := NewECPolicyResolver(source, configProvider) - - rules := policyRules{ - // Rule with multiple pipeline intentions including build - "security.build_and_deploy_check": rule.Info{ - Package: "security", - Code: "build_and_deploy_check", - PipelineIntention: []string{"build", "deploy"}, - }, - // Rule with multiple pipeline intentions not including build - "security.deploy_and_release_check": rule.Info{ - Package: "security", - Code: "deploy_and_release_check", - PipelineIntention: []string{"deploy", "release"}, - }, - // Rule with single build intention - "security.build_only_check": rule.Info{ - Package: "security", - Code: "build_only_check", - PipelineIntention: []string{"build"}, - }, - } - - result := resolver.ResolvePolicy(rules, "test-target") - - // Rules with build in their pipeline intentions should be included - assert.True(t, result.IncludedRules["build_and_deploy_check"], - "build_and_deploy_check should be included (includes build intention)") - assert.True(t, result.IncludedRules["build_only_check"], - "build_only_check should be included (build intention)") - - // Rules without build in their pipeline intentions should be excluded - assert.False(t, result.IncludedRules["deploy_and_release_check"], - "deploy_and_release_check should be excluded (no build intention)") - - // Verify package inclusion - assert.True(t, result.IncludedPackages["security"], - "security package should be included (has included rules)") - }) -} diff --git a/internal/evaluator/opa_evaluator_test.go b/internal/evaluator/opa_evaluator_test.go deleted file mode 100644 index 278b42fd4..000000000 --- a/internal/evaluator/opa_evaluator_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -package evaluator - -import ( - "context" - "os" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" -) - -// TestNewOPAEvaluator tests the constructor NewOPAEvaluator. -func TestNewOPAEvaluator(t *testing.T) { - evaluator, err := NewOPAEvaluator() - assert.NoError(t, err, "Expected no error from NewOPAEvaluator") - assert.Equal(t, evaluator, opaEvaluator{}) -} - -func TestEvaluate(t *testing.T) { - opaEval := opaEvaluator{} - - outcomes, err := opaEval.Evaluate(context.Background(), EvaluationTarget{}) - assert.NoError(t, err, "Expected no error from Evaluate") - assert.Equal(t, []Outcome{}, outcomes) -} - -// Test Destroy method of opaEvaluator. -func TestDestroy(t *testing.T) { - // Setup an in-memory filesystem - fs := afero.NewMemMapFs() - workDir := "/tmp/workdir" - - // Define test cases - testCases := []struct { - name string - workDir string - EC_DEBUG bool - expectRemove bool - }{ - { - name: "Empty workDir, EC_DEBUG not set", - workDir: "", - EC_DEBUG: false, - expectRemove: false, - }, - { - name: "Non-empty workDir, EC_DEBUG not set", - workDir: workDir, - EC_DEBUG: false, - expectRemove: true, - }, - { - name: "Non-empty workDir, EC_DEBUG set", - workDir: workDir, - EC_DEBUG: true, - expectRemove: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Set up the environment - if tc.workDir != "" { - err := fs.MkdirAll(tc.workDir, 0755) - assert.NoError(t, err, "Failed to create workDir in in-memory filesystem") - } - - if tc.EC_DEBUG { - os.Setenv("EC_DEBUG", "true") - } else { - os.Unsetenv("EC_DEBUG") - } - - // Initialize the evaluator - opaEval := opaEvaluator{ - workDir: tc.workDir, - fs: fs, - } - - // Call Destroy - opaEval.Destroy() - - // Verify the result - exists, err := afero.DirExists(fs, tc.workDir) - assert.NoError(t, err, "Error checking if workDir exists after Destroy") - - if tc.expectRemove { - assert.False(t, exists, "workDir should be removed") - } else { - assert.True(t, exists, "workDir should not be removed") - } - - // Clean up for next test - _ = fs.RemoveAll(tc.workDir) - os.Unsetenv("EC_DEBUG") - }) - } -} - -// TestCapabilitiesPath tests the CapabilitiesPath method of opaEvaluator. -func TestCapabilitiesPath(t *testing.T) { - // Define test cases - testCases := []struct { - name string - workDir string - expected string - }{ - { - name: "Non-empty workDir", - workDir: "/tmp/workdir", - expected: "/tmp/workdir/capabilities.json", - }, - { - name: "Root workDir", - workDir: "/", - expected: "/capabilities.json", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Create a mock filesystem (though not strictly needed for this test) - fs := afero.NewMemMapFs() - - // Initialize the evaluator with test data - opaEval := opaEvaluator{ - workDir: tc.workDir, - fs: fs, - } - - // Call CapabilitiesPath - result := opaEval.CapabilitiesPath() - - // Verify the result - assert.Equal(t, tc.expected, result, "CapabilitiesPath should return the expected path") - }) - } -} diff --git a/internal/utils/templates_test.go b/internal/utils/templates_test.go deleted file mode 100644 index c29d9d611..000000000 --- a/internal/utils/templates_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright The Conforma Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -//go:build unit - -package utils - -import ( - "bytes" - "embed" - _ "embed" - "errors" - "testing" - - "github.com/stretchr/testify/assert" -) - -//go:embed test_templates/*.tmpl -var testTemplatesFS embed.FS - -func TestSetupTemplate(t *testing.T) { - tmpl, err := SetupTemplate(testTemplatesFS) - assert.NoError(t, err) - - tests := []struct { - main string - expected string - input map[string]string - }{ - { - input: map[string]string{"name": "friend"}, - expected: "✓ Hello and greetings, friend.\n\n", - }, - { - main: "main.tmpl", - expected: "✓ Hello and greetings, spam.\n\n", - input: map[string]string{"name": "spam"}, - }, - { - main: "_name.tmpl", - expected: "and hola, amigo.\n", - input: map[string]string{"greeting": "hola", "name": "amigo"}, - }, - } - for _, tt := range tests { - var buf bytes.Buffer - var err error - if tt.main == "" { - err = tmpl.Execute(&buf, tt.input) - assert.NoError(t, err) - } else { - err = tmpl.ExecuteTemplate(&buf, tt.main, tt.input) - assert.NoError(t, err) - } - assert.Equal(t, tt.expected, buf.String()) - } -} - -func TestTemplateHelpers(t *testing.T) { - tmpl, err := SetupTemplate(testTemplatesFS) - assert.NoError(t, err) - - tests := []struct { - main string - expected string - expectedErr error - input map[string]interface{} - }{ - { - main: "helpers.tmpl", - input: map[string]interface{}{ - "colorText": map[string]interface{}{ - "color": "success", - "str": "color test", - }, - "indicator": map[string]interface{}{ - "color": "warning", - }, - "colorIndicator": map[string]interface{}{ - "color": "violation", - }, - "wrap": map[string]interface{}{ - "width": 3, - "s": "wrapped string", - }, - "indent": map[string]interface{}{ - "n": 3, - "s": "indentation test", - }, - "indentWrap": map[string]interface{}{ - "n": 3, - "width": 10, - "s": "indent wrapped test", - }, - "toMap": map[string]interface{}{ - "k1": "key1", - "v1": "value1", - "k2": "key2", - "v2": "value2", - }, - "isString": map[string]interface{}{ - "value": "str", - }, - "joinStrSlice": map[string]interface{}{ - "slice": []interface{}{"one", "two", "three"}, - "sep": ",", - }, - }, - expected: "\x1b[32mcolor test\x1b[0m\n›indicator\n\x1b[31m✕\x1b[0mcolorIndicator\nwrapped\nstring\n indentation test\n indent\n wrapped\n test\nkey1: value1\nkey2: value2\n\ntrue\none,two,three", - expectedErr: nil, - }, - { - main: "helpers.tmpl", - input: map[string]interface{}{ - "colorText": map[string]interface{}{ - "color": "another", - "str": "color test", - }, - "isString": map[string]interface{}{ - "value": 2, - }, - }, - expected: "color test\nfalse\n", - expectedErr: nil, - }, - { - main: "helpers.tmpl", - input: map[string]interface{}{ - "joinStrSlice": map[string]interface{}{ - "slice": []interface{}{1, 2, 3}, - "sep": ",", - }, - }, - expected: "", - expectedErr: errors.New("joinStrSlice argument must be a slice of strings"), - }, - { - main: "helpers.tmpl", - input: map[string]interface{}{ - "toMap": map[string]interface{}{ - "k1": 1, - "v1": "value1", - "k2": 2, - "v2": "value2", - }, - }, - expected: "", - expectedErr: errors.New("toMap keys must be strings"), - }, - } - for _, tt := range tests { - var buf bytes.Buffer - - SetColorEnabled(false, true) - err := tmpl.ExecuteTemplate(&buf, tt.main, tt.input) - - if tt.expectedErr != nil { - assert.ErrorContains(t, err, tt.expectedErr.Error()) - } else { - assert.Nil(t, err) - } - assert.Equal(t, tt.expected, buf.String()) - } -}