diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9ee25c073..ec74617dd 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -31,25 +31,89 @@ jobs: echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" exit 0 fi + integration-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Group 1: Scan creation and management tests (largest group, uses getRootProject) + - group: scan-create + name: "Scan Creation" + pattern: "^Test(CreateScan|ScanCreate|Scan_|ScanTypeApi|ValidateScanTypes|ScanGenerating|ScanWithPolicy|ScanList|ScanTimeout|ScanASCA|ExecuteASCAScan)" + # Group 2: Scan operations - E2E, incremental, cancel, filters + - group: scan-ops + name: "Scan Operations" + pattern: "^Test(ScansE2E|ScansUpdate|FastScan|LightQueries|RecommendedExclusions|IncrementalScan|BranchPrimary|CancelScan|ScanCreateInclude|ScanCreateWith|ScanCreateIgnore|ScanCreateFilter|ScanWorkflow|ScanLogs|InvalidSource|ScanShow|RequiredScan|ScaResolver|BrokenLink|PartialScan|FailedScan|RunKics|RunSca|ScanGLReport)" + # Group 3: Results and reports tests (uses getRootScan - read-only) + - group: results + name: "Results & Reports" + pattern: "^Test(Result|CodeBashing|RiskManagement)" + # Group 4: PR decoration tests (uses getRootScan) + - group: pr-decoration + name: "PR Decoration" + pattern: "^TestPR" + # Group 5: Project management tests + - group: projects + name: "Projects" + pattern: "^Test(Project|CreateEmpty|CreateAlready)" + # Group 6: Predicates and BFL tests (uses getRootScan) + - group: predicates + name: "Predicates & BFL" + pattern: "^Test(Predicate|Bfl|Triage)" + # Group 7: Container-specific tests (from scan_test.go and container files) + - group: containers + name: "Container Tests" + pattern: "^Test(Container|EmptyFolder|ImagesValidation|EdgeCase)" + # Group 8: Realtime scanning tests (IaC, OSS, Secrets, ASCA) and engine tests + - group: realtime + name: "Realtime Scanning" + pattern: "^Test(IacRealtime|OssRealtime|Secrets_Realtime|Containers_Realtime|EngineNameResolution)" + # Group 9: Auth, config, tenant, root tests + - group: auth-config + name: "Auth & Config" + pattern: "^Test(Auth|Config|Tenant|Root|Log|SetLog|FeatureFlag|RateLimit|FailProxy)" + # Group 10: SCM user count tests (GitHub, GitLab, Azure, Bitbucket) + - group: user-count + name: "User Count" + pattern: "^Test(GitHub|GitLab|Azure|Bitbucket)(UserCount|Count)" + # Group 11: Misc tests (chat, import, telemetry, remediation, pre-commit, pre-receive) + - group: misc + name: "Miscellaneous" + pattern: "^Test(Chat|Import|Learn|Telemetry|Mask|Sca?Remediation|Kics?Remediation|PreCommit|Pre_?Receive|Pre_Receive|Get)" + name: Integration - ${{ matrix.name }} steps: - name: Checkout the repository uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + - name: Set up Go version uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 with: go-version-file: go.mod + - run: go version + - name: Go Build run: go build -o ./bin/cx ./cmd - - name: Install gocovmerge - run: go install github.com/wadey/gocovmerge@latest - - name: Install pre-commit + + - name: Start Squid proxy run: | - pip install pre-commit - pre-commit install - - name: Go Integration test + docker run \ + --name squid \ + -d \ + -p 3128:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta + + - name: Download ScaResolver + run: | + wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz + tar -xzvf ScaResolver-linux64.tar.gz -C /tmp + rm -rf ScaResolver-linux64.tar.gz + + - name: Run ${{ matrix.name }} tests shell: bash env: CX_BASE_URI: ${{ secrets.CX_BASE_URI }} @@ -92,9 +156,82 @@ jobs: PR_BITBUCKET_REPO_NAME: "cliIntegrationTest" PR_BITBUCKET_ID: 1 run: | - sudo chmod +x ./internal/commands/.scripts/integration_up.sh ./internal/commands/.scripts/integration_down.sh - ./internal/commands/.scripts/integration_up.sh - ./internal/commands/.scripts/integration_down.sh + set -o pipefail # Preserve exit code from go test, not tee + echo "Running test group: ${{ matrix.name }}" + echo "Pattern: ${{ matrix.pattern }}" + + # Run tests matching the pattern with coverage + go test \ + -tags integration \ + -v \ + -timeout 90m \ + -run "${{ matrix.pattern }}" \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover-${{ matrix.group }}.out \ + github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output_${{ matrix.group }}.log + + # Check for failed tests and rerun them once + if grep -q "^--- FAIL:" test_output_${{ matrix.group }}.log; then + echo "Some tests failed, attempting rerun..." + FAILED_TESTS=$(grep -E "^--- FAIL: " test_output_${{ matrix.group }}.log | awk '{print $3}' | tr '\n' '|' | sed 's/|$//') + if [ -n "$FAILED_TESTS" ]; then + echo "Rerunning failed tests: $FAILED_TESTS" + go test \ + -tags integration \ + -v \ + -timeout 30m \ + -run "^($FAILED_TESTS)$" \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover-${{ matrix.group }}-rerun.out \ + github.com/checkmarx/ast-cli/test/integration + fi + fi + + - name: Stop Squid proxy + if: always() + run: docker stop squid || true && docker rm squid || true + + - name: Upload coverage artifact + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: coverage-${{ matrix.group }} + path: cover-*.out + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: test-logs-${{ matrix.group }} + path: test_output_*.log + + merge-coverage: + runs-on: ubuntu-latest + needs: integration-tests + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Download all coverage artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4 + with: + pattern: coverage-* + path: coverage-files + merge-multiple: true + + - name: Merge coverage files + run: | + echo "Merging coverage files..." + ls -la coverage-files/ + gocovmerge coverage-files/cover-*.out > cover.out + go tool cover -html=cover.out -o coverage.html - name: Coverage report uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 @@ -115,6 +252,18 @@ jobs: echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" exit 0 fi + + - name: Run cleandata to clean up projects + env: + CX_BASE_URI: ${{ secrets.CX_BASE_URI }} + CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} + CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }} + CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }} + CX_APIKEY: ${{ secrets.CX_APIKEY }} + CX_TENANT: ${{ secrets.CX_TENANT }} + run: | + go test -v github.com/checkmarx/ast-cli/test/cleandata || true + lint: name: lint runs-on: ubuntu-latest diff --git a/test/integration/project_test.go b/test/integration/project_test.go index cf6e73d19..21e143ebe 100644 --- a/test/integration/project_test.go +++ b/test/integration/project_test.go @@ -368,8 +368,9 @@ func TestCreateProjectWithSSHKey(t *testing.T) { } func TestProjectShow_MainBranch_Exist(t *testing.T) { - - projectID, projectName := createProject(t, Tags, Groups) + // Use unique project name to avoid conflicts with other tests using getRootProject() + projectName := GenerateRandomProjectNameForScan() + projectID, _ := createNewProject(t, Tags, Groups, projectName) defer deleteProject(t, projectID) args := []string{ diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 6e0128832..44c2fdbf2 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -2708,8 +2708,8 @@ func TestCreateScanWithExistingProjectAnd_AssignApplication(t *testing.T) { flag(params.SourcesFlag), "data/sources-gitignore.zip", flag(params.ApplicationName), "cli-application", } - err, _ := executeCommand(t, args...) - assert.NilError(t, err, "Project should be assigned to application") + // Use 10-minute timeout instead of default 5 minutes to handle slow scans + _ = executeCmdWithTimeOutNilAssertion(t, "Project should be assigned to application", 10*time.Minute, args...) } func TestCreateScanWithNewProjectName_Assign_Groups(t *testing.T) {