From 885a0b56bd0d5be50db0088bdfdcb41da11f461a Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Tue, 11 Mar 2025 19:07:32 -0400 Subject: [PATCH 1/4] Updates parrot to latest image --- lib/charts/parrot/values.yaml | 2 +- lib/docker/test_env/parrot.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/charts/parrot/values.yaml b/lib/charts/parrot/values.yaml index 5c99f3039..88ad1aac0 100644 --- a/lib/charts/parrot/values.yaml +++ b/lib/charts/parrot/values.yaml @@ -2,7 +2,7 @@ replicas: 1 image: repository: kalverra/parrot - version: v0.6.1 + version: v0.6.2 imagePullPolicy: IfNotPresent logLevel: trace diff --git a/lib/docker/test_env/parrot.go b/lib/docker/test_env/parrot.go index aac648c71..1cf9b8251 100644 --- a/lib/docker/test_env/parrot.go +++ b/lib/docker/test_env/parrot.go @@ -22,7 +22,7 @@ import ( const ( defaultParrotImage = "kalverra/parrot" - defaultParrotVersion = "v0.6.1" + defaultParrotVersion = "v0.6.2" defaultParrotPort = "80" ) From 844993528f1da5426f9bf9a1fceb991305a209c8 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Tue, 11 Mar 2025 19:09:47 -0400 Subject: [PATCH 2/4] Go mod --- lib/go.mod | 2 +- lib/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/go.mod b/lib/go.mod index b6cf949dc..27ef34328 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -32,7 +32,7 @@ require ( github.com/prometheus/common v0.62.0 github.com/rs/zerolog v1.33.0 github.com/slack-go/slack v0.15.0 - github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.1 + github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.2 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 diff --git a/lib/go.sum b/lib/go.sum index b234a7e26..272b28e0d 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -463,8 +463,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slack-go/slack v0.15.0 h1:LE2lj2y9vqqiOf+qIIy0GvEoxgF1N5yLGZffmEZykt0= github.com/slack-go/slack v0.15.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= -github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.1 h1:yVALM0MlRNU/rMKUyEjg1LaW+hZmjdkrOvKeXLkU3JM= -github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.1/go.mod h1:Z4K5VJLjsfqIIaBcZ1Sfccxu0xsCxBjPa6zF+5gtQaM= +github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.2 h1:cWUHB6QETyKbmh0B988f5AKIKb3aBDWugfrZ04jAUUY= +github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.2/go.mod h1:Z4K5VJLjsfqIIaBcZ1Sfccxu0xsCxBjPa6zF+5gtQaM= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 h1:Yf+n3T/fnUWcYyfe7bsygV4sWAkNo0QhN58APJFIKIc= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10/go.mod h1:05duR85P8YHuIfIkA7sn2bvrhKo/pDpFKV2rliYHNOo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= From b3bb9fb7de5a3d99b370000d37c2f0f6d806e137 Mon Sep 17 00:00:00 2001 From: skudasov Date: Wed, 12 Mar 2025 00:10:28 +0100 Subject: [PATCH 3/4] dump ctf-ci results to a file --- book/src/framework/components/analyze_ci.md | 2 + framework/cmd/ci.go | 68 +++++++++++---------- framework/cmd/main.go | 1 + 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/book/src/framework/components/analyze_ci.md b/book/src/framework/components/analyze_ci.md index 0c40a2929..dc0ff37d4 100644 --- a/book/src/framework/components/analyze_ci.md +++ b/book/src/framework/components/analyze_ci.md @@ -35,3 +35,5 @@ ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs -s 5 -e 3 ctf ci -r "smartcontractkit/chainlink-testing-framework" -w "Framework Golden Tests Examples" -t jobs -s 3 ``` You can also use `-debug` flag to dump all the workflow runs/jobs to `ctf-ci-debug` folder. + +All the results are also saved by default into `ctf-ci-$uuid.json` files so you can use this tool as CI performance linter. diff --git a/framework/cmd/ci.go b/framework/cmd/ci.go index 2cd154714..879c1549a 100644 --- a/framework/cmd/ci.go +++ b/framework/cmd/ci.go @@ -39,9 +39,10 @@ var ( SlowTestThreshold = 5 * time.Minute ExtremelySlowTestThreshold = 10 * time.Minute - DebugDirRoot = "ctf-ci-debug" - DebugSubDirWF = filepath.Join(DebugDirRoot, "workflows") - DebugSubDirJobs = filepath.Join(DebugDirRoot, "jobs") + DebugSubDirWF = filepath.Join(DebugDirRoot, "workflows") + DebugSubDirJobs = filepath.Join(DebugDirRoot, "jobs") + DefaultResultsDir = "." + DefaultResultsFile = "ctf-ci" ) type GitHubActionsClient interface { @@ -50,38 +51,38 @@ type GitHubActionsClient interface { } type AnalysisConfig struct { - Debug bool - Owner string - Repo string - WorkflowName string - TimeDaysBeforeStart int - TimeStart time.Time - TimeDaysBeforeEnd int - TimeEnd time.Time - Typ string - ResultsFile string + Debug bool `json:"debug"` + Owner string `json:"owner"` + Repo string `json:"repo"` + WorkflowName string `json:"workflow_name"` + TimeDaysBeforeStart int `json:"time_days_before_start"` + TimeStart time.Time `json:"time_start"` + TimeDaysBeforeEnd int `json:"time_days_before_end"` + TimeEnd time.Time `json:"time_end"` + Typ string `json:"type"` + ResultsFile string `json:"results_file"` } type Stat struct { - Name string - Successes int - Failures int - Cancels int - ReRuns int - P50 time.Duration - P95 time.Duration - P99 time.Duration - TotalDuration time.Duration - Durations []time.Duration + Name string `json:"name"` + Successes int `json:"successes"` + Failures int `json:"failures"` + Cancels int `json:"cancels"` + ReRuns int `json:"reRuns"` + P50 time.Duration `json:"p50"` + P95 time.Duration `json:"p95"` + P99 time.Duration `json:"p99"` + TotalDuration time.Duration `json:"totalDuration"` + Durations []time.Duration `json:"-"` } type Stats struct { - Mu *sync.Mutex - Runs int - CancelledRuns int - IgnoredRuns int - Jobs map[string]*Stat - Steps map[string]*Stat + Mu *sync.Mutex `json:"-"` + Runs int `json:"runs"` + CancelledRuns int `json:"cancelled_runs"` + IgnoredRuns int `json:"ignored_runs"` + Jobs map[string]*Stat `json:"jobs"` + Steps map[string]*Stat `json:"steps"` } func calculatePercentiles(stat *Stat) *Stat { @@ -103,7 +104,7 @@ func refreshDebugDirs() { } } -func writeStruct(enabled bool, dir, name string, data interface{}) error { +func dumpResults(enabled bool, dir, name string, data interface{}) error { if enabled { d, err := json.MarshalIndent(data, "", " ") if err != nil { @@ -145,7 +146,7 @@ func AnalyzeJobsSteps(ctx context.Context, client GitHubActionsClient, cfg *Anal // analyze workflow name := *wr.Name framework.L.Debug().Str("Name", name).Msg("Analyzing workflow run") - _ = writeStruct(cfg.Debug, DebugSubDirWF, name, wr) + _ = dumpResults(cfg.Debug, DebugSubDirWF, name, wr) eg.Go(func() error { rlJobs.Take() jobs, _, err := client.ListWorkflowJobs(ctx, cfg.Owner, cfg.Repo, *wr.ID, &github.ListWorkflowJobsOptions{ @@ -159,7 +160,7 @@ func AnalyzeJobsSteps(ctx context.Context, client GitHubActionsClient, cfg *Anal defer stats.Mu.Unlock() for _, j := range jobs.Jobs { name := *j.Name - _ = writeStruct(cfg.Debug, DebugSubDirJobs, name, wr) + _ = dumpResults(cfg.Debug, DebugSubDirJobs, name, wr) if skippedOrInProgressJob(j) { stats.IgnoredRuns++ continue @@ -283,6 +284,9 @@ func AnalyzeCIRuns(cfg *AnalysisConfig) (*Stats, error) { if err != nil { return nil, fmt.Errorf("failed to fetch workflow runs: %w", err) } + if cfg.ResultsFile != "" { + _ = dumpResults(true, DefaultResultsDir, DefaultResultsFile, stats) + } return stats, nil } diff --git a/framework/cmd/main.go b/framework/cmd/main.go index 4f1d8c0a9..64d3ec88a 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -263,6 +263,7 @@ func main() { TimeDaysBeforeStart: c.Int("start"), TimeDaysBeforeEnd: c.Int("end"), Typ: typ, + ResultsFile: "ctf-ci.json", }) return err }, From 9893e655348b4653e7fcb89f10104562170383c3 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Tue, 11 Mar 2025 19:10:40 -0400 Subject: [PATCH 4/4] Changeset --- lib/.changeset/v1.52.3.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 lib/.changeset/v1.52.3.md diff --git a/lib/.changeset/v1.52.3.md b/lib/.changeset/v1.52.3.md new file mode 100644 index 000000000..fb9a6c9fb --- /dev/null +++ b/lib/.changeset/v1.52.3.md @@ -0,0 +1 @@ +- Updates Parrot to static binary \ No newline at end of file