From dc1a66c44f1b73cd522e8213d2940303edfbf07c Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 19 Jan 2026 15:47:28 +0100 Subject: [PATCH 01/14] multi-product env template --- framework/cmd/main.go | 18 +- framework/leak/cmd/main.go | 3 +- framework/leak/detector_hog_test.go | 8 +- framework/{tmpl_gen.go => tmpl_gen_env.go} | 877 +++++---------------- framework/tmpl_gen_product.go | 373 +++++++++ framework/tmpl_gen_product_fakes.go | 226 ++++++ framework/tmpl_gen_test.go | 9 +- 7 files changed, 805 insertions(+), 709 deletions(-) rename framework/{tmpl_gen.go => tmpl_gen_env.go} (74%) create mode 100644 framework/tmpl_gen_product.go create mode 100644 framework/tmpl_gen_product_fakes.go diff --git a/framework/cmd/main.go b/framework/cmd/main.go index 7afdefb2c..c82a7eacb 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -64,12 +64,6 @@ Usage: Aliases: []string{"r"}, Usage: "Your product name", }, - &cli.StringFlag{ - Name: "product-configuration-type", - Aliases: []string{"p"}, - Value: "evm-single", - Usage: "Product configuration type/layout (single network, multi-network, etc)", - }, &cli.IntFlag{ Name: "nodes", Aliases: []string{"n"}, @@ -79,7 +73,6 @@ Usage: }, Action: func(c *cli.Context) error { outputDir := c.String("output-dir") - productConfType := c.String("product-configuration-type") nodes := c.Int("nodes") cliName := c.String("cli") if cliName == "" { @@ -93,17 +86,22 @@ Usage: Str("OutputDir", outputDir). Str("Name", cliName). Int("CLNodes", nodes). - Str("ProductConfigurationType", productConfType). Msg("Generating developer environment") - cg, err := framework.NewEnvBuilder(cliName, nodes, productConfType, productName). + cg, err := framework.NewEnvBuilder(cliName, nodes, productName). OutputDir(outputDir). Build() if err != nil { return fmt.Errorf("failed to create codegen: %w", err) } if err := cg.Write(); err != nil { - return fmt.Errorf("failed to generate module: %w", err) + return fmt.Errorf("failed to generate environment: %w", err) + } + if err := cg.WriteFakes(); err != nil { + return fmt.Errorf("failed to generate fakes: %w", err) + } + if err := cg.WriteProducts(); err != nil { + return fmt.Errorf("failed to generate products: %w", err) } fmt.Println() diff --git a/framework/leak/cmd/main.go b/framework/leak/cmd/main.go index 54a025b30..a34525005 100644 --- a/framework/leak/cmd/main.go +++ b/framework/leak/cmd/main.go @@ -11,7 +11,7 @@ import ( ) const ( - StepTick = 3 * time.Minute + StepTick = 20 * time.Minute ) func main() { @@ -20,7 +20,6 @@ func main() { workersSchedule := os.Getenv("WORKERS") memorySchedule := os.Getenv("MEMORY") repeatStr := os.Getenv("REPEAT") - leaks := make([][]byte, 0) workerCounter := 0 diff --git a/framework/leak/detector_hog_test.go b/framework/leak/detector_hog_test.go index c01915e66..331744c7f 100644 --- a/framework/leak/detector_hog_test.go +++ b/framework/leak/detector_hog_test.go @@ -16,19 +16,19 @@ import ( ) func TestCyclicHog(t *testing.T) { - t.Skip("unskip when debugging new queries") + // t.Skip("unskip when debugging new queries") ctx := context.Background() hog, err := SetupResourceHog( ctx, "resource-hog:latest", map[string]string{ - "WORKERS": "1,2,3,2,1", - "MEMORY": "1,2,3,2,1", + "WORKERS": "1,2,3,4,5,5,4,3,2,1", + "MEMORY": "1,2,3,4,5,5,4,3,2,1", "REPEAT": "1", }, ) require.NoError(t, err) - time.Sleep(15 * time.Minute) + time.Sleep(2 * time.Hour) t.Cleanup(func() { if err := hog.Terminate(ctx); err != nil { log.Printf("Failed to terminate container: %v", err) diff --git a/framework/tmpl_gen.go b/framework/tmpl_gen_env.go similarity index 74% rename from framework/tmpl_gen.go rename to framework/tmpl_gen_env.go index c910a9ed1..2c1c1bfa4 100644 --- a/framework/tmpl_gen.go +++ b/framework/tmpl_gen_env.go @@ -48,6 +48,47 @@ test load ↵ 🔄 **Enforce** quality standards in CI: copy .github/workflows to your CI folder, commit and make them pass ` + // ProductsInterfaceTmpl common interface for arbitrary products deployed in devenv + ProductsInterfaceTmpl = `package {{ .PackageName }} + +import ( + "context" + + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake" + + nodeset "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" +) + +// Product describes a minimal set of methods that each legacy product must implement +type Product interface { + Load() error + + Store(path string, instanceIdx int) error + + GenerateNodesSecrets( + ctx context.Context, + fs *fake.Input, + bc *blockchain.Input, + ns *nodeset.Input, + ) (string, error) + + GenerateNodesConfig( + ctx context.Context, + fs *fake.Input, + bc *blockchain.Input, + ns *nodeset.Input, + ) (string, error) + + ConfigureJobsAndContracts( + ctx context.Context, + fs *fake.Input, + bc *blockchain.Input, + ns *nodeset.Input, + ) error +} +` + // GoModTemplate go module template GoModTemplate = `module {{.ModuleName}} @@ -808,18 +849,14 @@ env-out.toml` "weekStart": "" }` // ConfigTOMLTmpl is a default env.toml template for devenv describind components configuration - ConfigTOMLTmpl = `[on_chain] - link_contract_address = "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" - cl_nodes_funding_eth = 50 - cl_nodes_funding_link = 50 - verification_timeout_sec = 400 - contracts_configuration_timeout_sec = 60 - verify = false - - [on_chain.gas_settings] - fee_cap_multiplier = 2 - tip_cap_multiplier = 2 + ConfigTOMLTmpl = ` +[[products]] +name = "{{ .ProductName }}" +instances = 1 +[fake_server] + image = "ocr2-fakes:latest" + port = 9111 [[blockchains]] chain_id = "1337" @@ -1233,8 +1270,7 @@ var restartCmd = &cobra.Command{ if err != nil { return fmt.Errorf("failed to clean Docker resources: %w", err) } - _, err = devenv.NewEnvironment() - return err + return devenv.NewEnvironment(context.Background()) }, } @@ -1253,11 +1289,7 @@ var upCmd = &cobra.Command{ framework.L.Info().Str("Config", configFile).Msg("Creating development environment") _ = os.Setenv("CTF_CONFIGS", configFile) _ = os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") - _, err := devenv.NewEnvironment() - if err != nil { - return err - } - return nil + return devenv.NewEnvironment(context.Background()) }, } @@ -1337,8 +1369,8 @@ var obsUpCmd = &cobra.Command{ if err != nil { return fmt.Errorf("observability up failed: %w", err) } - devenv.Plog.Info().Msgf("{{ .ProductName }} Dashboard: %s", Local{{ .ProductName }}Dashboard) - devenv.Plog.Info().Msgf("{{ .ProductName }} Load Test Dashboard: %s", LocalWASPLoadDashboard) + devenv.L.Info().Msgf("{{ .ProductName }} Dashboard: %s", Local{{ .ProductName }}Dashboard) + devenv.L.Info().Msgf("{{ .ProductName }} Load Test Dashboard: %s", LocalWASPLoadDashboard) return nil }, } @@ -1370,8 +1402,8 @@ var obsRestartCmd = &cobra.Command{ if err != nil { return fmt.Errorf("observability up failed: %w", err) } - devenv.Plog.Info().Msgf("{{ .ProductName }} Dashboard: %s", Local{{ .ProductName }}Dashboard) - devenv.Plog.Info().Msgf("{{ .ProductName }} Load Test Dashboard: %s", LocalWASPLoadDashboard) + devenv.L.Info().Msgf("{{ .ProductName }} Dashboard: %s", Local{{ .ProductName }}Dashboard) + devenv.L.Info().Msgf("{{ .ProductName }} Load Test Dashboard: %s", LocalWASPLoadDashboard) return nil }, } @@ -1462,7 +1494,7 @@ func main() { return } if err := rootCmd.Execute(); err != nil { - devenv.Plog.Err(err).Send() + devenv.L.Err(err).Send() os.Exit(1) } }` @@ -1600,7 +1632,7 @@ import ( f "github.com/smartcontractkit/chainlink-testing-framework/framework" ) -var L = de.Plog +var L = de.L func TestSmoke(t *testing.T) { in, err := de.LoadOutput[de.Cfg]("../env-out.toml") @@ -1658,53 +1690,22 @@ func assertResources(t *testing.T, in *de.Cfg, start, end time.Time) { } } ` - // CLDFTmpl is a Chainlink Deployments Framework template - CLDFTmpl = `package {{ .PackageName }} + // JDTmpl is a JobDistributor client wrappers + JDTmpl = `package {{ .PackageName }} import ( "context" - "crypto/ecdsa" "errors" "fmt" - "math/big" - "strings" - "time" - "github.com/Masterminds/semver/v3" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-deployments-framework/datastore" - "github.com/smartcontractkit/chainlink-deployments-framework/operations" - "github.com/smartcontractkit/chainlink-evm/gethwrappers/shared/generated/link_token" - "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - chainsel "github.com/smartcontractkit/chain-selectors" - cldfchain "github.com/smartcontractkit/chainlink-deployments-framework/chain" - cldfevm "github.com/smartcontractkit/chainlink-deployments-framework/chain/evm" - cldfevmprovider "github.com/smartcontractkit/chainlink-deployments-framework/chain/evm/provider" - cldf "github.com/smartcontractkit/chainlink-deployments-framework/deployment" csav1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/csa" jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" ) -const ( - AnvilKey0 = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - DefaultNativeTransferGasPrice = 21000 -) - -const LinkToken cldf.ContractType = "LinkToken" - -var _ cldf.ChangeSet[[]uint64] = DeployLinkToken - type JobDistributor struct { nodev1.NodeServiceClient jobv1.JobServiceClient @@ -1717,208 +1718,6 @@ type JDConfig struct { WSRPC string } -// DeployLinkToken deploys a link token contract to the chain identified by the ChainSelector. -func DeployLinkToken(e cldf.Environment, chains []uint64) (cldf.ChangesetOutput, error) { //nolint:gocritic - newAddresses := cldf.NewMemoryAddressBook() - deployGrp := errgroup.Group{} - for _, chain := range chains { - family, err := chainsel.GetSelectorFamily(chain) - if err != nil { - return cldf.ChangesetOutput{AddressBook: newAddresses}, err - } - var deployFn func() error - switch family { - case chainsel.FamilyEVM: - // Deploy EVM LINK token - deployFn = func() error { - _, err := deployLinkTokenContractEVM( - e.Logger, e.BlockChains.EVMChains()[chain], newAddresses, - ) - return err - } - default: - return cldf.ChangesetOutput{}, fmt.Errorf("unsupported chain family %s", family) - } - deployGrp.Go(func() error { - err := deployFn() - if err != nil { - e.Logger.Errorw("Failed to deploy link token", "chain", chain, "err", err) - return fmt.Errorf("failed to deploy link token for chain %d: %w", chain, err) - } - return nil - }) - } - return cldf.ChangesetOutput{AddressBook: newAddresses}, deployGrp.Wait() -} - -func deployLinkTokenContractEVM( - lggr logger.Logger, - chain cldfevm.Chain, //nolint:gocritic - ab cldf.AddressBook, -) (*cldf.ContractDeploy[*link_token.LinkToken], error) { - linkToken, err := cldf.DeployContract[*link_token.LinkToken](lggr, chain, ab, - func(chain cldfevm.Chain) cldf.ContractDeploy[*link_token.LinkToken] { - var ( - linkTokenAddr common.Address - tx *types.Transaction - linkToken *link_token.LinkToken - err2 error - ) - if !chain.IsZkSyncVM { - linkTokenAddr, tx, linkToken, err2 = link_token.DeployLinkToken( - chain.DeployerKey, - chain.Client, - ) - } else { - linkTokenAddr, _, linkToken, err2 = link_token.DeployLinkTokenZk( - nil, - chain.ClientZkSyncVM, - chain.DeployerKeyZkSyncVM, - chain.Client, - ) - } - return cldf.ContractDeploy[*link_token.LinkToken]{ - Address: linkTokenAddr, - Contract: linkToken, - Tx: tx, - Tv: cldf.NewTypeAndVersion(LinkToken, *semver.MustParse("1.0.0")), - Err: err2, - } - }) - if err != nil { - lggr.Errorw("Failed to deploy link token", "chain", chain.String(), "err", err) - return linkToken, err - } - return linkToken, nil -} - -// LoadCLDFEnvironment loads CLDF environment with a memory data store and JD client. -func LoadCLDFEnvironment(in *Cfg) (cldf.Environment, error) { - ctx := context.Background() - - getCtx := func() context.Context { - return ctx - } - - // This only generates a brand new datastore and does not load any existing data. - // We will need to figure out how data will be persisted and loaded in the future. - ds := datastore.NewMemoryDataStore().Seal() - - lggr, err := logger.NewWith(func(config *zap.Config) { - config.Development = true - config.Encoding = "console" - }) - if err != nil { - return cldf.Environment{}, fmt.Errorf("failed to create logger: %w", err) - } - - blockchains, err := loadCLDFChains(in.Blockchains) - if err != nil { - return cldf.Environment{}, fmt.Errorf("failed to load CLDF chains: %w", err) - } - - jd, err := NewJDClient(ctx, JDConfig{ - GRPC: in.JD.Out.ExternalGRPCUrl, - WSRPC: in.JD.Out.ExternalWSRPCUrl, - }) - if err != nil { - return cldf.Environment{}, - fmt.Errorf("failed to load offchain client: %w", err) - } - - opBundle := operations.NewBundle( - getCtx, - lggr, - operations.NewMemoryReporter(), - operations.WithOperationRegistry(operations.NewOperationRegistry()), - ) - - return cldf.Environment{ - Name: "local", - Logger: lggr, - ExistingAddresses: cldf.NewMemoryAddressBook(), - DataStore: ds, - Offchain: jd, - GetContext: getCtx, - OperationsBundle: opBundle, - BlockChains: cldfchain.NewBlockChainsFromSlice(blockchains), - }, nil -} - -func loadCLDFChains(bcis []*blockchain.Input) ([]cldfchain.BlockChain, error) { - blockchains := make([]cldfchain.BlockChain, 0) - for _, bci := range bcis { - switch bci.Type { - case "anvil": - bc, err := loadEVMChain(bci) - if err != nil { - return blockchains, fmt.Errorf("failed to load EVM chain %s: %w", bci.ChainID, err) - } - - blockchains = append(blockchains, bc) - default: - return blockchains, fmt.Errorf("unsupported chain type %s", bci.Type) - } - } - - return blockchains, nil -} - -func loadEVMChain(bci *blockchain.Input) (cldfchain.BlockChain, error) { - if bci.Out == nil { - return nil, fmt.Errorf("output configuration for %s blockchain %s is not set", bci.Type, bci.ChainID) - } - - chainDetails, err := chainsel.GetChainDetailsByChainIDAndFamily(bci.ChainID, chainsel.FamilyEVM) - if err != nil { - return nil, fmt.Errorf("failed to get chain details for %s: %w", bci.ChainID, err) - } - - chain, err := cldfevmprovider.NewRPCChainProvider( - chainDetails.ChainSelector, - cldfevmprovider.RPCChainProviderConfig{ - DeployerTransactorGen: cldfevmprovider.TransactorFromRaw( - // TODO: we need to figure out a reliable way to get secrets here that is - // TODO: - easy for developers - // TODO: - works the same way locally, in K8s and in CI - // TODO: - do not require specific AWS access like AWSSecretsManager - // TODO: for now it's just an Anvil 0 key - AnvilKey0, - ), - RPCs: []cldf.RPC{ - { - Name: "default", - WSURL: bci.Out.Nodes[0].ExternalWSUrl, - HTTPURL: bci.Out.Nodes[0].ExternalHTTPUrl, - PreferredURLScheme: cldf.URLSchemePreferenceHTTP, - }, - }, - ConfirmFunctor: cldfevmprovider.ConfirmFuncGeth(1 * time.Minute), - }, - ).Initialize(context.Background()) - if err != nil { - return nil, fmt.Errorf("failed to initialize EVM chain %s: %w", bci.ChainID, err) - } - - return chain, nil -} - -// NewJDClient creates a new JobDistributor client. -func NewJDClient(ctx context.Context, cfg JDConfig) (cldf.OffchainClient, error) { - conn, err := NewJDConnection(cfg) - if err != nil { - return nil, fmt.Errorf("failed to connect Job Distributor service. Err: %w", err) - } - jd := &JobDistributor{ - WSRPC: cfg.WSRPC, - NodeServiceClient: nodev1.NewNodeServiceClient(conn), - JobServiceClient: jobv1.NewJobServiceClient(conn), - CSAServiceClient: csav1.NewCSAServiceClient(conn), - } - - return jd, err -} - func (jd JobDistributor) GetCSAPublicKey(ctx context.Context) (string, error) { keypairs, err := jd.ListKeypairs(ctx, &csav1.ListKeypairsRequest{}) if err != nil { @@ -1962,119 +1761,6 @@ func NewJDConnection(cfg JDConfig) (*grpc.ClientConn, error) { return conn, nil } - -// FundNodeEIP1559 funds CL node using RPC URL, recipient address and amount of funds to send (ETH). -// Uses EIP-1559 transaction type. -func FundNodeEIP1559(c *ethclient.Client, pkey, recipientAddress string, amountOfFundsInETH float64) error { - amount := new(big.Float).Mul(big.NewFloat(amountOfFundsInETH), big.NewFloat(1e18)) - amountWei, _ := amount.Int(nil) - Plog.Info().Str("Addr", recipientAddress).Str("Wei", amountWei.String()).Msg("Funding Node") - - chainID, err := c.NetworkID(context.Background()) - if err != nil { - return err - } - privateKeyStr := strings.TrimPrefix(pkey, "0x") - privateKey, err := crypto.HexToECDSA(privateKeyStr) - if err != nil { - return err - } - publicKey := privateKey.Public() - publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) - if !ok { - return errors.New("error casting public key to ECDSA") - } - fromAddress := crypto.PubkeyToAddress(*publicKeyECDSA) - - nonce, err := c.PendingNonceAt(context.Background(), fromAddress) - if err != nil { - return err - } - feeCap, err := c.SuggestGasPrice(context.Background()) - if err != nil { - return err - } - tipCap, err := c.SuggestGasTipCap(context.Background()) - if err != nil { - return err - } - recipient := common.HexToAddress(recipientAddress) - tx := types.NewTx(&types.DynamicFeeTx{ - ChainID: chainID, - Nonce: nonce, - To: &recipient, - Value: amountWei, - Gas: DefaultNativeTransferGasPrice, - GasFeeCap: feeCap, - GasTipCap: tipCap, - }) - signedTx, err := types.SignTx(tx, types.NewLondonSigner(chainID), privateKey) - if err != nil { - return err - } - err = c.SendTransaction(context.Background(), signedTx) - if err != nil { - return err - } - if _, err := bind.WaitMined(context.Background(), c, signedTx); err != nil { - return err - } - Plog.Info().Str("Wei", amountWei.String()).Msg("Funded with ETH") - return nil -} - -/* -This is just a basic ETH client, CLDF should provide something like this -*/ - -// ETHClient creates a basic Ethereum client using PRIVATE_KEY env var and tip/cap gas settings -func ETHClient(in *Cfg) (*ethclient.Client, *bind.TransactOpts, string, error) { - rpcURL := in.Blockchains[0].Out.Nodes[0].ExternalWSUrl - client, err := ethclient.Dial(rpcURL) - if err != nil { - return nil, nil, "", fmt.Errorf("could not connect to eth client: %w", err) - } - privateKey, err := crypto.HexToECDSA(GetNetworkPrivateKey()) - if err != nil { - return nil, nil, "", fmt.Errorf("could not parse private key: %w", err) - } - publicKey := privateKey.PublicKey - address := crypto.PubkeyToAddress(publicKey).String() - chainID, err := client.ChainID(context.Background()) - if err != nil { - return nil, nil, "", fmt.Errorf("could not get chain ID: %w", err) - } - auth, err := bind.NewKeyedTransactorWithChainID(privateKey, chainID) - if err != nil { - return nil, nil, "", fmt.Errorf("could not create transactor: %w", err) - } - gasSettings := in.OnChain.GasSettings - fc, tc, err := MultiplyEIP1559GasPrices(client, gasSettings.FeeCapMultiplier, gasSettings.TipCapMultiplier) - if err != nil { - return nil, nil, "", fmt.Errorf("could not get bumped gas price: %w", err) - } - auth.GasFeeCap = fc - auth.GasTipCap = tc - Plog.Info(). - Str("GasFeeCap", fc.String()). - Str("GasTipCap", tc.String()). - Msg("Default gas prices set") - return client, auth, address, nil -} - -// MultiplyEIP1559GasPrices returns bumped EIP1159 gas prices increased by multiplier -func MultiplyEIP1559GasPrices(client *ethclient.Client, fcMult, tcMult int64) (*big.Int, *big.Int, error) { - feeCap, err := client.SuggestGasPrice(context.Background()) - if err != nil { - return nil, nil, err - } - tipCap, err := client.SuggestGasTipCap(context.Background()) - if err != nil { - return nil, nil, err - } - - return new(big.Int).Mul(feeCap, big.NewInt(fcMult)), new(big.Int).Mul(tipCap, big.NewInt(tcMult)), nil -} ` // DebugToolsTmpl is a template for various debug tools, tracing, tx debug, etc DebugToolsTmpl = `package {{ .PackageName }} @@ -2102,18 +1788,6 @@ func tracing() func() { // ConfigTmpl is a template for reading and writing devenv configuration (env.toml, env-out.toml) ConfigTmpl = `package {{ .PackageName }} -/* -This file provides a simple boilerplate for TOML configuration with overrides -It has 4 functions: Load[T], Store[T], LoadCache[T] and GetNetworkPrivateKey - -To configure the environment we use a set of files we read from the env var CTF_CONFIGS=env.toml,overrides.toml (can be more than 2) in Load[T] -To store infra or product component outputs we use Store[T] that creates env-cache.toml file. -This file can be used in tests or in any other code that integrated with dev environment. -LoadCache[T] is used if you need to write outputs the second time. - -GetNetworkPrivateKey is used to get your network private key from the env var we are using across all our environments, or fallback to default Anvil's key. -*/ - import ( "errors" "fmt" @@ -2124,7 +1798,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog" - "github.com/rs/zerolog/log" ) const ( @@ -2138,8 +1811,6 @@ const ( DefaultAnvilKey = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" ) -var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.InfoLevel) - // Load loads TOML configurations from environment variable, ex.: CTF_CONFIGS=env.toml,overrides.toml // and unmarshalls the files from left to right overriding keys. func Load[T any]() (*T, error) { @@ -2225,293 +1896,141 @@ func GetNetworkPrivateKey() string { return pk } ` + // EnvironmentTmpl is an environment.go template - main file for environment composition EnvironmentTmpl = `package {{ .PackageName }} import ( + "context" + "errors" "fmt" + "os" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/smartcontractkit/chainlink-testing-framework/framework" "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake" "github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd" + "github.com/smartcontractkit/{{ .ProductName }}/devenv/products/{{ .ProductName }}" ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" ) +var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "{{ .ProductName }}"}).Logger() + +type ProductInfo struct { + Name string ` + "`" + `toml:"name"` + "`" + ` + Instances int ` + "`" + `toml:"instances"` + "`" + ` +} + type Cfg struct { - OnChain *OnChain ` + "`" + `toml:"on_chain"` + "`" + ` - Blockchains []*blockchain.Input ` + "`" + `toml:"blockchains" validate:"required"` + "`" + ` - NodeSets []*ns.Input ` + "`" + `toml:"nodesets" validate:"required"` + "`" + ` - JD *jd.Input ` + "`" + `toml:"jd"` + "`" + ` + Products []*ProductInfo ` + "`" + `toml:"products"` + "`" + ` + Blockchains []*blockchain.Input ` + "`" + `toml:"blockchains" validate:"required"` + "`" + ` + FakeServer *fake.Input ` + "`" + `toml:"fake_server" validate:"required"` + "`" + ` + NodeSets []*ns.Input ` + "`" + `toml:"nodesets" validate:"required"` + "`" + ` + JD *jd.Input ` + "`" + `toml:"jd"` + "`" + ` } -func NewEnvironment() (*Cfg, error) { - endTracing := tracing() - defer endTracing() +func newProduct(name string) (Product, error) { + switch name { + case "{{ .ProductName }}": + return {{ .ProductName }}.NewConfigurator(), nil + default: + return nil, fmt.Errorf("unknown product type: %s", name) + } +} +func NewEnvironment(ctx context.Context) error { if err := framework.DefaultNetwork(nil); err != nil { - return nil, err + return err } in, err := Load[Cfg]() if err != nil { - return nil, fmt.Errorf("failed to load configuration: %w", err) + return fmt.Errorf("failed to load configuration: %w", err) } _, err = blockchain.NewBlockchainNetwork(in.Blockchains[0]) if err != nil { - return nil, fmt.Errorf("failed to create blockchain network 1337: %w", err) + return fmt.Errorf("failed to create blockchain network 1337: %w", err) } - if err := DefaultProductConfiguration(in, ConfigureNodesNetwork); err != nil { - return nil, fmt.Errorf("failed to setup default CLDF orchestration: %w", err) + if os.Getenv("FAKE_SERVER_IMAGE") != "" { + in.FakeServer.Image = os.Getenv("FAKE_SERVER_IMAGE") } - _, err = ns.NewSharedDBNodeSet(in.NodeSets[0], nil) + _, err = fake.NewDockerFakeDataProvider(in.FakeServer) if err != nil { - return nil, fmt.Errorf("failed to create new shared db node set: %w", err) - } - if err := DefaultProductConfiguration(in, ConfigureProductContractsJobs); err != nil { - return nil, fmt.Errorf("failed to setup default CLDF orchestration: %w", err) + return fmt.Errorf("failed to create fake data provider: %w", err) } - return in, Store[Cfg](in) -} -` - // SingleNetworkProductConfigurationTmpl is an single-network EVM product configuration template - SingleNetworkEVMProductConfigurationTmpl = `package {{ .PackageName }} -import ( - "context" - "fmt" - "math/big" - "os" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/smartcontractkit/chainlink-evm/gethwrappers/shared/generated/link_token" - "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" -) - -const ( - ConfigureNodesNetwork ConfigPhase = iota - ConfigureProductContractsJobs -) - -var Plog = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "on_chain"}).Logger() - -type OnChain struct { - LinkContractAddress string ` + "`" + `toml:"link_contract_address"` + "`" + ` - CLNodesFundingETH float64 ` + "`" + `toml:"cl_nodes_funding_eth"` + "`" + ` - CLNodesFundingLink float64 ` + "`" + `toml:"cl_nodes_funding_link"` + "`" + ` - VerificationTimeoutSec time.Duration ` + "`" + `toml:"verification_timeout_sec"` + "`" + ` - ContractsConfigurationTimeoutSec time.Duration ` + "`" + `toml:"contracts_configuration_timeout_sec"` + "`" + ` - GasSettings *GasSettings ` + "`" + `toml:"gas_settings"` + "`" + ` - Verify bool ` + "`" + `toml:"verify"` + "`" + ` - DeployedContracts *DeployedContracts ` + "`" + `toml:"deployed_contracts"` + "`" + ` -} - -type DeployedContracts struct { - SomeContractAddr string ` + "`" + `toml:"some_contract_addr"` + "`" + ` -} - - -type GasSettings struct { - FeeCapMultiplier int64 ` + "`" + `toml:"fee_cap_multiplier"` + "`" + ` - TipCapMultiplier int64 ` + "`" + `toml:"tip_cap_multiplier"` + "`" + ` -} - -type Jobs struct { - ConfigPollIntervalSeconds time.Duration ` + "`" + `toml:"config_poll_interval_sec"` + "`" + ` - MaxTaskDurationSec time.Duration ` + "`" + `toml:"max_task_duration_sec"` + "`" + ` -} - -type ConfigPhase int - -// deployLinkAndMint is a universal action that deploys link token and mints required amount of LINK token for all the nodes. -func deployLinkAndMint(ctx context.Context, in *Cfg, c *ethclient.Client, auth *bind.TransactOpts, rootAddr string, transmitters []common.Address) (*link_token.LinkToken, error) { - addr, tx, lt, err := link_token.DeployLinkToken(auth, c) - if err != nil { - return nil, fmt.Errorf("could not create link token contract: %w", err) - } - _, err = bind.WaitDeployed(ctx, c, tx) - if err != nil { - return nil, err - } - Plog.Info().Str("Address", addr.Hex()).Msg("Deployed link token contract") - tx, err = lt.GrantMintRole(auth, common.HexToAddress(rootAddr)) - if err != nil { - return nil, fmt.Errorf("could not grant mint role: %w", err) - } - _, err = bind.WaitMined(ctx, c, tx) - if err != nil { - return nil, err - } - // mint for public keys of nodes directly instead of transferring - for _, transmitter := range transmitters { - amount := new(big.Float).Mul(big.NewFloat(in.OnChain.CLNodesFundingLink), big.NewFloat(1e18)) - amountWei, _ := amount.Int(nil) - Plog.Info().Msgf("Minting LINK for transmitter address: %s", transmitter.Hex()) - tx, err = lt.Mint(auth, transmitter, amountWei) + // get all the product orchestrations, generate product specific overrides + productConfigurators := make([]Product, 0) + nodeConfigs := make([]string, 0) + nodeSecrets := make([]string, 0) + for _, product := range in.Products { + p, err := newProduct(product.Name) if err != nil { - return nil, fmt.Errorf("could not transfer link token contract: %w", err) + return err + } + if err = p.Load(); err != nil { + return fmt.Errorf("failed to load product config: %w", err) } - _, err = bind.WaitMined(ctx, c, tx) + + cfg, err := p.GenerateNodesConfig(ctx, in.FakeServer, in.Blockchains[0], in.NodeSets[0]) if err != nil { - return nil, err + return fmt.Errorf("failed to generate CL nodes config: %w", err) } - } - return lt, nil -} + nodeConfigs = append(nodeConfigs, cfg) + secrets, err := p.GenerateNodesSecrets(ctx, in.FakeServer, in.Blockchains[0], in.NodeSets[0]) + if err != nil { + return fmt.Errorf("failed to generate CL nodes config: %w", err) + } + nodeSecrets = append(nodeSecrets, secrets) -func configureContracts(in *Cfg, c *ethclient.Client, auth *bind.TransactOpts, cl []*clclient.ChainlinkClient, rootAddr string, transmitters []common.Address) (*DeployedContracts, error) { - ctx, cancel := context.WithTimeout(context.Background(), in.OnChain.ContractsConfigurationTimeoutSec*time.Second) - defer cancel() - Plog.Info().Msg("Deploying LINK token contract") - _, err := deployLinkAndMint(ctx, in, c, auth, rootAddr, transmitters) - if err != nil { - return nil, fmt.Errorf("could not create link token contract and mint: %w", err) + productConfigurators = append(productConfigurators, p) } - // TODO: use client and deploy your contracts - return &DeployedContracts{ - SomeContractAddr: "", - }, nil -} -func configureJobs(in *Cfg, clNodes []*clclient.ChainlinkClient, contracts *DeployedContracts) error { - bootstrapNode := clNodes[0] - workerNodes := clNodes[1:] - // TODO: define your jobs - job := "" - _, _, err := bootstrapNode.CreateJobRaw(job) + // merge overrides, spin up node sets and write infrastructure outputs + // infra is always common for all the products, if it can't be we should fail + // user should use different infra layout in env.toml then + for _, ns := range in.NodeSets[0].NodeSpecs { + ns.Node.TestConfigOverrides = strings.Join(nodeConfigs, "\n") + ns.Node.TestSecretsOverrides = strings.Join(nodeSecrets, "\n") + if os.Getenv("CHAINLINK_IMAGE") != "" { + ns.Node.Image = os.Getenv("CHAINLINK_IMAGE") + } + } + _, err = ns.NewSharedDBNodeSet(in.NodeSets[0], nil) if err != nil { - return fmt.Errorf("creating bootstrap job have failed: %w", err) + return fmt.Errorf("failed to create new shared db node set: %w", err) } - - for _, chainlinkNode := range workerNodes { - // TODO: define your job for nodes here - job := "" - _, _, err = chainlinkNode.CreateJobRaw(job) - if err != nil { - return fmt.Errorf("creating job on node have failed: %w", err) - } + if err := Store[Cfg](in); err != nil { + return err } - return nil -} -// DefaultProductConfiguration is default product configuration that includes: -// - Deploying required prerequisites (LINK token, shared contracts) -// - Applying product-specific changesets -// - Creating cldf.Environment, connecting to components, see *Cfg fields -// - Generating CL nodes configs -// All the data can be added *Cfg struct like and is synced between local machine and remote environment -// so later both local and remote tests can use it. -func DefaultProductConfiguration(in *Cfg, phase ConfigPhase) error { - pkey := GetNetworkPrivateKey() - if pkey == "" { - return fmt.Errorf("PRIVATE_KEY environment variable not set") - } - switch phase { - case ConfigureNodesNetwork: - Plog.Info().Msg("Applying default CL nodes configuration") - node := in.Blockchains[0].Out.Nodes[0] - chainID := in.Blockchains[0].ChainID - // configure node set and generate CL nodes configs - netConfig := fmt.Sprintf(` + "`" + ` - [[EVM]] - LogPollInterval = '1s' - BlockBackfillDepth = 100 - LinkContractAddress = '%s' - ChainID = '%s' - MinIncomingConfirmations = 1 - MinContractPayment = '0.0000001 link' - FinalityDepth = %d - - [[EVM.Nodes]] - Name = 'default' - WsUrl = '%s' - HttpUrl = '%s' - - [Feature] - FeedsManager = true - LogPoller = true - UICSAKeys = true - [OCR2] - Enabled = true - SimulateTransactions = false - DefaultTransactionQueueDepth = 1 - [P2P.V2] - Enabled = true - ListenAddresses = ['0.0.0.0:6690'] - - [Log] - JSONConsole = true - Level = 'debug' - [Pyroscope] - ServerAddress = 'http://host.docker.internal:4040' - Environment = 'local' - [WebServer] - SessionTimeout = '999h0m0s' - HTTPWriteTimeout = '3m' - SecureCookies = false - HTTPPort = 6688 - [WebServer.TLS] - HTTPSPort = 0 - [WebServer.RateLimit] - Authenticated = 5000 - Unauthenticated = 5000 - [JobPipeline] - [JobPipeline.HTTPRequest] - DefaultTimeout = '1m' - [Log.File] - MaxSize = '0b' -` + "`" + `, in.OnChain.LinkContractAddress, chainID, 5, node.InternalWSUrl, node.InternalHTTPUrl) - for _, nodeSpec := range in.NodeSets[0].NodeSpecs { - nodeSpec.Node.TestConfigOverrides = netConfig - } - Plog.Info().Msg("Nodes network configuration is finished") - case ConfigureProductContractsJobs: - Plog.Info().Msg("Connecting to CL nodes") - nodeClients, err := clclient.New(in.NodeSets[0].Out.CLNodes) - if err != nil { - return err - } - transmitters := make([]common.Address, 0) - ethKeyAddresses := make([]string, 0) - for i, nc := range nodeClients { - addr, err := nc.ReadPrimaryETHKey(in.Blockchains[0].ChainID) + // deploy all products and all instances, + // product config function controls what to read and how to orchestrate each instance + // via their own TOML part, we only deploy N instances of product M + for productIdx, productInfo := range in.Products { + for productInstance := range productInfo.Instances { + err = productConfigurators[productIdx].ConfigureJobsAndContracts( + ctx, + in.FakeServer, + in.Blockchains[0], + in.NodeSets[0], + ) if err != nil { - return err + return fmt.Errorf("failed to setup default product deployment: %w", err) } - ethKeyAddresses = append(ethKeyAddresses, addr.Attributes.Address) - transmitters = append(transmitters, common.HexToAddress(addr.Attributes.Address)) - Plog.Info(). - Int("Idx", i). - Str("ETH", addr.Attributes.Address). - Msg("Node info") - } - // ETH examples - c, auth, rootAddr, err := ETHClient(in) - if err != nil { - return fmt.Errorf("could not create basic eth client: %w", err) - } - for _, addr := range ethKeyAddresses { - if err := FundNodeEIP1559(c, pkey, addr, in.OnChain.CLNodesFundingETH); err != nil { - return err + if err := productConfigurators[productIdx].Store("env-out.toml", productInstance); err != nil { + return errors.New("failed to store product config") } } - contracts, err := configureContracts(in, c, auth, nodeClients, rootAddr, transmitters) - if err != nil { - return err - } - if err := configureJobs(in, nodeClients, contracts); err != nil { - return err - } - Plog.Info().Str("BootstrapNode", in.NodeSets[0].Out.CLNodes[0].Node.ExternalURL).Send() - for _, n := range in.NodeSets[0].Out.CLNodes[1:] { - Plog.Info().Str("Node", n.Node.ExternalURL).Send() - } - in.OnChain.DeployedContracts = contracts + } + L.Info().Str("BootstrapNode", in.NodeSets[0].Out.CLNodes[0].Node.ExternalURL).Send() + for _, n := range in.NodeSets[0].Out.CLNodes[1:] { + L.Info().Str("Node", n.Node.ExternalURL).Send() } return nil } @@ -2586,6 +2105,7 @@ type GrafanaDashboardParams struct { // ConfigTOMLParams default env.toml params type ConfigTOMLParams struct { PackageName string + ProductName string Nodes int NodeIndices []int } @@ -2626,9 +2146,15 @@ type ConfigParams struct { PackageName string } +// DevEnvInterfaceParams interface.go file params +type DevEnvInterfaceParams struct { + PackageName string +} + // EnvParams environment.go file params type EnvParams struct { PackageName string + ProductName string } // ProductConfigurationSimple product_configuration.go file params @@ -2660,7 +2186,6 @@ type EnvBuilder struct { outputDir string packageName string cliName string - productType string moduleName string } @@ -2670,14 +2195,13 @@ type EnvCodegen struct { } // NewEnvBuilder creates a new Chainlink Cluster developer environment -func NewEnvBuilder(cliName string, nodes int, productType string, productName string) *EnvBuilder { +func NewEnvBuilder(cliName string, nodes int, productName string) *EnvBuilder { return &EnvBuilder{ productName: productName, cliName: cliName, nodes: nodes, packageName: "devenv", outputDir: "devenv", - productType: productType, } } @@ -2727,7 +2251,7 @@ func (g *EnvCodegen) Read() error { return nil } -// Write generates a complete boilerplate, can be multiple files +// Write generates a complete devenv boilerplate, can be multiple files func (g *EnvCodegen) Write() error { // Create output directory if err := os.MkdirAll( //nolint:gosec @@ -2849,19 +2373,32 @@ func (g *EnvCodegen) Write() error { return fmt.Errorf("failed to write config file: %w", err) } - // Generate cldf.go - cldfContents, err := g.GenerateCLDF() + // Generate jd.go + cldfContents, err := g.GenerateJD() if err != nil { return err } if err := os.WriteFile( //nolint:gosec - filepath.Join(g.cfg.outputDir, "cldf.go"), + filepath.Join(g.cfg.outputDir, "jd.go"), []byte(cldfContents), os.ModePerm, ); err != nil { return fmt.Errorf("failed to write config file: %w", err) } + // Generate interface.go + interfaceContents, err := g.GenerateProductsInterface() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(g.cfg.outputDir, "interface.go"), + []byte(interfaceContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write products interface file: %w", err) + } + // Generate environment.go envFileContents, err := g.GenerateEnvironment() if err != nil { @@ -2875,26 +2412,6 @@ func (g *EnvCodegen) Write() error { return fmt.Errorf("failed to write environment file: %w", err) } - // Generate product_configuration.go - switch g.cfg.productType { - case "evm-single": - prodConfigFileContents, err := g.GenerateSingleNetworkProductConfiguration() - if err != nil { - return err - } - if err := os.WriteFile( //nolint:gosec - filepath.Join(g.cfg.outputDir, "product_configuration.go"), - []byte(prodConfigFileContents), - os.ModePerm, - ); err != nil { - return fmt.Errorf("failed to write product configuration file: %w", err) - } - case "multi-network": - return fmt.Errorf("product configuration 'multi-network' is not supported yet") - default: - return fmt.Errorf("unknown product configuration type: %s, known types are 'evm-single' or 'multi-network'", g.cfg.productType) - } - // create CI directory ciDir := filepath.Join(g.cfg.outputDir, ".github", "workflows") if err := os.MkdirAll( //nolint:gosec @@ -3000,26 +2517,6 @@ func (g *EnvCodegen) Write() error { return fmt.Errorf("failed to write gitignore file: %w", err) } - // tidy and finalize - currentDir, err := os.Getwd() - if err != nil { - return err - } - - // nolint - defer os.Chdir(currentDir) - if err := os.Chdir(g.cfg.outputDir); err != nil { - return err - } - log.Info().Msg("Downloading dependencies and running 'go mod tidy' ..") - _, err = exec.Command("go", "mod", "tidy").CombinedOutput() - if err != nil { - return fmt.Errorf("failed to tidy generated module: %w", err) - } - log.Info(). - Str("OutputDir", g.cfg.outputDir). - Str("Module", g.cfg.moduleName). - Msg("Developer environment generated") return nil } @@ -3112,6 +2609,7 @@ func (g *EnvCodegen) GenerateDefaultTOMLConfig() (string, error) { log.Info().Msg("Generating default environment config (env.toml)") p := ConfigTOMLParams{ PackageName: g.cfg.packageName, + ProductName: g.cfg.productName, Nodes: g.cfg.nodes, NodeIndices: make([]int, g.cfg.nodes), } @@ -3151,31 +2649,23 @@ func (g *EnvCodegen) GenerateCLI(dashboardUUID string) (string, error) { return render(CLITmpl, p) } -// GenerateSingleNetworkProductConfiguration generate a single-network EVM product configuration -func (g *EnvCodegen) GenerateSingleNetworkProductConfiguration() (string, error) { - log.Info().Msg("Configuring EVM network") - p := ProductConfigurationSimple{ - PackageName: g.cfg.packageName, - } - return render(SingleNetworkEVMProductConfigurationTmpl, p) -} - // GenerateEnvironment generate environment.go, our environment composition function func (g *EnvCodegen) GenerateEnvironment() (string, error) { log.Info().Msg("Generating environment composition (environment.go)") p := EnvParams{ PackageName: g.cfg.packageName, + ProductName: g.cfg.productName, } return render(EnvironmentTmpl, p) } -// GenerateCLDF generate CLDF helpers -func (g *EnvCodegen) GenerateCLDF() (string, error) { - log.Info().Msg("Generating CLDF helpers") +// GenerateJD generate JD helpers +func (g *EnvCodegen) GenerateJD() (string, error) { + log.Info().Msg("Generating JD helpers") p := CLDFParams{ PackageName: g.cfg.packageName, } - return render(CLDFTmpl, p) + return render(JDTmpl, p) } // GenerateDebugTools generate debug tools (tracing) @@ -3196,6 +2686,15 @@ func (g *EnvCodegen) GenerateConfig() (string, error) { return render(ConfigTmpl, p) } +// GenerateProductsInterface generate devenv interface to run arbitrary products +func (g *EnvCodegen) GenerateProductsInterface() (string, error) { + log.Info().Msg("Generating devenv interface") + p := DevEnvInterfaceParams{ + PackageName: g.cfg.packageName, + } + return render(ProductsInterfaceTmpl, p) +} + // GenerateTableTest generates all possible experiments for a namespace // first generate all small pieces then insert into a table test template func (g *EnvCodegen) GenerateTableTest() (string, error) { diff --git a/framework/tmpl_gen_product.go b/framework/tmpl_gen_product.go new file mode 100644 index 000000000..b23f1fadd --- /dev/null +++ b/framework/tmpl_gen_product.go @@ -0,0 +1,373 @@ +package framework + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +/* Templates */ + +const ( + ProductSoakConfigTmpl = `# This file describes how many instances of your product we should deploy for soak test +# you can also override keys from other configs here, for example your [[{{ .ProductName }}]] or [[blockchains]] / [[nodesets]] +[[products]] +name = "ocr2" +instances = 1 +` + + ProductBasicConfigTmpl = `[[{{ .ProductName}}]] +# TODO: define your product configuration here, see configurator.go ProductConfig` + + ProductsImplTmpl = `package {{ .ProductName }} + +import ( + "context" + "fmt" + "os" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake" + nodeset "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" + "github.com/smartcontractkit/{{ .ProductName }}/devenv/products" +) + +var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "{{ .ProductName }}"}).Logger() + +type ProductConfig struct{} + +type Configurator struct { + Config []*ProductConfig ` + "`" + `toml:"productone"` + "`" + ` +} + +func NewConfigurator() *Configurator { + return &Configurator{} +} + +func (m *Configurator) Load() error { + cfg, err := products.Load[Configurator]() + if err != nil { + return fmt.Errorf("failed to load product config: %w", err) + } + m.Config = cfg.Config + return nil +} + +func (m *Configurator) Store(path string, idx int) error { + if err := products.Store(".", m); err != nil { + return fmt.Errorf("failed to store product config: %w", err) + } + return nil +} + +func (m *Configurator) GenerateNodesConfig( + ctx context.Context, + fs *fake.Input, + bc *blockchain.Input, + ns *nodeset.Input, +) (string, error) { + L.Info().Msg("Generating Chainlink node config") + // node + _ = bc.Out.Nodes[0] + // chain ID + _ = bc.Out.ChainID + return "", nil +} + +func (m *Configurator) GenerateNodesSecrets( + ctx context.Context, + fs *fake.Input, + bc *blockchain.Input, + ns *nodeset.Input, +) (string, error) { + L.Info().Msg("Generating Chainlink node secrets") + // node + _ = bc.Out.Nodes[0] + // chain ID + _ = bc.Out.ChainID + return "", nil +} + +func (m *Configurator) ConfigureJobsAndContracts( + ctx context.Context, + fake *fake.Input, + bc *blockchain.Input, + ns *nodeset.Input, +) error { + L.Info().Msg("Configuring product: productone") + return nil +} +` + + ProductsConfigTmpl = `package products + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +const ( + EnvVarTestConfigs = "CTF_CONFIGS" +) + +var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "product_config"}).Logger() + +func Load[T any]() (*T, error) { + var config T + paths := strings.Split(os.Getenv(EnvVarTestConfigs), ",") + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read product config file path %s: %w", path, err) + } + L.Trace().Str("ProductConfig", string(data)).Send() + + decoder := toml.NewDecoder(strings.NewReader(string(data))) + + if err := decoder.Decode(&config); err != nil { + return nil, fmt.Errorf("failed to decode TOML config, strict mode: %w", err) + } + } + return &config, nil +} + +// Store writes config to a file, adds -cache.toml suffix if it's an initial configuration. +func Store[T any](path string, cfg *T) error { + baseConfigPath, err := BaseConfigPath(EnvVarTestConfigs) + if err != nil { + return err + } + newCacheName := strings.ReplaceAll(baseConfigPath, ".toml", "") + var outCacheName string + if strings.Contains(newCacheName, "cache") { + L.Info().Str("Cache", baseConfigPath).Msg("Cache file already exists, overriding") + outCacheName = baseConfigPath + } else { + outCacheName = strings.ReplaceAll(baseConfigPath, ".toml", "") + "-out.toml" + } + L.Info().Str("OutputFile", outCacheName).Msg("Storing configuration output") + d, err := toml.Marshal(cfg) + if err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(path, outCacheName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + if _, err := f.Write(d); err != nil { + return err + } + return nil +} + +// LoadOutput loads config output file from path. +func LoadOutput[T any](path string) (*T, error) { + _ = os.Setenv(EnvVarTestConfigs, path) + return Load[T]() +} + +// BaseConfigPath returns base config path, ex. env.toml,overrides.toml -> env.toml. +func BaseConfigPath(envVar string) (string, error) { + configs := os.Getenv(envVar) + if configs == "" { + return "", fmt.Errorf("no %s env var is provided, you should provide at least one test config in TOML", envVar) + } + L.Debug().Str("Configs", configs).Msg("Getting base config path") + return strings.Split(configs, ",")[0], nil +} +` + ProductsCommonTmpl = `package products + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + + "github.com/ethereum/go-ethereum/core/types" +) + +// WaitMinedFast is a method for Anvil's instant blocks mode to ovecrome bind.WaitMined ticker hardcode. +func WaitMinedFast(ctx context.Context, b bind.DeployBackend, txHash common.Hash) (*types.Receipt, error) { + queryTicker := time.NewTicker(5 * time.Millisecond) + defer queryTicker.Stop() + for { + receipt, err := b.TransactionReceipt(ctx, txHash) + if err == nil { + return receipt, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-queryTicker.C: + } + } +} + ` +) + +type ProductCommonParams struct{} + +func (g *EnvCodegen) GenerateProductCommon() (string, error) { + log.Info().Msg("Generating products common") + p := ProductCommonParams{} + return render(ProductsCommonTmpl, p) +} + +type ProductConfigParams struct{} + +func (g *EnvCodegen) GenerateProductsConfig() (string, error) { + log.Info().Msg("Generating products config") + p := ProductCommonParams{} + return render(ProductsConfigTmpl, p) +} + +type ProductImplParams struct { + ProductName string +} + +func (g *EnvCodegen) GenerateProductImpl() (string, error) { + log.Info().Msg("Generating product implementation") + p := ProductImplParams{ + ProductName: g.cfg.productName, + } + return render(ProductsImplTmpl, p) +} + +type ProductBasicConfigParams struct { + ProductName string +} + +func (g *EnvCodegen) GenerateProductBasicConfigParams() (string, error) { + log.Info().Msg("Generating product basic config") + p := ProductBasicConfigParams{ + ProductName: g.cfg.productName, + } + return render(ProductBasicConfigTmpl, p) +} + +type ProductSoakConfigParams struct { + ProductName string +} + +func (g *EnvCodegen) GenerateProductSoakConfigParams() (string, error) { + log.Info().Msg("Generating product soak config") + p := ProductSoakConfigParams{ + ProductName: g.cfg.productName, + } + return render(ProductSoakConfigTmpl, p) +} + +// WriteProducts generates a complete products boilerplate +func (g *EnvCodegen) WriteProducts() error { + productsRoot := filepath.Join(g.cfg.outputDir, "products") + productRoot := filepath.Join(productsRoot, g.cfg.productName) + // Create products directory with one product + if err := os.MkdirAll( //nolint:gosec + productRoot, + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to create products directory: %w", err) + } + + // generate common.go + commonContents, err := g.GenerateProductCommon() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(productsRoot, "common.go"), + []byte(commonContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write products common file: %w", err) + } + + // generate config.go + cfgContents, err := g.GenerateProductsConfig() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(productsRoot, "config.go"), + []byte(cfgContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write product config file: %w", err) + } + + // generate configurator.go (product implementation) + productImplContents, err := g.GenerateProductImpl() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(productRoot, "configurator.go"), + []byte(productImplContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write product implementation file: %w", err) + } + + // generate basic TOML config for product + basicCfgContents, err := g.GenerateProductBasicConfigParams() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(productRoot, "basic.toml"), + []byte(basicCfgContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write product basic config file: %w", err) + } + + // generate soak TOML config for product + soakCfgContents, err := g.GenerateProductSoakConfigParams() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(productRoot, "soak.toml"), + []byte(soakCfgContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write product soak config file: %w", err) + } + + // tidy and finalize + currentDir, err := os.Getwd() + if err != nil { + return err + } + + // nolint + defer os.Chdir(currentDir) + if err := os.Chdir(g.cfg.outputDir); err != nil { + return err + } + log.Info().Msg("Downloading dependencies and running 'go mod tidy' ..") + _, err = exec.Command("go", "mod", "tidy").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to tidy generated module: %w", err) + } + log.Info(). + Str("OutputDir", g.cfg.outputDir). + Str("Module", g.cfg.moduleName). + Msg("Developer environment generated") + return nil +} diff --git a/framework/tmpl_gen_product_fakes.go b/framework/tmpl_gen_product_fakes.go new file mode 100644 index 000000000..daead4ff0 --- /dev/null +++ b/framework/tmpl_gen_product_fakes.go @@ -0,0 +1,226 @@ +package framework + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/rs/zerolog/log" +) + +/* Templates */ + +const ( + ProductFakesJustfile = `IMAGE_NAME := "{{ .ProductName }}-fakes" + +run: + docker run --rm -it -v $(pwd):/app -p 9111:9111 {{ "{{" }}IMAGE_NAME{{ "}}" }}:latest + +build: + docker build -f Dockerfile -t {{ "{{" }}IMAGE_NAME{{ "}}" }}:latest . + +push registry: + docker build --platform linux/amd64 -f Dockerfile -t {{ "{{" }}IMAGE_NAME{{ "}}" }}:latest . + aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin {{ "{{" }}registry{{ "}}" }} + docker tag {{ "{{" }}IMAGE_NAME{{ "}}" }}:latest {{ "{{" }}registry{{ "}}" }}/{{ "{{" }}IMAGE_NAME{{ "}}" }} + docker push {{ "{{" }}registry{{ "}}" }}/{{ "{{" }}IMAGE_NAME{{ "}}" }} + +clean: + docker rmi {{ "{{" }}IMAGE_NAME{{ "}}" }}:latest +` + ProductFakesImplTmpl = `package main + +import ( + "os" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/gin-gonic/gin" + + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake" +) + +var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "ocr2"}).Logger() + +// example mock service +func main() { + _, err := fake.NewFakeDataProvider(&fake.Input{Port: fake.DefaultFakeServicePort}) + if err != nil { + panic(err) + } + err = fake.Func("POST", "/example_fake", func(ctx *gin.Context) { + ctx.JSON(200, gin.H{ + "data": map[string]any{ + "result": "ok", + }, + }) + }) + if err != nil { + panic(err) + } + select {} +} +` + ProductFakesGoModuleTmpl = `module github.com/smartcontractkit/{{ .ProductName}}/devenv/fakes + +go {{.RuntimeVersion}} + +require ( + github.com/gin-gonic/gin v1.10.1 + github.com/rs/zerolog v1.34.0 + github.com/smartcontractkit/chainlink-testing-framework/framework v0.10.1 + github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.1-0.20250711120409-5078050f9db4 +)` + + ProductFakesDockerfileTmpl = `FROM golang:1.25 AS builder + +ENV GOPRIVATE=github.com/smartcontractkit/* + +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY ../.. . +RUN CGO_ENABLED=0 GOOS=linux go build -o /fake main.go + +FROM alpine:latest +RUN apk --no-cache add ca-certificates +COPY --from=builder /fake /fake +EXPOSE 9111 +CMD ["/fake"] +` +) + +type ProductFakesImplParams struct { + ProductName string +} + +func (g *EnvCodegen) GenerateFakesImpl() (string, error) { + log.Info().Msg("Generating fakes implementation") + p := ProductFakesImplParams{} + return render(ProductFakesImplTmpl, p) +} + +type ProductFakesJustfileParams struct { + ProductName string +} + +func (g *EnvCodegen) GenerateFakesJustfile() (string, error) { + log.Info().Msg("Generating fakes Justfile") + p := ProductFakesJustfileParams{ + ProductName: g.cfg.productName, + } + return render(ProductFakesJustfile, p) +} + +type ProductFakesDockerfileParams struct{} + +func (g *EnvCodegen) GenerateFakesDockerfile() (string, error) { + log.Info().Msg("Generating fakes Dockerfile") + p := ProductFakesDockerfileParams{} + return render(ProductFakesDockerfileTmpl, p) +} + +type ProductFakesGoModuleParams struct { + ProductName string + RuntimeVersion string +} + +func (g *EnvCodegen) GenerateFakesGoModule() (string, error) { + log.Info().Msg("Generating fakes go.mod") + p := ProductFakesGoModuleParams{ + ProductName: g.cfg.productName, + RuntimeVersion: strings.ReplaceAll(runtime.Version(), "go", ""), + } + return render(ProductFakesGoModuleTmpl, p) +} + +// WriteFakes writes all files related to fake services used in testing +func (g *EnvCodegen) WriteFakes() error { + fakesRoot := filepath.Join(g.cfg.outputDir, "fakes") + if err := os.MkdirAll( //nolint:gosec + fakesRoot, + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to create fakes directory: %w", err) + } + + // generate Dockerfile + dockerfileContents, err := g.GenerateFakesDockerfile() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(fakesRoot, "Dockerfile"), + []byte(dockerfileContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write fakes Dockerfile file: %w", err) + } + + // generate fakes go.mod + fakesGoModContents, err := g.GenerateFakesGoModule() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(fakesRoot, "go.mod"), + []byte(fakesGoModContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write fakes Go module file: %w", err) + } + + // generate fakes implementation + implContents, err := g.GenerateFakesImpl() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(fakesRoot, "main.go"), + []byte(implContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write fakes implementation file: %w", err) + } + + // generate fakes Justfile + justfileContents, err := g.GenerateFakesJustfile() + if err != nil { + return err + } + if err := os.WriteFile( //nolint:gosec + filepath.Join(fakesRoot, "Justfile"), + []byte(justfileContents), + os.ModePerm, + ); err != nil { + return fmt.Errorf("failed to write fakes Just file: %w", err) + } + + // tidy and finalize + currentDir, err := os.Getwd() + if err != nil { + return err + } + + // nolint + defer os.Chdir(currentDir) + if err := os.Chdir(fakesRoot); err != nil { + return err + } + log.Info().Msg("Downloading dependencies and running 'go mod tidy' (fakes) ..") + _, err = exec.Command("go", "mod", "tidy").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to tidy generated module for fakes: %w", err) + } + + log.Info().Msg("Building fakes image") + _, err = exec.Command("just", "build").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to build fakes Docker image: %w", err) + } + return nil +} diff --git a/framework/tmpl_gen_test.go b/framework/tmpl_gen_test.go index 1002fdc39..e937bbfa4 100644 --- a/framework/tmpl_gen_test.go +++ b/framework/tmpl_gen_test.go @@ -44,7 +44,6 @@ func TestSmokeGenerateDevEnv(t *testing.T) { name string cliName string productName string - productType string outputDir string nodes int }{ @@ -54,7 +53,6 @@ func TestSmokeGenerateDevEnv(t *testing.T) { name: "basic 2 nodes env", cliName: "tcli", productName: "myproduct", - productType: "evm-single", outputDir: "test-env", nodes: 2, }, @@ -64,12 +62,11 @@ func TestSmokeGenerateDevEnv(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Cleanup(func() { runCmd(t, tt.outputDir, tt.cliName, `down`) - os.RemoveAll(tt.outputDir) + // os.RemoveAll(tt.outputDir) }) cg, err := framework.NewEnvBuilder( tt.cliName, tt.nodes, - tt.productType, tt.productName, ). OutputDir(tt.outputDir). @@ -77,6 +74,10 @@ func TestSmokeGenerateDevEnv(t *testing.T) { require.NoError(t, err) err = cg.Write() require.NoError(t, err) + err = cg.WriteFakes() + require.NoError(t, err) + err = cg.WriteProducts() + require.NoError(t, err) runCmd(t, filepath.Join(tt.outputDir, "cmd", tt.cliName), `go`, `install`, `.`) runCmd(t, tt.outputDir, tt.cliName, `up`) From e6ae798bd8d07529f198cf449b565693b9f97c72 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 19 Jan 2026 15:53:24 +0100 Subject: [PATCH 02/14] changeset --- framework/.changeset/v0.13.4.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 framework/.changeset/v0.13.4.md diff --git a/framework/.changeset/v0.13.4.md b/framework/.changeset/v0.13.4.md new file mode 100644 index 000000000..8b8262d38 --- /dev/null +++ b/framework/.changeset/v0.13.4.md @@ -0,0 +1,2 @@ +- Multi-product env template +- Split product and infra configuration code and files \ No newline at end of file From e30844b9fb19f89c6c67116fe30c6e6563a135a8 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 19 Jan 2026 15:55:34 +0100 Subject: [PATCH 03/14] fix test cleanup --- framework/tmpl_gen_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/tmpl_gen_test.go b/framework/tmpl_gen_test.go index e937bbfa4..dfa9cbb57 100644 --- a/framework/tmpl_gen_test.go +++ b/framework/tmpl_gen_test.go @@ -62,7 +62,7 @@ func TestSmokeGenerateDevEnv(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Cleanup(func() { runCmd(t, tt.outputDir, tt.cliName, `down`) - // os.RemoveAll(tt.outputDir) + os.RemoveAll(tt.outputDir) }) cg, err := framework.NewEnvBuilder( tt.cliName, From a1d20f9bea4d4587999851c4263fc05a09d82255 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 19 Jan 2026 16:07:36 +0100 Subject: [PATCH 04/14] add Just to CI --- .github/workflows/framework-codegen.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/framework-codegen.yml b/.github/workflows/framework-codegen.yml index c7d771eb3..4edd8cbba 100644 --- a/.github/workflows/framework-codegen.yml +++ b/.github/workflows/framework-codegen.yml @@ -51,6 +51,10 @@ jobs: go-modules-${{ runner.os }} - name: Install dependencies run: go mod download + - name: Install Just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + with: + just-version: "1.40.0" - name: Run Codegen Tests run: | go test -timeout ${{ matrix.test.timeout }} -v -count ${{ matrix.test.count }} -run ${{ matrix.test.name }} From 52cef86728c3041f1aace6b4e9d31046467760e6 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 19 Jan 2026 16:21:18 +0100 Subject: [PATCH 05/14] fix config --- .gitignore | 4 +++- framework/tmpl_gen_env.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4da9be63e..1b7c829dc 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,6 @@ tag.py parrot/*.json parrot/*.log # Executable -parrot/parrot \ No newline at end of file +parrot/parrot +# Devenv (generated manually) +devenv/ \ No newline at end of file diff --git a/framework/tmpl_gen_env.go b/framework/tmpl_gen_env.go index 2c1c1bfa4..76594c57e 100644 --- a/framework/tmpl_gen_env.go +++ b/framework/tmpl_gen_env.go @@ -855,7 +855,7 @@ name = "{{ .ProductName }}" instances = 1 [fake_server] - image = "ocr2-fakes:latest" + image = "{{ .ProductName }}-fakes:latest" port = 9111 [[blockchains]] From 8032400de7a68ddd7979fe015d9de5bb813647b9 Mon Sep 17 00:00:00 2001 From: skudasov Date: Mon, 19 Jan 2026 16:49:18 +0100 Subject: [PATCH 06/14] metrics for default 2h test --- framework/leak/detector_cl_node.go | 6 +++--- framework/leak/detector_hog_test.go | 17 +++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/framework/leak/detector_cl_node.go b/framework/leak/detector_cl_node.go index d12aca307..a42d615f3 100644 --- a/framework/leak/detector_cl_node.go +++ b/framework/leak/detector_cl_node.go @@ -55,9 +55,9 @@ func NewCLNodesLeakDetector(c *ResourceLeakChecker, opts ...func(*CLNodesLeakDet } switch cd.Mode { case "devenv": - // aggregate it on 5m interval with 2m step for mitigating spikes - cd.CPUQuery = `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="don-node%d"}[5m])) * 100)[5m:2m])` - cd.MemoryQuery = `avg_over_time(container_memory_rss{name="don-node%d"}[5m]) / 1024 / 1024` + // avg from intervals of 30m with 5m step to mitigate spikes + cd.CPUQuery = `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="don-node%d"}[30m])))[30m:5m]) * 100` + cd.MemoryQuery = `avg_over_time(container_memory_rss{name="don-node%d"}[30m]) / 1024 / 1024` case "griddle": return nil, fmt.Errorf("not implemented yet") default: diff --git a/framework/leak/detector_hog_test.go b/framework/leak/detector_hog_test.go index 331744c7f..865c7a745 100644 --- a/framework/leak/detector_hog_test.go +++ b/framework/leak/detector_hog_test.go @@ -16,7 +16,7 @@ import ( ) func TestCyclicHog(t *testing.T) { - // t.Skip("unskip when debugging new queries") + t.Skip("unskip when debugging new queries") ctx := context.Background() hog, err := SetupResourceHog( ctx, @@ -41,19 +41,20 @@ func TestVerifyCyclicHog(t *testing.T) { lc := leak.NewResourceLeakChecker() // cpu diff, err := lc.MeasureDelta(&leak.CheckConfig{ - Query: `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="resource-hog"}[5m])) * 100)[5m:2m])`, - Start: mustTime("2026-01-16T13:20:30Z"), - End: mustTime("2026-01-16T13:32:40Z"), - WarmUpDuration: 2 * time.Minute, + Query: `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="resource-hog"}[30m])))[30m:5m]) * 100`, + Start: mustTime("2026-01-19T10:30:00Z"), + End: mustTime("2026-01-19T12:29:15Z"), + WarmUpDuration: 10 * time.Minute, }) fmt.Println(diff) require.NoError(t, err) // mem diff, err = lc.MeasureDelta(&leak.CheckConfig{ - Query: `avg_over_time(container_memory_rss{name="resource-hog"}[5m]) / 1024 / 1024`, - Start: mustTime("2026-01-16T13:20:30Z"), - End: mustTime("2026-01-16T13:38:25Z"), + Query: `avg_over_time(container_memory_rss{name="resource-hog"}[30m]) / 1024 / 1024`, + Start: mustTime("2026-01-19T10:30:00Z"), + End: mustTime("2026-01-19T12:29:15Z"), + WarmUpDuration: 10 * time.Minute, }) fmt.Println(diff) require.NoError(t, err) From c97af5f24fbd7fd63793e4e5b469378f6e036166 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 12:09:11 +0100 Subject: [PATCH 07/14] download pprof via profilecli --- framework/leak/detector_cl_node.go | 59 +++++++++- framework/leak/detector_test.go | 17 ++- framework/leak/profilecli.go | 170 +++++++++++++++++++++++++++++ framework/observability.go | 1 + 4 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 framework/leak/profilecli.go diff --git a/framework/leak/detector_cl_node.go b/framework/leak/detector_cl_node.go index a42d615f3..6528a8ca9 100644 --- a/framework/leak/detector_cl_node.go +++ b/framework/leak/detector_cl_node.go @@ -3,6 +3,7 @@ package leak import ( "errors" "fmt" + "strconv" "time" "github.com/smartcontractkit/chainlink-testing-framework/framework" @@ -23,9 +24,9 @@ type CLNodesCheck struct { // CLNodesLeakDetector is Chainlink node specific resource leak detector // can be used with both local and remote Chainlink node sets (DONs) type CLNodesLeakDetector struct { - Mode string - CPUQuery, MemoryQuery string - c *ResourceLeakChecker + Mode string + CPUQuery, MemoryQuery, ContainerAliveQuery string + c *ResourceLeakChecker } // WithCPUQuery allows to override CPU leak query (Prometheus) @@ -55,9 +56,10 @@ func NewCLNodesLeakDetector(c *ResourceLeakChecker, opts ...func(*CLNodesLeakDet } switch cd.Mode { case "devenv": + cd.ContainerAliveQuery = `time() - container_start_time_seconds{name=~"don-node%d"}` // avg from intervals of 30m with 5m step to mitigate spikes - cd.CPUQuery = `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="don-node%d"}[30m])))[30m:5m]) * 100` - cd.MemoryQuery = `avg_over_time(container_memory_rss{name="don-node%d"}[30m]) / 1024 / 1024` + cd.CPUQuery = `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="don-node%d"}[1h])))[1h:30m]) * 100` + cd.MemoryQuery = `avg_over_time(container_memory_rss{name="don-node%d"}[1h:30m]) / 1024 / 1024` case "griddle": return nil, fmt.Errorf("not implemented yet") default: @@ -66,6 +68,31 @@ func NewCLNodesLeakDetector(c *ResourceLeakChecker, opts ...func(*CLNodesLeakDet return cd, nil } +func (cd *CLNodesLeakDetector) checkContainerUptime(t *CLNodesCheck, nodeIdx int) (float64, error) { + uptimeResp, err := cd.c.c.Query(fmt.Sprintf(cd.ContainerAliveQuery, nodeIdx), t.End) + if err != nil { + return 0, fmt.Errorf("failed to execute container alive query: %w", err) + } + uptimeResult := uptimeResp.Data.Result + if len(uptimeResult) == 0 { + return 0, fmt.Errorf("no results for end timestamp: %s", t.End) + } + + uptimeResultValue, resOk := uptimeResult[0].Value[1].(string) + if !resOk { + return 0, fmt.Errorf("invalid Prometheus response value for timestamp: %s, value: %v", t.End, uptimeResult[0].Value[1]) + } + + uptimeResultValueFloat, err := strconv.ParseFloat(uptimeResultValue, 64) + if err != nil { + return 0, fmt.Errorf("uptime can't be parsed from string: %w", err) + } + if uptimeResultValueFloat <= float64(t.End.Unix())-float64(t.Start.Unix()) { + return uptimeResultValueFloat, fmt.Errorf("container hasn't lived long enough and was killed while the test was running") + } + return uptimeResultValueFloat, nil +} + // Check runs all resource leak checks and returns errors if threshold reached for any of them func (cd *CLNodesLeakDetector) Check(t *CLNodesCheck) error { if t.NumNodes == 0 { @@ -73,6 +100,7 @@ func (cd *CLNodesLeakDetector) Check(t *CLNodesCheck) error { } memoryDiffs := make([]float64, 0) cpuDiffs := make([]float64, 0) + uptimes := make([]float64, 0) errs := make([]error, 0) for i := range t.NumNodes { memoryDiff, err := cd.c.MeasureDelta(&CheckConfig{ @@ -108,10 +136,31 @@ func (cd *CLNodesLeakDetector) Check(t *CLNodesCheck) error { i, t.Start, t.End, cpuDiff, )) } + uptime, err := cd.checkContainerUptime(t, i) + if err != nil { + errs = append(errs, fmt.Errorf( + "Container uptime issue for node %d and interval: [%s -> %s], uptime: %.f, err: %w", + i, t.Start, t.End, uptime, err, + )) + } + uptimes = append(uptimes, uptime) } framework.L.Info(). Any("MemoryDiffs", memoryDiffs). Any("CPUDiffs", cpuDiffs). + Any("Uptimes", uptimes). + Str("TestDuration", t.End.Sub(t.Start).String()). + Float64("TestDurationSec", t.End.Sub(t.Start).Seconds()). Msg("Leaks info") + framework.L.Info().Msg("Downloading pprof profile..") + dumper := NewProfileDumper(framework.LocalPyroscopeBaseURL) + profilePath, err := dumper.MemoryProfile(&ProfileDumperConfig{ + ServiceName: "chainlink-node", + }) + if err != nil { + errs = append(errs, fmt.Errorf("failed to download Pyroscopt profile: %w", err)) + return errors.Join(errs...) + } + framework.L.Info().Str("Path", profilePath).Msg("Saved pprof profile") return errors.Join(errs...) } diff --git a/framework/leak/detector_test.go b/framework/leak/detector_test.go index 3300fff18..049bda564 100644 --- a/framework/leak/detector_test.go +++ b/framework/leak/detector_test.go @@ -2,6 +2,7 @@ package leak_test import ( "fmt" + "strconv" "testing" "time" @@ -20,6 +21,14 @@ func mustTime(start string) time.Time { return s } +func mustUnixEpoch(start string) string { + s, err := time.Parse(time.RFC3339, start) + if err != nil { + panic("can't convert time from RFC3339") + } + return strconv.Itoa(int(s.Unix())) +} + func TestMeasure(t *testing.T) { qc := leak.NewFakeQueryClient() lc := leak.NewResourceLeakChecker(leak.WithQueryClient(qc)) @@ -124,15 +133,15 @@ func TestMeasure(t *testing.T) { } func TestRealCLNodesLeakDetectionLocalDevenv(t *testing.T) { - t.Skip(`this test requires a real load run, see docs here https://github.com/smartcontractkit/chainlink/tree/develop/devenv, spin up the env and run "cl test load"`) + // t.Skip(`this test requires a real load run, see docs here https://github.com/smartcontractkit/chainlink/tree/develop/devenv, spin up the env and run "cl test load"`) cnd, err := leak.NewCLNodesLeakDetector(leak.NewResourceLeakChecker()) require.NoError(t, err) errs := cnd.Check(&leak.CLNodesCheck{ NumNodes: 4, - Start: mustTime("2026-01-15T01:14:00Z"), - End: mustTime("2026-01-15T02:04:00Z"), - CPUThreshold: 20.0, + Start: mustTime("2026-01-19T17:23:14Z"), + End: mustTime("2026-01-19T18:00:51Z"), + CPUThreshold: 100.0, MemoryThreshold: 20.0, }) require.NoError(t, errs) diff --git a/framework/leak/profilecli.go b/framework/leak/profilecli.go new file mode 100644 index 000000000..cc479027d --- /dev/null +++ b/framework/leak/profilecli.go @@ -0,0 +1,170 @@ +package leak + +/* + * This is a simple utility to download last 12h pprof for memory using Pyroscope's profilecli. + */ + +import ( + "fmt" + "os" + "os/exec" + "runtime" + + f "github.com/smartcontractkit/chainlink-testing-framework/framework" +) + +const ( + DefaultPyroscopeBinaryVersion = "1.18.0" + DefaultOutputPath = "alloc.pprof" + DefaultProfileType = "memory:alloc_space:bytes:space:bytes" + DefaultFrom = "now-12h" + DefaultTo = "now" +) + +// ProfileDumperConfig describes profile dump configuration +type ProfileDumperConfig struct { + PyroscopeURL string + ServiceName string + ProfileType string + From string // Relative time like "now-30m" or unix timestamp (seconds) + To string // Relative time like "now" or unix timestamp (seconds) + OutputPath string // Output file path +} + +// ProfileDumper is responsible for downloading profiles from Pyroscope +type ProfileDumper struct { + pyroscopeURL string + profileCLIPath string +} + +// NewProfileDumper creates a new profile dumper instance +func NewProfileDumper(pyroscopeURL string, opts ...func(*ProfileDumper)) *ProfileDumper { + dumper := &ProfileDumper{ + pyroscopeURL: pyroscopeURL, + profileCLIPath: "", + } + for _, opt := range opts { + opt(dumper) + } + return dumper +} + +// WithProfileCLIPath sets a custom path to profilecli binary +func WithProfileCLIPath(path string) func(*ProfileDumper) { + return func(d *ProfileDumper) { + d.profileCLIPath = path + } +} + +// InstallProfileCLI downloads and installs profilecli if not already available +func (d *ProfileDumper) InstallProfileCLI() (string, error) { + // Check if profilecli is already available + if d.profileCLIPath != "" { + if _, err := os.Stat(d.profileCLIPath); err == nil { + f.L.Info().Str("path", d.profileCLIPath).Msg("Using existing profilecli") + return d.profileCLIPath, nil + } + } + if path, err := exec.LookPath("profilecli"); err == nil { + f.L.Info().Str("path", path).Msg("profilecli found in PATH") + d.profileCLIPath = path + return path, nil + } + + // detect OS + osName := runtime.GOOS + if osName != "darwin" && osName != "linux" { + return "", fmt.Errorf("unsupported OS: %s", osName) + } + arch := runtime.GOARCH + if arch != "amd64" && arch != "arm64" { + return "", fmt.Errorf("unsupported architecture: %s", arch) + } + + downloadURL := fmt.Sprintf( + "https://github.com/grafana/pyroscope/releases/download/v%s/profilecli_%s_%s_%s.tar.gz", + DefaultPyroscopeBinaryVersion, DefaultPyroscopeBinaryVersion, osName, arch, + ) + + f.L.Info().Str("url", downloadURL).Msg("Downloading profilecli") + + // Download and extract in current directory + cmd := exec.Command("sh", "-c", fmt.Sprintf("curl -fL %s | tar xvz", downloadURL)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to download and extract profilecli: %w", err) + } + binaryPath := "./profilecli" + if err := os.Chmod(binaryPath, 0o755); err != nil { + return "", fmt.Errorf("failed to make profilecli executable: %w", err) + } + + d.profileCLIPath = binaryPath + f.L.Info().Str("path", binaryPath).Msg("profilecli ready to use") + return binaryPath, nil +} + +func defaults(config *ProfileDumperConfig) { + if config.OutputPath == "" { + config.OutputPath = DefaultOutputPath + } + if config.ProfileType == "" { + config.ProfileType = DefaultProfileType + } + if config.From == "" { + config.From = DefaultFrom + } + if config.To == "" { + config.To = DefaultTo + } +} + +// DownloadProfile runs profilecli to download a profile +func (d *ProfileDumper) DownloadProfile(config *ProfileDumperConfig) (string, error) { + defaults(config) + + cmdArgs := []string{ + "query", "merge", + fmt.Sprintf("--query={service_name=\"%s\"}", config.ServiceName), + fmt.Sprintf("--profile-type=%s", config.ProfileType), + fmt.Sprintf("--from=%s", config.From), + fmt.Sprintf("--to=%s", config.To), + fmt.Sprintf("--output=pprof=%s", config.OutputPath), + } + + env := os.Environ() + env = append(env, fmt.Sprintf("PROFILECLI_URL=%s", d.pyroscopeURL)) + cmd := exec.Command(d.profileCLIPath, cmdArgs...) + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + f.L.Info(). + Str("command", cmd.String()). + Str("output", config.OutputPath). + Msg("Downloading profile from Pyroscope") + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to download profile: %w", err) + } + if _, err := os.Stat(config.OutputPath); err != nil { + return "", fmt.Errorf("profile file not created: %w", err) + } + + fileInfo, _ := os.Stat(config.OutputPath) + f.L.Info(). + Str("path", config.OutputPath). + Str("size", fmt.Sprintf("%d bytes", fileInfo.Size())). + Msg("Profile downloaded successfully") + return config.OutputPath, nil +} + +// MemoryProfile downloads memory profile and saves it +func (d *ProfileDumper) MemoryProfile(config *ProfileDumperConfig) (string, error) { + if _, err := d.InstallProfileCLI(); err != nil { + return "", fmt.Errorf("installation failed: %w", err) + } + return d.DownloadProfile(config) +} diff --git a/framework/observability.go b/framework/observability.go index dd2c8a35a..31e65fc60 100644 --- a/framework/observability.go +++ b/framework/observability.go @@ -16,6 +16,7 @@ const ( LocalGrafanaBaseURL = "http://localhost:3000" LocalLokiBaseURL = "http://localhost:3030" LocalPrometheusBaseURL = "http://localhost:9099" + LocalPyroscopeBaseURL = "http://localhost:4040" LocalCLNodeErrorsURL = "http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1&refresh=5s" LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=5s&from=now-15m&to=now" LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22ctf%5C%22%7D%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" From d8b2642d0625f32a5ca0a6d85dd4fc5bdb35f1a5 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 12:13:13 +0100 Subject: [PATCH 08/14] fix comment --- framework/leak/detector_cl_node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/leak/detector_cl_node.go b/framework/leak/detector_cl_node.go index 6528a8ca9..3da7620de 100644 --- a/framework/leak/detector_cl_node.go +++ b/framework/leak/detector_cl_node.go @@ -57,7 +57,7 @@ func NewCLNodesLeakDetector(c *ResourceLeakChecker, opts ...func(*CLNodesLeakDet switch cd.Mode { case "devenv": cd.ContainerAliveQuery = `time() - container_start_time_seconds{name=~"don-node%d"}` - // avg from intervals of 30m with 5m step to mitigate spikes + // avg from intervals of 1h with 30m step to mitigate spikes cd.CPUQuery = `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="don-node%d"}[1h])))[1h:30m]) * 100` cd.MemoryQuery = `avg_over_time(container_memory_rss{name="don-node%d"}[1h:30m]) / 1024 / 1024` case "griddle": From 557e1b309d5294ae822b465c245f1c0cc3ccb387 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 12:14:47 +0100 Subject: [PATCH 09/14] skip manual test --- framework/leak/detector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/leak/detector_test.go b/framework/leak/detector_test.go index 049bda564..e5a055b71 100644 --- a/framework/leak/detector_test.go +++ b/framework/leak/detector_test.go @@ -133,7 +133,7 @@ func TestMeasure(t *testing.T) { } func TestRealCLNodesLeakDetectionLocalDevenv(t *testing.T) { - // t.Skip(`this test requires a real load run, see docs here https://github.com/smartcontractkit/chainlink/tree/develop/devenv, spin up the env and run "cl test load"`) + t.Skip(`this test requires a real load run, see docs here https://github.com/smartcontractkit/chainlink/tree/develop/devenv, spin up the env and run "cl test load"`) cnd, err := leak.NewCLNodesLeakDetector(leak.NewResourceLeakChecker()) require.NoError(t, err) From cfd298403e08240ac265415ab2ce528bc69ff383 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 12:25:10 +0100 Subject: [PATCH 10/14] cleanup, turn on leak unit tests --- framework/leak/detector_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/framework/leak/detector_test.go b/framework/leak/detector_test.go index e5a055b71..f6af5e740 100644 --- a/framework/leak/detector_test.go +++ b/framework/leak/detector_test.go @@ -2,7 +2,6 @@ package leak_test import ( "fmt" - "strconv" "testing" "time" @@ -21,15 +20,7 @@ func mustTime(start string) time.Time { return s } -func mustUnixEpoch(start string) string { - s, err := time.Parse(time.RFC3339, start) - if err != nil { - panic("can't convert time from RFC3339") - } - return strconv.Itoa(int(s.Unix())) -} - -func TestMeasure(t *testing.T) { +func TestSmokeMeasure(t *testing.T) { qc := leak.NewFakeQueryClient() lc := leak.NewResourceLeakChecker(leak.WithQueryClient(qc)) testCases := []struct { From a24015261b7807e772ee4c0beb8e2167325340a8 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 12:27:12 +0100 Subject: [PATCH 11/14] fix lint --- framework/leak/profilecli.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/leak/profilecli.go b/framework/leak/profilecli.go index cc479027d..1ac80bf91 100644 --- a/framework/leak/profilecli.go +++ b/framework/leak/profilecli.go @@ -89,7 +89,7 @@ func (d *ProfileDumper) InstallProfileCLI() (string, error) { f.L.Info().Str("url", downloadURL).Msg("Downloading profilecli") // Download and extract in current directory - cmd := exec.Command("sh", "-c", fmt.Sprintf("curl -fL %s | tar xvz", downloadURL)) + cmd := exec.Command("sh", "-c", fmt.Sprintf("curl -fL %s | tar xvz", downloadURL)) //nolint cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -136,7 +136,7 @@ func (d *ProfileDumper) DownloadProfile(config *ProfileDumperConfig) (string, er env := os.Environ() env = append(env, fmt.Sprintf("PROFILECLI_URL=%s", d.pyroscopeURL)) - cmd := exec.Command(d.profileCLIPath, cmdArgs...) + cmd := exec.Command(d.profileCLIPath, cmdArgs...) //nolint:gosec cmd.Env = env cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr From 3e84fe8ec0df1f926c6366f82f7a0e645bac5116 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 15:21:00 +0100 Subject: [PATCH 12/14] example product fields --- framework/tmpl_gen_env.go | 85 ++++++++---------- framework/tmpl_gen_product.go | 162 ++++++++++++++++++++++++++++++++-- 2 files changed, 189 insertions(+), 58 deletions(-) diff --git a/framework/tmpl_gen_env.go b/framework/tmpl_gen_env.go index 76594c57e..1c7020765 100644 --- a/framework/tmpl_gen_env.go +++ b/framework/tmpl_gen_env.go @@ -1115,10 +1115,11 @@ func getSubCommands(parent string) []prompt.Suggest { fallthrough case "restart": return []prompt.Suggest{ - {Text: "env.toml", Description: "Spin up Anvil <> Anvil local chains, all services, 4 CL nodes"}, - {Text: "env.toml,env-cl-rebuild.toml", Description: "Spin up Anvil <> Anvil local chains, all services, 4 CL nodes (custom build)"}, - {Text: "env.toml,env-geth.toml", Description: "Spin up Geth <> Geth local chains (clique), all services, 4 CL nodes"}, - {Text: "env.toml,env-fuji-fantom.toml", Description: "Spin up testnets: Fuji <> Fantom, all services, 4 CL nodes"}, + {Text: "env.toml,products/{{ .ProductName }}/basic.toml", Description: "Spin up Anvil <> Anvil local chains, all services, 4 CL nodes"}, + {Text: "env.toml,products/{{ .ProductName }}/basic.toml,products/{{ .ProductName }}/soak.toml", Description: "Spin up Anvil <> Anvil local chains, all services, 4 CL nodes"}, + {Text: "env.toml,products/{{ .ProductName }}/basic.toml,env-cl-rebuild.toml", Description: "Spin up Anvil <> Anvil local chains, all services, 4 CL nodes (custom build)"}, + {Text: "env.toml,products/{{ .ProductName }}/basic.toml,env-geth.toml", Description: "Spin up Geth <> Geth local chains (clique), all services, 4 CL nodes"}, + {Text: "env.toml,products/{{ .ProductName }}/basic.toml,env-fuji-fantom.toml", Description: "Spin up testnets: Fuji <> Fantom, all services, 4 CL nodes"}, } default: return []prompt.Suggest{} @@ -1260,7 +1261,7 @@ var restartCmd = &cobra.Command{ if len(args) > 0 { configFile = args[0] } else { - configFile = "env.toml" + configFile = "env.toml,products/{{ .ProductName }}/basic.toml" } framework.L.Info().Str("Config", configFile).Msg("Reconfiguring development environment") _ = os.Setenv("CTF_CONFIGS", configFile) @@ -1284,7 +1285,7 @@ var upCmd = &cobra.Command{ if len(args) > 0 { configFile = args[0] } else { - configFile = "env.toml" + configFile = "env.toml,products/{{ .ProductName }}/basic.toml" } framework.L.Info().Str("Config", configFile).Msg("Creating development environment") _ = os.Setenv("CTF_CONFIGS", configFile) @@ -1510,6 +1511,9 @@ import ( de "{{ .GoModName }}" + "github.com/smartcontractkit/{{ .ProductName }}/devenv/products" + "github.com/smartcontractkit/{{ .ProductName }}/devenv/products/{{ .ProductName }}" + "github.com/smartcontractkit/chainlink-testing-framework/framework/chaos" "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" "github.com/smartcontractkit/chainlink-testing-framework/wasp" @@ -1547,6 +1551,10 @@ func (m *ExampleGun) Call(l *wasp.Generator) *wasp.Response { func TestLoadChaos(t *testing.T) { in, err := de.LoadOutput[de.Cfg]("../env-out.toml") require.NoError(t, err) + inProduct, err := products.LoadOutput[productone.Configurator]("../env-out.toml") + require.NoError(t, err) + + _ = inProduct clNodes, err := clclient.New(in.NodeSets[0].Out.CLNodes) require.NoError(t, err) @@ -1615,21 +1623,19 @@ func TestLoadChaos(t *testing.T) { require.False(t, failed) } ` - // SmokeTestTmpl is a smoke test template - SmokeTestTmpl = `package devenv_test + + SmokeTestImplTmpl = `package devenv_test import ( - "fmt" - "strconv" "testing" - "time" de "{{ .GoModName }}" + "github.com/smartcontractkit/{{ .ProductName }}/devenv/products" + "github.com/smartcontractkit/{{ .ProductName }}/devenv/products/{{ .ProductName }}" + "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" "github.com/stretchr/testify/require" - - f "github.com/smartcontractkit/chainlink-testing-framework/framework" ) var L = de.L @@ -1637,11 +1643,14 @@ var L = de.L func TestSmoke(t *testing.T) { in, err := de.LoadOutput[de.Cfg]("../env-out.toml") require.NoError(t, err) - c, _, _, err := de.ETHClient(in) + inProduct, err := products.LoadOutput[productone.Configurator]("../env-out.toml") require.NoError(t, err) clNodes, err := clclient.New(in.NodeSets[0].Out.CLNodes) require.NoError(t, err) + _ = in + _ = inProduct + tests := []struct { name string clNodes []*clclient.ChainlinkClient @@ -1656,39 +1665,10 @@ func TestSmoke(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, _ = c, clNodes + _ = clNodes }) } } - -// assertResources is a simple assertion on resources if you run with the observability stack (obs up) -func assertResources(t *testing.T, in *de.Cfg, start, end time.Time) { - pc := f.NewPrometheusQueryClient(f.LocalPrometheusBaseURL) - // no more than 10% CPU for this test - maxCPU := 10.0 - cpuResp, err := pc.Query("sum(rate(container_cpu_usage_seconds_total{name=~\".*don.*\"}[5m])) by (name) *100", end) - require.NoError(t, err) - cpu := f.ToLabelsMap(cpuResp) - for i := 0; i < in.NodeSets[0].Nodes; i++ { - nodeLabel := fmt.Sprintf("name:don-node%d", i) - nodeCpu, err := strconv.ParseFloat(cpu[nodeLabel][0].(string), 64) - L.Info().Int("Node", i).Float64("CPU", nodeCpu).Msg("CPU usage percentage") - require.NoError(t, err) - require.LessOrEqual(t, nodeCpu, maxCPU) - } - // no more than 200mb for this test - maxMem := int(200e6) // 200mb - memoryResp, err := pc.Query("sum(container_memory_rss{name=~\".*don.*\"}) by (name)", end) - require.NoError(t, err) - mem := f.ToLabelsMap(memoryResp) - for i := 0; i < in.NodeSets[0].Nodes; i++ { - nodeLabel := fmt.Sprintf("name:don-node%d", i) - nodeMem, err := strconv.Atoi(mem[nodeLabel][0].(string)) - L.Info().Int("Node", i).Int("Memory", nodeMem).Msg("Total memory") - require.NoError(t, err) - require.LessOrEqual(t, nodeMem, maxMem) - } -} ` // JDTmpl is a JobDistributor client wrappers JDTmpl = `package {{ .PackageName }} @@ -1831,7 +1811,6 @@ func Load[T any]() (*T, error) { } decoder := toml.NewDecoder(strings.NewReader(string(data))) - decoder.DisallowUnknownFields() if err := decoder.Decode(&config); err != nil { var details *toml.StrictMissingError @@ -2060,12 +2039,14 @@ push-fakes: // SmokeTestParams params for generating end-to-end test template type SmokeTestParams struct { - GoModName string + GoModName string + ProductName string } // LoadTestParams params for generating end-to-end test template type LoadTestParams struct { - GoModName string + GoModName string + ProductName string } // CISmokeParams params for generating CI smoke tests file @@ -2119,6 +2100,7 @@ type JustfileParams struct { // CLICompletionParams cli.go file params type CLICompletionParams struct { PackageName string + ProductName string CLIName string } @@ -2524,7 +2506,8 @@ func (g *EnvCodegen) Write() error { func (g *EnvCodegen) GenerateLoadTests() (string, error) { log.Info().Msg("Generating load test template") data := LoadTestParams{ - GoModName: g.cfg.moduleName, + GoModName: g.cfg.moduleName, + ProductName: g.cfg.productName, } return render(LoadTestTmpl, data) } @@ -2533,9 +2516,10 @@ func (g *EnvCodegen) GenerateLoadTests() (string, error) { func (g *EnvCodegen) GenerateSmokeTests() (string, error) { log.Info().Msg("Generating smoke test template") data := SmokeTestParams{ - GoModName: g.cfg.moduleName, + GoModName: g.cfg.moduleName, + ProductName: g.cfg.productName, } - return render(SmokeTestTmpl, data) + return render(SmokeTestImplTmpl, data) } // GenerateCILoadChaos generates a load&chaos test CI workflow @@ -2631,6 +2615,7 @@ func (g *EnvCodegen) GenerateCLICompletion() (string, error) { log.Info().Msg("Generating shell completion") p := CLICompletionParams{ PackageName: g.cfg.packageName, + ProductName: g.cfg.productName, CLIName: g.cfg.cliName, } return render(CompletionTmpl, p) diff --git a/framework/tmpl_gen_product.go b/framework/tmpl_gen_product.go index b23f1fadd..ba73edd56 100644 --- a/framework/tmpl_gen_product.go +++ b/framework/tmpl_gen_product.go @@ -15,12 +15,15 @@ const ( ProductSoakConfigTmpl = `# This file describes how many instances of your product we should deploy for soak test # you can also override keys from other configs here, for example your [[{{ .ProductName }}]] or [[blockchains]] / [[nodesets]] [[products]] -name = "ocr2" -instances = 1 +name = "{{ .ProductName }}" +instances = 10 ` - ProductBasicConfigTmpl = `[[{{ .ProductName}}]] -# TODO: define your product configuration here, see configurator.go ProductConfig` + ProductBasicConfigTmpl = `# TODO: define your product configuration here, see configurator.go ProductConfig +[[{{ .ProductName}}]] +# TODO: define your product configurator outputs so tests can verify your product +[{{ .ProductName }}.out] +` ProductsImplTmpl = `package {{ .ProductName }} @@ -40,10 +43,16 @@ import ( var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "{{ .ProductName }}"}).Logger() -type ProductConfig struct{} +type ProductConfig struct { + Out *ProductConfigOutput ` + "`" +`toml:"out"` + "`" + ` +} + +type ProductConfigOutput struct { + ExampleField string ` + "`" +`toml:"example"` + "`" + ` +} type Configurator struct { - Config []*ProductConfig ` + "`" + `toml:"productone"` + "`" + ` + Config []*ProductConfig ` + "`" + `toml:"{{ .ProductName }}"` + "`" + ` } func NewConfigurator() *Configurator { @@ -100,7 +109,12 @@ func (m *Configurator) ConfigureJobsAndContracts( bc *blockchain.Input, ns *nodeset.Input, ) error { - L.Info().Msg("Configuring product: productone") + // write an example output of your product configuration + // contract addresses, URLs, etc + // in soak test case it may hold multiple configs and have different outputs + // for each instance + m.Config[0].Out = &ProductConfigOutput{ExampleField: "my_data"} + L.Info().Msg("Configuring product: {{ .ProductName }}") return nil } ` @@ -192,15 +206,147 @@ func BaseConfigPath(envVar string) (string, error) { ProductsCommonTmpl = `package products import ( - "context" +"context" + "crypto/ecdsa" + "errors" + "fmt" + "math/big" + "os" + "strings" "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/rs/zerolog" "github.com/ethereum/go-ethereum/core/types" ) +const ( + AnvilKey0 = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + DefaultNativeTransferGasPrice = 21000 +) + +// FundNodeEIP1559 funds CL node using RPC URL, recipient address and amount of funds to send (ETH). +// Uses EIP-1559 transaction type. +func FundNodeEIP1559(ctx context.Context, c *ethclient.Client, pkey, recipientAddress string, amountOfFundsInETH float64) error { + l := zerolog.Ctx(ctx) + amount := new(big.Float).Mul(big.NewFloat(amountOfFundsInETH), big.NewFloat(1e18)) + amountWei, _ := amount.Int(nil) + l.Info().Str("Addr", recipientAddress).Str("Wei", amountWei.String()).Msg("Funding Node") + + chainID, err := c.NetworkID(context.Background()) + if err != nil { + return err + } + privateKeyStr := strings.TrimPrefix(pkey, "0x") + privateKey, err := crypto.HexToECDSA(privateKeyStr) + if err != nil { + return err + } + publicKey := privateKey.Public() + publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) + if !ok { + return errors.New("error casting public key to ECDSA") + } + fromAddress := crypto.PubkeyToAddress(*publicKeyECDSA) + + nonce, err := c.PendingNonceAt(context.Background(), fromAddress) + if err != nil { + return err + } + feeCap, err := c.SuggestGasPrice(context.Background()) + if err != nil { + return err + } + tipCap, err := c.SuggestGasTipCap(context.Background()) + if err != nil { + return err + } + recipient := common.HexToAddress(recipientAddress) + tx := types.NewTx(&types.DynamicFeeTx{ + ChainID: chainID, + Nonce: nonce, + To: &recipient, + Value: amountWei, + Gas: DefaultNativeTransferGasPrice, + GasFeeCap: feeCap, + GasTipCap: tipCap, + }) + signedTx, err := types.SignTx(tx, types.NewLondonSigner(chainID), privateKey) + if err != nil { + return err + } + err = c.SendTransaction(context.Background(), signedTx) + if err != nil { + return err + } + if _, err := WaitMinedFast(context.Background(), c, signedTx.Hash()); err != nil { + return err + } + l.Info().Str("Wei", amountWei.String()).Msg("Funded with ETH") + return nil +} + +// ETHClient creates a basic Ethereum client using PRIVATE_KEY env var and tip/cap gas settings +func ETHClient(ctx context.Context, rpcURL string, feeCapMult int64, tipCapMult int64) (*ethclient.Client, *bind.TransactOpts, string, error) { + l := zerolog.Ctx(ctx) + client, err := ethclient.Dial(rpcURL) + if err != nil { + return nil, nil, "", fmt.Errorf("could not connect to eth client: %w", err) + } + privateKey, err := crypto.HexToECDSA(getNetworkPrivateKey()) + if err != nil { + return nil, nil, "", fmt.Errorf("could not parse private key: %w", err) + } + publicKey := privateKey.PublicKey + address := crypto.PubkeyToAddress(publicKey).String() + chainID, err := client.ChainID(context.Background()) + if err != nil { + return nil, nil, "", fmt.Errorf("could not get chain ID: %w", err) + } + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, chainID) + if err != nil { + return nil, nil, "", fmt.Errorf("could not create transactor: %w", err) + } + fc, tc, err := multiplyEIP1559GasPrices(client, feeCapMult, tipCapMult) + if err != nil { + return nil, nil, "", fmt.Errorf("could not get bumped gas price: %w", err) + } + auth.GasFeeCap = fc + auth.GasTipCap = tc + l.Info(). + Str("GasFeeCap", fc.String()). + Str("GasTipCap", tc.String()). + Msg("Default gas prices set") + return client, auth, address, nil +} + +// multiplyEIP1559GasPrices returns bumped EIP1159 gas prices increased by multiplier +func multiplyEIP1559GasPrices(client *ethclient.Client, fcMult, tcMult int64) (*big.Int, *big.Int, error) { //nolint:revive // trivial function + feeCap, err := client.SuggestGasPrice(context.Background()) + if err != nil { + return nil, nil, err + } + tipCap, err := client.SuggestGasTipCap(context.Background()) + if err != nil { + return nil, nil, err + } + + return new(big.Int).Mul(feeCap, big.NewInt(fcMult)), new(big.Int).Mul(tipCap, big.NewInt(tcMult)), nil +} + +func getNetworkPrivateKey() string { + pk := os.Getenv("PRIVATE_KEY") + if pk == "" { + // that's the first Anvil and Geth private key, serves as a fallback for local testing if not overridden + return AnvilKey0 + } + return pk +} + // WaitMinedFast is a method for Anvil's instant blocks mode to ovecrome bind.WaitMined ticker hardcode. func WaitMinedFast(ctx context.Context, b bind.DeployBackend, txHash common.Hash) (*types.Receipt, error) { queryTicker := time.NewTicker(5 * time.Millisecond) From 16fa6ff02cbe6317544bb6f94f05445c7d378653 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 15:21:53 +0100 Subject: [PATCH 13/14] lint --- framework/tmpl_gen_product.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/tmpl_gen_product.go b/framework/tmpl_gen_product.go index ba73edd56..3450d0373 100644 --- a/framework/tmpl_gen_product.go +++ b/framework/tmpl_gen_product.go @@ -44,11 +44,11 @@ import ( var L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel).With().Fields(map[string]any{"component": "{{ .ProductName }}"}).Logger() type ProductConfig struct { - Out *ProductConfigOutput ` + "`" +`toml:"out"` + "`" + ` + Out *ProductConfigOutput ` + "`" + `toml:"out"` + "`" + ` } type ProductConfigOutput struct { - ExampleField string ` + "`" +`toml:"example"` + "`" + ` + ExampleField string ` + "`" + `toml:"example"` + "`" + ` } type Configurator struct { From 93c2580ad1035b60e1c2876076e86446bf4a8bc0 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 20 Jan 2026 15:53:50 +0100 Subject: [PATCH 14/14] comments --- framework/leak/detector_hog_test.go | 2 ++ framework/leak/detector_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/framework/leak/detector_hog_test.go b/framework/leak/detector_hog_test.go index 865c7a745..bf0f7849c 100644 --- a/framework/leak/detector_hog_test.go +++ b/framework/leak/detector_hog_test.go @@ -42,6 +42,7 @@ func TestVerifyCyclicHog(t *testing.T) { // cpu diff, err := lc.MeasureDelta(&leak.CheckConfig{ Query: `avg_over_time((sum(rate(container_cpu_usage_seconds_total{name="resource-hog"}[30m])))[30m:5m]) * 100`, + // set timestamps for the run you are analyzing Start: mustTime("2026-01-19T10:30:00Z"), End: mustTime("2026-01-19T12:29:15Z"), WarmUpDuration: 10 * time.Minute, @@ -52,6 +53,7 @@ func TestVerifyCyclicHog(t *testing.T) { // mem diff, err = lc.MeasureDelta(&leak.CheckConfig{ Query: `avg_over_time(container_memory_rss{name="resource-hog"}[30m]) / 1024 / 1024`, + // set timestamps for the run you are analyzing Start: mustTime("2026-01-19T10:30:00Z"), End: mustTime("2026-01-19T12:29:15Z"), WarmUpDuration: 10 * time.Minute, diff --git a/framework/leak/detector_test.go b/framework/leak/detector_test.go index f6af5e740..beb040647 100644 --- a/framework/leak/detector_test.go +++ b/framework/leak/detector_test.go @@ -130,6 +130,7 @@ func TestRealCLNodesLeakDetectionLocalDevenv(t *testing.T) { require.NoError(t, err) errs := cnd.Check(&leak.CLNodesCheck{ NumNodes: 4, + // set timestamps for the run you are analyzing Start: mustTime("2026-01-19T17:23:14Z"), End: mustTime("2026-01-19T18:00:51Z"), CPUThreshold: 100.0, @@ -150,6 +151,7 @@ func TestRealPrometheusLowLevelAPI(t *testing.T) { for i := range donNodes { diff, err := lc.MeasureDelta(&leak.CheckConfig{ Query: fmt.Sprintf(`quantile_over_time(0.5, container_memory_rss{name="don-node%d"}[1h]) / 1024 / 1024`, i), + // set timestamps for the run you are analyzing Start: mustTime("2026-01-12T21:53:00Z"), End: mustTime("2026-01-13T10:11:00Z"), WarmUpDuration: 1 * time.Hour,