From c4fa7cbe370429eb0de4f6dfb327de3615049166 Mon Sep 17 00:00:00 2001 From: rongxin Date: Fri, 28 Nov 2025 08:58:03 +0800 Subject: [PATCH 1/8] feat: support benchmark test --- .github/workflows/benchmark-test.yml | 104 +++++ Makefile | 6 +- go.mod | 11 + go.sum | 22 + internal/adc/cache/store.go | 2 +- internal/adc/client/client.go | 1 + internal/controller/status/updater.go | 6 +- internal/controller/utils.go | 2 +- internal/manager/readiness/manager.go | 11 +- internal/provider/apisix/provider.go | 17 +- test/benchmark/benchmark_test.go | 430 ++++++++++++++++++ test/benchmark/suite_test.go | 41 ++ test/benchmark/utis.go | 69 +++ .../manifests/apisix-standalone.yaml | 16 + test/e2e/framework/manifests/apisix.yaml | 17 + test/e2e/scaffold/scaffold.go | 107 +++++ 16 files changed, 847 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/benchmark-test.yml create mode 100644 test/benchmark/benchmark_test.go create mode 100644 test/benchmark/suite_test.go create mode 100644 test/benchmark/utis.go diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml new file mode 100644 index 0000000000..d81380fccd --- /dev/null +++ b/.github/workflows/benchmark-test.yml @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Benchmark Test + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + inputs: + routes: + description: "Number of routes for benchmark test" + required: false + default: "2000" + consumers: + description: "Number of consumers for benchmark test" + required: false + default: "2000" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ADC_VERSION: dev + BENCHMARK_ROUTES: ${{ github.event.inputs.routes }} + BENCHMARK_CONSUMERS: ${{ github.event.inputs.consumers }} + +jobs: + e2e-test: + strategy: + matrix: + provider_type: + - apisix-standalone + - apisix + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Go Env + uses: actions/setup-go@v4 + with: + go-version: "1.24" + + - name: Install kind + run: | + go install sigs.k8s.io/kind@v0.23.0 + + - name: Install ginkgo + run: | + make install-ginkgo + + - name: Build images + env: + TAG: dev + ARCH: amd64 + ENABLE_PROXY: "false" + BASE_IMAGE_TAG: "debug" + run: | + echo "building images..." + make build-image + + - name: Launch Kind Cluster + run: | + make kind-up + + - name: Loading Docker Image to Kind Cluster + run: | + make kind-load-images + + - name: Install Gateway API And CRDs + run: | + make install + + - name: Run Benchmark Test + shell: bash + env: + PROVIDER_TYPE: ${{ matrix.provider_type }} + TEST_LABEL: ${{ matrix.cases_subset }} + TEST_ENV: CI + run: | + make benchmark-test diff --git a/Makefile b/Makefile index 39b1d6b0f8..55c75a1395 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ IMG ?= apache/apisix-ingress-controller:$(IMAGE_TAG) ENVTEST_K8S_VERSION = 1.30.0 KIND_NAME ?= apisix-ingress-cluster -ADC_VERSION ?= 0.21.2 +ADC_VERSION ?= 0.22.1 DIR := $(shell pwd) @@ -153,6 +153,10 @@ conformance-test: --conformance-profiles=$(CONFORMANCE_PROFILES) \ --report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT) +.PHONY: benchmark-test +benchmark-test: + go test -v ./test/benchmark -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v + .PHONY: lint lint: sort-import golangci-lint ## Run golangci-lint linter $(GOLANGCI_LINT) run diff --git a/go.mod b/go.mod index 60da4110ae..757f8e2824 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,10 @@ require ( github.com/hashicorp/go-memdb v1.3.4 github.com/imdario/mergo v0.3.16 github.com/incubator4/go-resty-expr v0.1.1 + github.com/olekukonko/tablewriter v1.1.1 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 + github.com/panjf2000/ants/v2 v2.11.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/samber/lo v1.47.0 @@ -33,6 +35,7 @@ require ( k8s.io/apiextensions-apiserver v0.32.3 k8s.io/apimachinery v0.32.3 k8s.io/client-go v0.32.3 + k8s.io/code-generator v0.32.3 k8s.io/kubectl v0.30.3 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 sigs.k8s.io/controller-runtime v0.20.4 @@ -91,6 +94,9 @@ require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clipperhouse/displaywidth v0.3.1 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect @@ -139,6 +145,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect github.com/miekg/dns v1.1.65 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -150,6 +157,9 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect + github.com/olekukonko/errors v1.1.0 // indirect + github.com/olekukonko/ll v0.1.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.4.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -203,6 +213,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect k8s.io/apiserver v0.32.3 // indirect k8s.io/component-base v0.32.3 // indirect + k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect moul.io/http2curl/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 8904243c0a..eddf72b525 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,12 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/displaywidth v0.3.1 h1:k07iN9gD32177o1y4O1jQMzbLdCrsGJh+blirVYybsk= +github.com/clipperhouse/displaywidth v0.3.1/go.mod h1:tgLJKKyaDOCadywag3agw4snxS5kYEuYR6Y9+qWDDYM= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= @@ -285,6 +291,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= @@ -311,12 +319,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= +github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= +github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.1.2 h1:lkg/k/9mlsy0SxO5aC+WEpbdT5K83ddnNhAepz7TQc0= +github.com/olekukonko/ll v0.1.2/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew= +github.com/olekukonko/tablewriter v1.1.1 h1:b3reP6GCfrHwmKkYwNRFh2rxidGHcT6cgxj/sHiDDx0= +github.com/olekukonko/tablewriter v1.1.1/go.mod h1:De/bIcTF+gpBDB3Alv3fEsZA+9unTsSzAg/ZGADCtn4= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= +github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -579,8 +597,12 @@ k8s.io/apiserver v0.32.3 h1:kOw2KBuHOA+wetX1MkmrxgBr648ksz653j26ESuWNY8= k8s.io/apiserver v0.32.3/go.mod h1:q1x9B8E/WzShF49wh3ADOh6muSfpmFL0I2t+TG0Zdgc= k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/code-generator v0.32.3 h1:31p2TVzC9+hVdSkAFruAk3JY+iSfzrJ83Qij1yZutyw= +k8s.io/code-generator v0.32.3/go.mod h1:+mbiYID5NLsBuqxjQTygKM/DAdKpAjvBzrJd64NU1G8= k8s.io/component-base v0.32.3 h1:98WJvvMs3QZ2LYHBzvltFSeJjEx7t5+8s71P7M74u8k= k8s.io/component-base v0.32.3/go.mod h1:LWi9cR+yPAv7cu2X9rZanTiFKB2kHA+JjmhkKjCZRpI= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= diff --git a/internal/adc/cache/store.go b/internal/adc/cache/store.go index 6c5a1f2394..d41188ed1a 100644 --- a/internal/adc/cache/store.go +++ b/internal/adc/cache/store.go @@ -40,7 +40,7 @@ func NewStore(log logr.Logger) *Store { return &Store{ cacheMap: make(map[string]Cache), pluginMetadataMap: make(map[string]adctypes.PluginMetadata), - log: log, + log: log.WithName("store"), } } diff --git a/internal/adc/client/client.go b/internal/adc/client/client.go index 8a498b1344..2419dde90a 100644 --- a/internal/adc/client/client.go +++ b/internal/adc/client/client.go @@ -198,6 +198,7 @@ func (c *Client) Sync(ctx context.Context) (map[string]types.ADCExecutionErrors, if resources == nil { continue } + c.log.Info("syncing resources for config", "service_number", len(resources.Services)) if err := c.sync(ctx, Task{ Name: name + "-sync", diff --git a/internal/controller/status/updater.go b/internal/controller/status/updater.go index d00aec76a8..ea63369969 100644 --- a/internal/controller/status/updater.go +++ b/internal/controller/status/updater.go @@ -37,7 +37,7 @@ import ( pkgmetrics "github.com/apache/apisix-ingress-controller/pkg/metrics" ) -const UpdateChannelBufferSize = 1000 +const UpdateChannelBufferSize = 50000 type Update struct { NamespacedName k8stypes.NamespacedName @@ -119,7 +119,7 @@ func (u *UpdateHandler) updateStatus(ctx context.Context, update Update) error { newObj.SetUID(obj.GetUID()) - u.log.Info("updating status", "name", update.NamespacedName.Name, + u.log.V(1).Info("updating status", "name", update.NamespacedName.Name, "namespace", update.NamespacedName.Namespace, "kind", types.KindOf(newObj), ) @@ -144,8 +144,6 @@ func (u *UpdateHandler) Start(ctx context.Context) error { "name", update.NamespacedName.Name, "kind", types.KindOf(update.Resource), ) - - u.apply(ctx, update) } } } diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 845d56dfb4..4c2e971b9f 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -1311,7 +1311,7 @@ func ProcessIngressClassParameters(tctx *provider.TranslateContext, c client.Cli return err } - log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) + log.V(1).Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) tctx.GatewayProxies[ingressClassKind] = *gatewayProxy tctx.ResourceParentRefs[objKind] = append(tctx.ResourceParentRefs[objKind], ingressClassKind) diff --git a/internal/manager/readiness/manager.go b/internal/manager/readiness/manager.go index eca380eb8c..b140bd833f 100644 --- a/internal/manager/readiness/manager.go +++ b/internal/manager/readiness/manager.go @@ -125,7 +125,8 @@ func (r *readinessManager) Start(ctx context.Context) error { }) } if len(expected) > 0 { - r.log.V(1).Info("registering readiness state", "gvk", gvk, "expected", expected) + r.log.Info("registering readiness state", "gvk", gvk, "registered_count", len(expected)) + r.log.V(1).Info("registered resources for readiness", "gvk", gvk, "resources", expected) r.registerState(gvk, expected) } } @@ -135,13 +136,12 @@ func (r *readinessManager) Start(ctx context.Context) error { r.isReady.Store(true) close(r.done) } + r.log.Info("readiness manager started") }) return err } func (r *readinessManager) registerState(gvk schema.GroupVersionKind, list []k8stypes.NamespacedName) { - r.mu.Lock() - defer r.mu.Unlock() if _, ok := r.state[gvk]; !ok { r.state[gvk] = make(map[k8stypes.NamespacedName]struct{}) } @@ -155,9 +155,12 @@ func (r *readinessManager) Done(obj client.Object, nn k8stypes.NamespacedName) { if r.IsReady() { return } + <-r.started + r.mu.Lock() defer r.mu.Unlock() gvk := types.GvkOf(obj) + r.log.Info("marking resource as done", "gvk", gvk, "name", nn, "state_count", len(r.state[gvk])) if _, ok := r.state[gvk]; !ok { return } @@ -191,7 +194,7 @@ func (r *readinessManager) WaitReady(ctx context.Context, timeout time.Duration) case <-ctx.Done(): return false case <-time.After(timeout): - return true + return false case <-r.done: return true } diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go index d0d8e48a05..5c3d414da8 100644 --- a/internal/provider/apisix/provider.go +++ b/internal/provider/apisix/provider.go @@ -76,7 +76,9 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan o.DefaultBackendMode = ProviderTypeAPISIX } - cli, err := adcclient.New(log, o.DefaultBackendMode, o.SyncTimeout) + logger := log.WithName("provider") + + cli, err := adcclient.New(logger, o.DefaultBackendMode, o.SyncTimeout) if err != nil { return nil, err } @@ -88,7 +90,7 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan updater: updater, readier: readier, syncCh: make(chan struct{}, 1), - log: log.WithName("provider"), + log: logger, }, nil } @@ -249,7 +251,10 @@ func (d *apisixProvider) buildConfig(tctx *provider.TranslateContext, nnk types. } func (d *apisixProvider) Start(ctx context.Context) error { + d.log.Info("starting provider, waiting for readiness") d.readier.WaitReady(ctx, 5*time.Minute) + d.log.Info("Ready detected, starting sync loop") + initalSyncDelay := d.InitSyncDelay if initalSyncDelay > 0 { time.AfterFunc(initalSyncDelay, d.syncNotify) @@ -283,8 +288,12 @@ func (d *apisixProvider) Start(ctx context.Context) error { } func (d *apisixProvider) sync(ctx context.Context) error { - statusesMap, err := d.client.Sync(ctx) - d.handleADCExecutionErrors(statusesMap) + now := time.Now() + d.log.Info("starting to sync ADC configs", "time", now) + _, err := d.client.Sync(ctx) + d.log.Info("finished syncing ADC configs", "duration", time.Since(now)) + // statusesMap, err := d.client.Sync(ctx) + //d.handleADCExecutionErrors(statusesMap) return err } diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go new file mode 100644 index 0000000000..06e194a256 --- /dev/null +++ b/test/benchmark/benchmark_test.go @@ -0,0 +1,430 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// // Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package benchmark + +import ( + "bytes" + "fmt" + "net/http" + "os" + "time" + + "github.com/api7/gopkg/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var report = &BenchmarkReport{} +var totalRoutes = 2000 +var totalConsumers = 2000 + +var _ = BeforeSuite(func() { + routes := os.Getenv("BENCHMARK_ROUTES") + if routes != "" { + _, err := fmt.Sscanf(routes, "%d", &totalRoutes) + Expect(err).NotTo(HaveOccurred(), "parsing BENCHMARK_ROUTES") + } + consumers := os.Getenv("BENCHMARK_CONSUMERS") + if consumers != "" { + _, err := fmt.Sscanf(consumers, "%d", &totalConsumers) + Expect(err).NotTo(HaveOccurred(), "parsing BENCHMARK_CONSUMERS") + } +}) +var _ = AfterSuite(func() { + report.PrintTable() +}) + +const gatewayProxyYaml = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + service: + name: %s + port: 9180 + auth: + type: AdminKey + adminKey: + value: "%s" +` + +var _ = Describe("Benchmark Test", func() { + var ( + s = scaffold.NewDefaultScaffold() + controlAPIClient scaffold.ControlAPIClient + ) + + BeforeEach(func() { + By("port-forward to control api service") + var err error + controlAPIClient, err = s.ControlAPIClient() + Expect(err).NotTo(HaveOccurred(), "create control api client") + }) + + Context("Benchmark ApisixRoute", func() { + const ingressClassYaml = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: apisix +spec: + controller: "%s" + parameters: + apiGroup: "apisix.apache.org" + kind: "GatewayProxy" + name: "apisix-proxy-config" + namespace: %s + scope: "Namespace" +` + + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /get + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + var apisixRouteSpecHeaders = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /headers + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + + var apisixUpstreamSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-service-e2e-test +spec: + ingressClassName: apisix + scheme: https +` + var apisixRouteSpecKeyAuth = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: key-auth +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /get + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + authentication: + enable: true + type: keyAuth +` + var keyAuth = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixConsumer +metadata: + name: %s +spec: + ingressClassName: apisix + authParameter: + keyAuth: + value: + key: %s +` + + getRouteName := func(i int) string { + return fmt.Sprintf("test-route-%04d", i) + } + + createBatchApisixRoutes := func(number int) string { + var buf bytes.Buffer + for i := 0; i < number; i++ { + name := getRouteName(i) + fmt.Fprintf(&buf, apisixRouteSpec, name, name) + buf.WriteString("\n---\n") + } + return buf.String() + } + getConsumerName := func(i int) string { + return fmt.Sprintf("consumer-%04d", i) + } + createBatchConsumers := func(number int) string { + var buf bytes.Buffer + for i := 0; i < number; i++ { + name := getConsumerName(i) + fmt.Fprintf(&buf, keyAuth, name, name) + buf.WriteString("\n---\n") + } + return buf.String() + } + + benchmark := func(scenario string) { + s.Deployer.ScaleIngress(0) + By(fmt.Sprintf("prepare %d ApisixRoutes", totalRoutes)) + err := s.CreateResourceFromString(createBatchApisixRoutes(totalRoutes)) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes") + s.Deployer.ScaleIngress(1) + + now := time.Now() + By(fmt.Sprintf("start cale time for applying %d ApisixRoutes to take effect", totalRoutes)) + err = s.EnsureNumService(controlAPIClient, func(actual int) bool { return actual == totalRoutes }) + Expect(err).ShouldNot(HaveOccurred()) + costTime := time.Since(now) + report.Add(scenario, fmt.Sprintf("Apply %d ApisixRoutes", totalRoutes), costTime) + + By("Test the time required for an ApisixRoute update to take effect") + name := getRouteName(int(time.Now().Unix())) + err = s.CreateResourceFromString(fmt.Sprintf(apisixRouteSpecHeaders, name, name)) + Expect(err).NotTo(HaveOccurred()) + now = time.Now() + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode + }).WithTimeout(5 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + report.Add(scenario, fmt.Sprintf("Update a single ApisixRoute base on %d ApisixRoutes", totalRoutes), time.Since(now)) + + By("Test the time required for a service endpoint change to take effect") + err = s.ScaleHTTPBIN(2) + Expect(err).NotTo(HaveOccurred(), "scale httpbin deployment") + now = time.Now() + err = s.EnsureNumUpstreamNodes(controlAPIClient, "", 2) + Expect(err).ShouldNot(HaveOccurred()) + costTime = time.Since(now) + report.Add(scenario, fmt.Sprintf("Service endpoint change base on %d ApisixRoutes", totalRoutes), costTime) + + By("Test the time required for an ApisixUpstream update to take effect") + err = s.CreateResourceFromString(apisixUpstreamSpec) + Expect(err).NotTo(HaveOccurred(), "creating ApisixUpstream") + now = time.Now() + s.ExpectUpstream(controlAPIClient, "", func(upstream adctypes.Upstream) bool { + if upstream.Scheme != "https" { + log.Warnf("expect upstream: [%s] scheme to be https, but got [%s]", upstream.Name, upstream.Scheme) + return false + } + return true + }) + costTime = time.Since(now) + report.Add(scenario, fmt.Sprintf("Update ApisixUpstream base on %d ApisixRoutes", totalRoutes), costTime) + } + + BeforeEach(func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, s.GetControllerName(), s.Namespace()), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass") + time.Sleep(5 * time.Second) + }) + It("benchmark ApisixRoute", func() { + benchmark("ApisixRoute Benchmark") + }) + It("10 apisix-standalone pod scale benchmark", func() { + if framework.ProviderType != framework.ProviderTypeAPISIXStandalone { + Skip("only apisix-standalone support scale benchmark") + } + s.Deployer.ScaleDataplane(10) + benchmark("ApisixRoute Benchmark with 10 apisix-standalone pods") + }) + It("ApisixRoute With Consumers benchmark", func() { + s.Deployer.ScaleIngress(0) + By(fmt.Sprintf("prepare %d ApisixConsumers", totalRoutes)) + err := s.CreateResourceFromString(createBatchConsumers(totalRoutes)) + Expect(err).NotTo(HaveOccurred(), "creating ApisixConsumers") + err = s.CreateResourceFromString(apisixRouteSpecKeyAuth) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute with KeyAuth") + s.Deployer.ScaleIngress(1) + + now := time.Now() + Eventually(func() int { + sunccess := 0 + for i := 0; i < totalConsumers; i++ { + consumerName := getConsumerName(i) + if s.NewAPISIXClient().GET("/get"). + WithHeader("apikey", consumerName). + Expect().Raw().StatusCode != http.StatusOK { + return sunccess + } + sunccess++ + } + return sunccess + }).WithTimeout(10 * time.Minute).ProbeEvery(500 * time.Millisecond).Should(Equal(totalConsumers)) + costTime := time.Since(now) + report.AddResult(TestResult{ + Scenario: "ApisixRoute With Consumers Benchmark", + CaseName: fmt.Sprintf("Apply %d ApisixConsumers and ApisixRoute with KeyAuth", totalConsumers), + CostTime: costTime, + IsRequestGateway: true, + }) + }) + }) + + Context("Benchmark HTTPRoute", func() { + const httpRouteSpec = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: %s +spec: + parentRefs: + - name: %s + rules: + - matches: + - path: + type: Exact + value: /get + headers: + - type: Exact + name: X-Route-Name + value: %s + # name: get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + const httpRouteSpecHeaders = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: %s +spec: + parentRefs: + - name: %s + rules: + - matches: + - path: + type: Exact + value: /headers + headers: + - type: Exact + name: X-Route-Name + value: %s + # name: get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + createBatchHTTPRoutes := func(number int, parentGateway string) string { + var buf bytes.Buffer + for i := 0; i < number; i++ { + name := getRouteName(i) + fmt.Fprintf(&buf, httpRouteSpec, name, parentGateway, name) + buf.WriteString("\n---\n") + } + return buf.String() + } + + BeforeEach(func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create GatewayClass") + Expect(s.CreateResourceFromString(s.GetGatewayClassYaml())).NotTo(HaveOccurred(), "creating GatewayClass") + + By("create Gateway") + Expect(s.CreateResourceFromString(s.GetGatewayYaml())).NotTo(HaveOccurred(), "creating Gateway") + time.Sleep(5 * time.Second) + }) + + It("benchmark HTTPRoute", func() { + s.Deployer.ScaleIngress(0) + By(fmt.Sprintf("prepare %d HTTPRoute", totalRoutes)) + err := s.CreateResourceFromString(createBatchHTTPRoutes(totalRoutes, s.Namespace())) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") + s.Deployer.ScaleIngress(1) + + now := time.Now() + By(fmt.Sprintf("start cale time for applying %d HTTPRoute to take effect", totalRoutes)) + err = s.EnsureNumService(controlAPIClient, func(actual int) bool { return actual == totalRoutes }) + Expect(err).ShouldNot(HaveOccurred()) + costTime := time.Since(now) + report.Add("HTTPRoute Benchmark", fmt.Sprintf("Apply %d HTTPRoute", totalRoutes), costTime) + + By("Test the time required for an HTTPRoute update to take effect") + name := getRouteName(int(time.Now().Unix())) + err = s.CreateResourceFromString(fmt.Sprintf(httpRouteSpecHeaders, name, s.Namespace(), name)) + Expect(err).NotTo(HaveOccurred()) + now = time.Now() + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode + }).WithTimeout(5 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + report.AddResult(TestResult{ + Scenario: "HTTPRoute Benchmark", + CaseName: fmt.Sprintf("Update a single HTTPRoute base on %d HTTPRoute", totalRoutes), + CostTime: time.Since(now), + }) + + By("Test the time required for a service endpoint change to take effect") + err = s.ScaleHTTPBIN(2) + Expect(err).NotTo(HaveOccurred(), "scale httpbin deployment") + now = time.Now() + err = s.EnsureNumUpstreamNodes(controlAPIClient, "", 2) + Expect(err).ShouldNot(HaveOccurred()) + costTime = time.Since(now) + report.Add("HTTPRoute Benchmark", fmt.Sprintf("Service endpoint change base on %d HTTPRoute", totalRoutes), costTime) + }) + }) +}) + +func getRouteName(i int) string { + return fmt.Sprintf("test-route-%04d", i) +} diff --git a/test/benchmark/suite_test.go b/test/benchmark/suite_test.go new file mode 100644 index 0000000000..32f60af83d --- /dev/null +++ b/test/benchmark/suite_test.go @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package benchmark + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +// Run long-term-stability tests using Ginkgo runner. +func TestBenchmark(t *testing.T) { + RegisterFailHandler(Fail) + var f = framework.NewFramework() + _ = f + + scaffold.NewDeployer = scaffold.NewAPISIXDeployer + + _, _ = fmt.Fprintf(GinkgoWriter, "Starting Benchmark Tests\n") + RunSpecs(t, "Benchmark Tests Suite") +} diff --git a/test/benchmark/utis.go b/test/benchmark/utis.go new file mode 100644 index 0000000000..f11ca98841 --- /dev/null +++ b/test/benchmark/utis.go @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// // Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package benchmark + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/olekukonko/tablewriter" +) + +type TestResult struct { + Scenario string `json:"scenario"` + CaseName string `json:"case_name"` + CostTime time.Duration `json:"cost_time"` + IsRequestGateway bool `json:"is_request_gateway,omitempty"` +} + +type BenchmarkReport struct { + Results []TestResult +} + +func (r *BenchmarkReport) PrintTable() { + table := tablewriter.NewWriter(os.Stdout) + table.Header([]string{"Scenario", "Case", "Cost", "IsRequestGateway"}) + + for _, res := range r.Results { + table.Append([]any{ + res.Scenario, + res.CaseName, + res.CostTime.String(), + res.IsRequestGateway, + }) + } + table.Render() +} + +func (r *BenchmarkReport) PrintJSON() { + b, _ := json.MarshalIndent(r.Results, "", " ") + fmt.Println(string(b)) +} + +func (r *BenchmarkReport) AddResult(result TestResult) { + r.Results = append(r.Results, result) +} + +func (r *BenchmarkReport) Add(scenario, caseName string, cost time.Duration) { + r.Results = append(r.Results, TestResult{ + Scenario: scenario, + CaseName: caseName, + CostTime: cost, + }) +} diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml index 0eda2bc8af..58a728fed3 100644 --- a/test/e2e/framework/manifests/apisix-standalone.yaml +++ b/test/e2e/framework/manifests/apisix-standalone.yaml @@ -151,3 +151,19 @@ spec: selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} +--- + +apiVersion: v1 +kind: Service +metadata: + name: apisix-control-api + labels: + app.kubernetes.io/name: apisix-control-api +spec: + ports: + - port: 9090 + name: control + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/name: apisix \ No newline at end of file diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index 31581bcc2b..fd85b9c732 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -168,3 +168,20 @@ spec: selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} + +--- + +apiVersion: v1 +kind: Service +metadata: + name: apisix-control-api + labels: + app.kubernetes.io/name: apisix-control-api +spec: + ports: + - port: 9090 + name: control + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/name: apisix \ No newline at end of file diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 432732acc1..a2991d52c9 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -20,11 +20,13 @@ package scaffold import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "net/url" "os" "strings" + "time" "github.com/api7/gopkg/pkg/log" "github.com/gavv/httpexpect/v2" @@ -32,9 +34,12 @@ import ( "github.com/gruntwork-io/terratest/modules/testing" . "github.com/onsi/ginkgo/v2" //nolint:staticcheck . "github.com/onsi/gomega" //nolint:staticcheck + "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + adctypes "github.com/apache/apisix-ingress-controller/api/adc" apiv2 "github.com/apache/apisix-ingress-controller/api/v2" "github.com/apache/apisix-ingress-controller/test/e2e/framework" ) @@ -553,3 +558,105 @@ func (s *Scaffold) GetMetricsEndpoint() string { s.addFinalizers(tunnel.Close) return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint()) } + +func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) { + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090) + if err := tunnel.ForwardPortE(s.t); err != nil { + return nil, err + } + s.addFinalizers(tunnel.Close) + + return &controlAPI{ + client: NewClient("http", tunnel.Endpoint()), + }, nil +} + +func (s *Scaffold) EnsureNumService(controlAPIClient ControlAPIClient, matcher func(result int) bool) error { + times := 0 + return wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + times++ + results, _, err := controlAPIClient.ListServices() + if err != nil { + log.Errorw("failed to ListServices", zap.Error(err)) + return false, nil + } + if !matcher(len(results)) { + log.Debugw("number of effective services", zap.Int("number", len(results)), zap.Int("times", times)) + return false, nil + } + return true, nil + }) +} + +func (s *Scaffold) ExpectUpstream(controlAPIClient ControlAPIClient, name string, matcher func(upstream adctypes.Upstream) bool) error { + times := 0 + return wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + times++ + //upstreams, err := s.Deployer.DefaultDataplaneResource().Upstream().List(context.Background()) + upstreams, _, err := controlAPIClient.ListUpstreams() + if err != nil { + log.Errorw("failed to ListServices", zap.Error(err)) + return false, nil + } + for _, upstream := range upstreams { + upsValue := upstream.(map[string]any) + data, err := json.Marshal(upsValue["value"]) + if err != nil { + return false, fmt.Errorf("failed to marshal upstream: %v", err) + } + + var ups adctypes.Upstream + if err := json.Unmarshal(data, &ups); err != nil { + return false, fmt.Errorf("failed to unmarshal upstream: %v", err) + } + if name != "" && ups.Name != name { + continue + } + if ok := matcher(ups); !ok { + return false, nil + } + } + return true, nil + }) +} + +func (s *Scaffold) EnsureNumUpstreamNodes(controlAPIClient ControlAPIClient, name string, number int) error { + return s.ExpectUpstream(controlAPIClient, name, func(upstream adctypes.Upstream) bool { + if len(upstream.Nodes) != number { + log.Warnf("expect upstream: [%s] nodes num to be %d, but got %d", upstream.Name, number, len(upstream.Nodes)) + return false + } + return true + }) +} + +type ControlAPIClient interface { + ListServices() ([]any, int64, error) + ListUpstreams() ([]any, int64, error) +} + +type controlAPI struct { + client *httpexpect.Expect +} + +func (c *controlAPI) ListUpstreams() (result []any, total int64, err error) { + resp := c.client.Request(http.MethodGet, "/v1/upstreams").Expect() + if resp.Raw().StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw()) + } + if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err) + } + return result, int64(len(result)), err +} + +func (c *controlAPI) ListServices() (result []any, total int64, err error) { + resp := c.client.Request(http.MethodGet, "/v1/services").Expect() + if resp.Raw().StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw()) + } + if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err) + } + return result, int64(len(result)), err +} From 7d40d23850ad637c164289860e74d3e6df688d21 Mon Sep 17 00:00:00 2001 From: alinsran Date: Mon, 1 Dec 2025 08:37:13 +0800 Subject: [PATCH 2/8] f --- .github/workflows/benchmark-test.yml | 6 ++++++ Makefile | 6 ++++++ internal/controller/status/updater.go | 5 +++-- test/benchmark/benchmark_test.go | 5 +++-- test/benchmark/utis.go | 12 +++++++++--- test/e2e/framework/manifests/apisix-standalone.yaml | 3 +++ test/e2e/framework/manifests/apisix.yaml | 3 +++ test/e2e/scaffold/scaffold.go | 1 - 8 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index d81380fccd..09841751b2 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -21,9 +21,15 @@ on: push: branches: - master +<<<<<<< HEAD pull_request: branches: - master +======= + - benchmark-test + tags: + - '*' +>>>>>>> benchmark-test workflow_dispatch: inputs: routes: diff --git a/Makefile b/Makefile index 55c75a1395..7ddda7143a 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,11 @@ CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-r ## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC,GATEWAY-TLS + +TEST_EXCLUDES ?= /e2e /conformance /benchmark + +TEST_PACKAGES = $(shell go list ./... $(foreach p,$(TEST_EXCLUDES),| grep -v $(p))) + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -129,6 +134,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /conformance) -coverprofile cover.out + KUBEBUILDER_ASSETS="$$( $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path )" go test $(TEST_PACKAGES) -coverprofile cover.out .PHONY: kind-e2e-test kind-e2e-test: kind-up build-image kind-load-images e2e-test diff --git a/internal/controller/status/updater.go b/internal/controller/status/updater.go index ea63369969..e2ef06edf3 100644 --- a/internal/controller/status/updater.go +++ b/internal/controller/status/updater.go @@ -37,7 +37,7 @@ import ( pkgmetrics "github.com/apache/apisix-ingress-controller/pkg/metrics" ) -const UpdateChannelBufferSize = 50000 +const UpdateChannelBufferSize = 10000 type Update struct { NamespacedName k8stypes.NamespacedName @@ -140,10 +140,11 @@ func (u *UpdateHandler) Start(ctx context.Context) error { case update := <-u.updateChannel: // Decrement queue length after removing item from queue pkgmetrics.DecStatusQueueLength() - u.log.V(1).Info("received a status update", "namespace", update.NamespacedName.Namespace, + u.log.Info("received a status update", "namespace", update.NamespacedName.Namespace, "name", update.NamespacedName.Name, "kind", types.KindOf(update.Resource), ) + u.apply(ctx, update) } } } diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index 06e194a256..462095a326 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -232,7 +232,7 @@ spec: now = time.Now() Eventually(func() int { return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode - }).WithTimeout(5 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + }).WithTimeout(15 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) report.Add(scenario, fmt.Sprintf("Update a single ApisixRoute base on %d ApisixRoutes", totalRoutes), time.Since(now)) By("Test the time required for a service endpoint change to take effect") @@ -248,13 +248,14 @@ spec: err = s.CreateResourceFromString(apisixUpstreamSpec) Expect(err).NotTo(HaveOccurred(), "creating ApisixUpstream") now = time.Now() - s.ExpectUpstream(controlAPIClient, "", func(upstream adctypes.Upstream) bool { + err = s.ExpectUpstream(controlAPIClient, "", func(upstream adctypes.Upstream) bool { if upstream.Scheme != "https" { log.Warnf("expect upstream: [%s] scheme to be https, but got [%s]", upstream.Name, upstream.Scheme) return false } return true }) + Expect(err).ShouldNot(HaveOccurred()) costTime = time.Since(now) report.Add(scenario, fmt.Sprintf("Update ApisixUpstream base on %d ApisixRoutes", totalRoutes), costTime) } diff --git a/test/benchmark/utis.go b/test/benchmark/utis.go index f11ca98841..c3c3a761f5 100644 --- a/test/benchmark/utis.go +++ b/test/benchmark/utis.go @@ -22,7 +22,9 @@ import ( "os" "time" + "github.com/api7/gopkg/pkg/log" "github.com/olekukonko/tablewriter" + "go.uber.org/zap" ) type TestResult struct { @@ -41,14 +43,18 @@ func (r *BenchmarkReport) PrintTable() { table.Header([]string{"Scenario", "Case", "Cost", "IsRequestGateway"}) for _, res := range r.Results { - table.Append([]any{ + if err := table.Append([]any{ res.Scenario, res.CaseName, res.CostTime.String(), res.IsRequestGateway, - }) + }); err != nil { + log.Errorw("failed to append row to table", zap.Error(err)) + } + } + if err := table.Render(); err != nil { + log.Errorw("failed to render table", zap.Error(err)) } - table.Render() } func (r *BenchmarkReport) PrintJSON() { diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml index 58a728fed3..d4c0fc3718 100644 --- a/test/e2e/framework/manifests/apisix-standalone.yaml +++ b/test/e2e/framework/manifests/apisix-standalone.yaml @@ -35,6 +35,9 @@ data: nginx_config: worker_processes: 2 error_log_level: info + meta: + lua_shared_dict: + standalone-config: 50m apisix: proxy_mode: http&stream stream_proxy: # TCP/UDP proxy diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index fd85b9c732..b25e96075e 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -42,6 +42,9 @@ data: nginx_config: worker_processes: 2 error_log_level: info + meta: + lua_shared_dict: + standalone-config: 50m apisix: proxy_mode: http&stream stream_proxy: # TCP/UDP proxy diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index a2991d52c9..1ec5c09c2b 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -592,7 +592,6 @@ func (s *Scaffold) ExpectUpstream(controlAPIClient ControlAPIClient, name string times := 0 return wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { times++ - //upstreams, err := s.Deployer.DefaultDataplaneResource().Upstream().List(context.Background()) upstreams, _, err := controlAPIClient.ListUpstreams() if err != nil { log.Errorw("failed to ListServices", zap.Error(err)) From 8e851f5804ea0fa7db5641a5f58a350c83976ce7 Mon Sep 17 00:00:00 2001 From: alinsran Date: Mon, 1 Dec 2025 08:44:36 +0800 Subject: [PATCH 3/8] f --- Makefile | 3 --- internal/provider/apisix/provider.go | 8 ++------ test/benchmark/benchmark_test.go | 5 +++-- test/benchmark/utis.go | 3 ++- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 7ddda7143a..eb1ef88982 100644 --- a/Makefile +++ b/Makefile @@ -57,9 +57,7 @@ CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-r ## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC,GATEWAY-TLS - TEST_EXCLUDES ?= /e2e /conformance /benchmark - TEST_PACKAGES = $(shell go list ./... $(foreach p,$(TEST_EXCLUDES),| grep -v $(p))) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -133,7 +131,6 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /conformance) -coverprofile cover.out KUBEBUILDER_ASSETS="$$( $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path )" go test $(TEST_PACKAGES) -coverprofile cover.out .PHONY: kind-e2e-test diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go index 5c3d414da8..029675e219 100644 --- a/internal/provider/apisix/provider.go +++ b/internal/provider/apisix/provider.go @@ -288,12 +288,8 @@ func (d *apisixProvider) Start(ctx context.Context) error { } func (d *apisixProvider) sync(ctx context.Context) error { - now := time.Now() - d.log.Info("starting to sync ADC configs", "time", now) - _, err := d.client.Sync(ctx) - d.log.Info("finished syncing ADC configs", "duration", time.Since(now)) - // statusesMap, err := d.client.Sync(ctx) - //d.handleADCExecutionErrors(statusesMap) + statusesMap, err := d.client.Sync(ctx) + d.handleADCExecutionErrors(statusesMap) return err } diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index 462095a326..fb7a03df18 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -6,8 +6,9 @@ // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the diff --git a/test/benchmark/utis.go b/test/benchmark/utis.go index c3c3a761f5..35de0b908b 100644 --- a/test/benchmark/utis.go +++ b/test/benchmark/utis.go @@ -7,7 +7,8 @@ // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, +// +// Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the From 8515ae0532f1847197cdeab6a58e1b904915ee03 Mon Sep 17 00:00:00 2001 From: alinsran Date: Mon, 1 Dec 2025 09:04:32 +0800 Subject: [PATCH 4/8] lint --- .../manifests/apisix-standalone.yaml | 172 ------------------ test/e2e/framework/manifests/apisix.yaml | 2 +- 2 files changed, 1 insertion(+), 173 deletions(-) delete mode 100644 test/e2e/framework/manifests/apisix-standalone.yaml diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml deleted file mode 100644 index d4c0fc3718..0000000000 --- a/test/e2e/framework/manifests/apisix-standalone.yaml +++ /dev/null @@ -1,172 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: apisix-conf -data: - config.yaml: | - deployment: - role: traditional - role_traditional: - config_provider: yaml - admin: - allow_admin: - - 0.0.0.0/0 - admin_key: - - key: {{ .AdminKey }} - name: admin - role: admin - nginx_config: - worker_processes: 2 - error_log_level: info - meta: - lua_shared_dict: - standalone-config: 50m - apisix: - proxy_mode: http&stream - stream_proxy: # TCP/UDP proxy - tcp: # TCP proxy port list - - 9100 - - addr: 9110 - tls: true - udp: # UDP proxy port list - - 9200 - discovery: - dns: - servers: - - "10.96.0.10:53" # use the real address of your dns server. - # currently we use KIND as the standard test environment, so here we can hard-code the default DNS address first. ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apisix - labels: - app.kubernetes.io/name: apisix -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: apisix - template: - metadata: - labels: - app.kubernetes.io/name: apisix - spec: - initContainers: - - name: config-setup - image: apache/apisix:dev - command: - - sh - - -c - - | - echo "Copying default config directory to writable volume" - cp -r /usr/local/apisix/conf/* /tmp/apisix-conf/ - echo "Overwriting config.yaml with custom configuration" - cp /tmp/config-source/config.yaml /tmp/apisix-conf/config.yaml - echo "Config setup completed successfully" - ls -la /tmp/apisix-conf/ - volumeMounts: - - name: config-source - mountPath: /tmp/config-source - - name: config-writable - mountPath: /tmp/apisix-conf - containers: - - name: apisix - image: apache/apisix:dev - ports: - - name: http - containerPort: 9080 - protocol: TCP - - name: https - containerPort: 9443 - protocol: TCP - - name: admin - containerPort: 9180 - protocol: TCP - - name: tcp - containerPort: 9100 - protocol: TCP - - name: udp - containerPort: 9200 - protocol: UDP - - name: tls - containerPort: 9110 - protocol: TCP - volumeMounts: - - name: config-writable - mountPath: /usr/local/apisix/conf - volumes: - - name: config-source - configMap: - name: apisix-conf - - name: config-writable - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ .ServiceName }} - labels: - app.kubernetes.io/name: apisix -spec: - ports: - - port: {{ .ServiceHTTPPort }} - name: http - protocol: TCP - targetPort: 9080 - - port: {{ .ServiceHTTPSPort }} - name: https - protocol: TCP - targetPort: 9443 - - port: 9180 - name: admin - protocol: TCP - targetPort: 9180 - - name: tcp - port: 9100 - protocol: TCP - targetPort: 9100 - - name: udp - port: 9200 - protocol: UDP - targetPort: 9200 - - name: tls - port: 9110 - protocol: TCP - targetPort: 9110 - selector: - app.kubernetes.io/name: apisix - type: {{ .ServiceType | default "NodePort" }} ---- - -apiVersion: v1 -kind: Service -metadata: - name: apisix-control-api - labels: - app.kubernetes.io/name: apisix-control-api -spec: - ports: - - port: 9090 - name: control - protocol: TCP - targetPort: 9090 - selector: - app.kubernetes.io/name: apisix \ No newline at end of file diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index b25e96075e..eada504150 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -187,4 +187,4 @@ spec: protocol: TCP targetPort: 9090 selector: - app.kubernetes.io/name: apisix \ No newline at end of file + app.kubernetes.io/name: apisix From 40e7081babfddfdb2008b065e73f72ff04984149 Mon Sep 17 00:00:00 2001 From: alinsran Date: Mon, 1 Dec 2025 13:42:44 +0800 Subject: [PATCH 5/8] fix ci --- .github/workflows/benchmark-test.yml | 6 ------ test/benchmark/benchmark_test.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 09841751b2..d81380fccd 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -21,15 +21,9 @@ on: push: branches: - master -<<<<<<< HEAD pull_request: branches: - master -======= - - benchmark-test - tags: - - '*' ->>>>>>> benchmark-test workflow_dispatch: inputs: routes: diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index fb7a03df18..080e757670 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -305,7 +305,7 @@ spec: sunccess++ } return sunccess - }).WithTimeout(10 * time.Minute).ProbeEvery(500 * time.Millisecond).Should(Equal(totalConsumers)) + }).WithTimeout(15 * time.Minute).ProbeEvery(1 * time.Second).Should(Equal(totalConsumers)) costTime := time.Since(now) report.AddResult(TestResult{ Scenario: "ApisixRoute With Consumers Benchmark", From 8dc9191dc37ff9fd08ae707bf545ec7fb665723b Mon Sep 17 00:00:00 2001 From: rongxin Date: Mon, 1 Dec 2025 17:46:07 +0800 Subject: [PATCH 6/8] typo --- test/benchmark/benchmark_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index 080e757670..da247ea1b1 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -294,17 +294,17 @@ spec: now := time.Now() Eventually(func() int { - sunccess := 0 + success := 0 for i := 0; i < totalConsumers; i++ { consumerName := getConsumerName(i) if s.NewAPISIXClient().GET("/get"). WithHeader("apikey", consumerName). Expect().Raw().StatusCode != http.StatusOK { - return sunccess + return success } - sunccess++ + success++ } - return sunccess + return success }).WithTimeout(15 * time.Minute).ProbeEvery(1 * time.Second).Should(Equal(totalConsumers)) costTime := time.Since(now) report.AddResult(TestResult{ From 228841daf53ff1f28411a672fe1ff2f08c24db7f Mon Sep 17 00:00:00 2001 From: alinsran Date: Mon, 8 Dec 2025 12:26:30 +0800 Subject: [PATCH 7/8] upgrade adc --- Makefile | 2 +- test/benchmark/benchmark_test.go | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index eb1ef88982..cb41483b13 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ IMG ?= apache/apisix-ingress-controller:$(IMAGE_TAG) ENVTEST_K8S_VERSION = 1.30.0 KIND_NAME ?= apisix-ingress-cluster -ADC_VERSION ?= 0.22.1 +ADC_VERSION ?= 0.23.1 DIR := $(shell pwd) diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index da247ea1b1..f32f8cf7c5 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -19,6 +19,7 @@ package benchmark import ( "bytes" + "context" "fmt" "net/http" "os" @@ -293,19 +294,16 @@ spec: s.Deployer.ScaleIngress(1) now := time.Now() - Eventually(func() int { - success := 0 - for i := 0; i < totalConsumers; i++ { - consumerName := getConsumerName(i) - if s.NewAPISIXClient().GET("/get"). - WithHeader("apikey", consumerName). - Expect().Raw().StatusCode != http.StatusOK { - return success - } - success++ + Eventually(func() error { + consumer, err := s.DefaultDataplaneResource().Consumer().List(context.Background()) + if err != nil { + return err + } + if len(consumer) != totalConsumers { + return fmt.Errorf("expect %d consumers, but got %d", totalConsumers, len(consumer)) } - return success - }).WithTimeout(15 * time.Minute).ProbeEvery(1 * time.Second).Should(Equal(totalConsumers)) + return nil + }).WithTimeout(15*time.Minute).ProbeEvery(1*time.Second).ShouldNot(HaveOccurred(), "waiting for all consumers to be synced to APISIX") costTime := time.Since(now) report.AddResult(TestResult{ Scenario: "ApisixRoute With Consumers Benchmark", From 12b0d02ea15762e8a80b7760ee8062ca695f6e54 Mon Sep 17 00:00:00 2001 From: alinsran Date: Mon, 8 Dec 2025 15:22:17 +0800 Subject: [PATCH 8/8] fix test --- .github/workflows/benchmark-test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index d81380fccd..565fb1d034 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -94,6 +94,15 @@ jobs: run: | make install + - name: Extract adc binary + if: ${{ env.ADC_VERSION == 'dev' }} + run: | + docker create --name adc-temp ghcr.io/api7/adc:dev + docker cp adc-temp:main.js adc.js + docker rm adc-temp + node $(pwd)/adc.js -v + echo "ADC_BIN=node $(pwd)/adc.js" >> $GITHUB_ENV + - name: Run Benchmark Test shell: bash env: