diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml new file mode 100644 index 0000000000..565fb1d034 --- /dev/null +++ b/.github/workflows/benchmark-test.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Benchmark Test + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + inputs: + routes: + description: "Number of routes for benchmark test" + required: false + default: "2000" + consumers: + description: "Number of consumers for benchmark test" + required: false + default: "2000" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ADC_VERSION: dev + BENCHMARK_ROUTES: ${{ github.event.inputs.routes }} + BENCHMARK_CONSUMERS: ${{ github.event.inputs.consumers }} + +jobs: + e2e-test: + strategy: + matrix: + provider_type: + - apisix-standalone + - apisix + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Go Env + uses: actions/setup-go@v4 + with: + go-version: "1.24" + + - name: Install kind + run: | + go install sigs.k8s.io/kind@v0.23.0 + + - name: Install ginkgo + run: | + make install-ginkgo + + - name: Build images + env: + TAG: dev + ARCH: amd64 + ENABLE_PROXY: "false" + BASE_IMAGE_TAG: "debug" + run: | + echo "building images..." + make build-image + + - name: Launch Kind Cluster + run: | + make kind-up + + - name: Loading Docker Image to Kind Cluster + run: | + make kind-load-images + + - name: Install Gateway API And CRDs + run: | + make install + + - name: Extract adc binary + if: ${{ env.ADC_VERSION == 'dev' }} + run: | + docker create --name adc-temp ghcr.io/api7/adc:dev + docker cp adc-temp:main.js adc.js + docker rm adc-temp + node $(pwd)/adc.js -v + echo "ADC_BIN=node $(pwd)/adc.js" >> $GITHUB_ENV + + - name: Run Benchmark Test + shell: bash + env: + PROVIDER_TYPE: ${{ matrix.provider_type }} + TEST_LABEL: ${{ matrix.cases_subset }} + TEST_ENV: CI + run: | + make benchmark-test diff --git a/Makefile b/Makefile index 39b1d6b0f8..cb41483b13 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ IMG ?= apache/apisix-ingress-controller:$(IMAGE_TAG) ENVTEST_K8S_VERSION = 1.30.0 KIND_NAME ?= apisix-ingress-cluster -ADC_VERSION ?= 0.21.2 +ADC_VERSION ?= 0.23.1 DIR := $(shell pwd) @@ -57,6 +57,9 @@ CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-r ## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC,GATEWAY-TLS +TEST_EXCLUDES ?= /e2e /conformance /benchmark +TEST_PACKAGES = $(shell go list ./... $(foreach p,$(TEST_EXCLUDES),| grep -v $(p))) + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -128,7 +131,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /conformance) -coverprofile cover.out + KUBEBUILDER_ASSETS="$$( $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path )" go test $(TEST_PACKAGES) -coverprofile cover.out .PHONY: kind-e2e-test kind-e2e-test: kind-up build-image kind-load-images e2e-test @@ -153,6 +156,10 @@ conformance-test: --conformance-profiles=$(CONFORMANCE_PROFILES) \ --report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT) +.PHONY: benchmark-test +benchmark-test: + go test -v ./test/benchmark -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v + .PHONY: lint lint: sort-import golangci-lint ## Run golangci-lint linter $(GOLANGCI_LINT) run diff --git a/go.mod b/go.mod index 60da4110ae..757f8e2824 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,10 @@ require ( github.com/hashicorp/go-memdb v1.3.4 github.com/imdario/mergo v0.3.16 github.com/incubator4/go-resty-expr v0.1.1 + github.com/olekukonko/tablewriter v1.1.1 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 + github.com/panjf2000/ants/v2 v2.11.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/samber/lo v1.47.0 @@ -33,6 +35,7 @@ require ( k8s.io/apiextensions-apiserver v0.32.3 k8s.io/apimachinery v0.32.3 k8s.io/client-go v0.32.3 + k8s.io/code-generator v0.32.3 k8s.io/kubectl v0.30.3 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 sigs.k8s.io/controller-runtime v0.20.4 @@ -91,6 +94,9 @@ require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clipperhouse/displaywidth v0.3.1 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect @@ -139,6 +145,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect github.com/miekg/dns v1.1.65 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -150,6 +157,9 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect + github.com/olekukonko/errors v1.1.0 // indirect + github.com/olekukonko/ll v0.1.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.4.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -203,6 +213,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect k8s.io/apiserver v0.32.3 // indirect k8s.io/component-base v0.32.3 // indirect + k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect moul.io/http2curl/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 8904243c0a..eddf72b525 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,12 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/displaywidth v0.3.1 h1:k07iN9gD32177o1y4O1jQMzbLdCrsGJh+blirVYybsk= +github.com/clipperhouse/displaywidth v0.3.1/go.mod h1:tgLJKKyaDOCadywag3agw4snxS5kYEuYR6Y9+qWDDYM= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= @@ -285,6 +291,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= @@ -311,12 +319,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= +github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= +github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.1.2 h1:lkg/k/9mlsy0SxO5aC+WEpbdT5K83ddnNhAepz7TQc0= +github.com/olekukonko/ll v0.1.2/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew= +github.com/olekukonko/tablewriter v1.1.1 h1:b3reP6GCfrHwmKkYwNRFh2rxidGHcT6cgxj/sHiDDx0= +github.com/olekukonko/tablewriter v1.1.1/go.mod h1:De/bIcTF+gpBDB3Alv3fEsZA+9unTsSzAg/ZGADCtn4= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= +github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -579,8 +597,12 @@ k8s.io/apiserver v0.32.3 h1:kOw2KBuHOA+wetX1MkmrxgBr648ksz653j26ESuWNY8= k8s.io/apiserver v0.32.3/go.mod h1:q1x9B8E/WzShF49wh3ADOh6muSfpmFL0I2t+TG0Zdgc= k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/code-generator v0.32.3 h1:31p2TVzC9+hVdSkAFruAk3JY+iSfzrJ83Qij1yZutyw= +k8s.io/code-generator v0.32.3/go.mod h1:+mbiYID5NLsBuqxjQTygKM/DAdKpAjvBzrJd64NU1G8= k8s.io/component-base v0.32.3 h1:98WJvvMs3QZ2LYHBzvltFSeJjEx7t5+8s71P7M74u8k= k8s.io/component-base v0.32.3/go.mod h1:LWi9cR+yPAv7cu2X9rZanTiFKB2kHA+JjmhkKjCZRpI= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= diff --git a/internal/adc/cache/store.go b/internal/adc/cache/store.go index 6c5a1f2394..d41188ed1a 100644 --- a/internal/adc/cache/store.go +++ b/internal/adc/cache/store.go @@ -40,7 +40,7 @@ func NewStore(log logr.Logger) *Store { return &Store{ cacheMap: make(map[string]Cache), pluginMetadataMap: make(map[string]adctypes.PluginMetadata), - log: log, + log: log.WithName("store"), } } diff --git a/internal/adc/client/client.go b/internal/adc/client/client.go index 8a498b1344..2419dde90a 100644 --- a/internal/adc/client/client.go +++ b/internal/adc/client/client.go @@ -198,6 +198,7 @@ func (c *Client) Sync(ctx context.Context) (map[string]types.ADCExecutionErrors, if resources == nil { continue } + c.log.Info("syncing resources for config", "service_number", len(resources.Services)) if err := c.sync(ctx, Task{ Name: name + "-sync", diff --git a/internal/controller/status/updater.go b/internal/controller/status/updater.go index d00aec76a8..e2ef06edf3 100644 --- a/internal/controller/status/updater.go +++ b/internal/controller/status/updater.go @@ -37,7 +37,7 @@ import ( pkgmetrics "github.com/apache/apisix-ingress-controller/pkg/metrics" ) -const UpdateChannelBufferSize = 1000 +const UpdateChannelBufferSize = 10000 type Update struct { NamespacedName k8stypes.NamespacedName @@ -119,7 +119,7 @@ func (u *UpdateHandler) updateStatus(ctx context.Context, update Update) error { newObj.SetUID(obj.GetUID()) - u.log.Info("updating status", "name", update.NamespacedName.Name, + u.log.V(1).Info("updating status", "name", update.NamespacedName.Name, "namespace", update.NamespacedName.Namespace, "kind", types.KindOf(newObj), ) @@ -140,11 +140,10 @@ func (u *UpdateHandler) Start(ctx context.Context) error { case update := <-u.updateChannel: // Decrement queue length after removing item from queue pkgmetrics.DecStatusQueueLength() - u.log.V(1).Info("received a status update", "namespace", update.NamespacedName.Namespace, + u.log.Info("received a status update", "namespace", update.NamespacedName.Namespace, "name", update.NamespacedName.Name, "kind", types.KindOf(update.Resource), ) - u.apply(ctx, update) } } diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 869efefc48..892d027c47 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -1368,7 +1368,7 @@ func ProcessIngressClassParameters(tctx *provider.TranslateContext, c client.Cli return err } - log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) + log.V(1).Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) tctx.GatewayProxies[ingressClassKind] = *gatewayProxy tctx.ResourceParentRefs[objKind] = append(tctx.ResourceParentRefs[objKind], ingressClassKind) diff --git a/internal/manager/readiness/manager.go b/internal/manager/readiness/manager.go index eca380eb8c..b140bd833f 100644 --- a/internal/manager/readiness/manager.go +++ b/internal/manager/readiness/manager.go @@ -125,7 +125,8 @@ func (r *readinessManager) Start(ctx context.Context) error { }) } if len(expected) > 0 { - r.log.V(1).Info("registering readiness state", "gvk", gvk, "expected", expected) + r.log.Info("registering readiness state", "gvk", gvk, "registered_count", len(expected)) + r.log.V(1).Info("registered resources for readiness", "gvk", gvk, "resources", expected) r.registerState(gvk, expected) } } @@ -135,13 +136,12 @@ func (r *readinessManager) Start(ctx context.Context) error { r.isReady.Store(true) close(r.done) } + r.log.Info("readiness manager started") }) return err } func (r *readinessManager) registerState(gvk schema.GroupVersionKind, list []k8stypes.NamespacedName) { - r.mu.Lock() - defer r.mu.Unlock() if _, ok := r.state[gvk]; !ok { r.state[gvk] = make(map[k8stypes.NamespacedName]struct{}) } @@ -155,9 +155,12 @@ func (r *readinessManager) Done(obj client.Object, nn k8stypes.NamespacedName) { if r.IsReady() { return } + <-r.started + r.mu.Lock() defer r.mu.Unlock() gvk := types.GvkOf(obj) + r.log.Info("marking resource as done", "gvk", gvk, "name", nn, "state_count", len(r.state[gvk])) if _, ok := r.state[gvk]; !ok { return } @@ -191,7 +194,7 @@ func (r *readinessManager) WaitReady(ctx context.Context, timeout time.Duration) case <-ctx.Done(): return false case <-time.After(timeout): - return true + return false case <-r.done: return true } diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go index d0d8e48a05..029675e219 100644 --- a/internal/provider/apisix/provider.go +++ b/internal/provider/apisix/provider.go @@ -76,7 +76,9 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan o.DefaultBackendMode = ProviderTypeAPISIX } - cli, err := adcclient.New(log, o.DefaultBackendMode, o.SyncTimeout) + logger := log.WithName("provider") + + cli, err := adcclient.New(logger, o.DefaultBackendMode, o.SyncTimeout) if err != nil { return nil, err } @@ -88,7 +90,7 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan updater: updater, readier: readier, syncCh: make(chan struct{}, 1), - log: log.WithName("provider"), + log: logger, }, nil } @@ -249,7 +251,10 @@ func (d *apisixProvider) buildConfig(tctx *provider.TranslateContext, nnk types. } func (d *apisixProvider) Start(ctx context.Context) error { + d.log.Info("starting provider, waiting for readiness") d.readier.WaitReady(ctx, 5*time.Minute) + d.log.Info("Ready detected, starting sync loop") + initalSyncDelay := d.InitSyncDelay if initalSyncDelay > 0 { time.AfterFunc(initalSyncDelay, d.syncNotify) diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go new file mode 100644 index 0000000000..f32f8cf7c5 --- /dev/null +++ b/test/benchmark/benchmark_test.go @@ -0,0 +1,430 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package benchmark + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "time" + + "github.com/api7/gopkg/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var report = &BenchmarkReport{} +var totalRoutes = 2000 +var totalConsumers = 2000 + +var _ = BeforeSuite(func() { + routes := os.Getenv("BENCHMARK_ROUTES") + if routes != "" { + _, err := fmt.Sscanf(routes, "%d", &totalRoutes) + Expect(err).NotTo(HaveOccurred(), "parsing BENCHMARK_ROUTES") + } + consumers := os.Getenv("BENCHMARK_CONSUMERS") + if consumers != "" { + _, err := fmt.Sscanf(consumers, "%d", &totalConsumers) + Expect(err).NotTo(HaveOccurred(), "parsing BENCHMARK_CONSUMERS") + } +}) +var _ = AfterSuite(func() { + report.PrintTable() +}) + +const gatewayProxyYaml = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + service: + name: %s + port: 9180 + auth: + type: AdminKey + adminKey: + value: "%s" +` + +var _ = Describe("Benchmark Test", func() { + var ( + s = scaffold.NewDefaultScaffold() + controlAPIClient scaffold.ControlAPIClient + ) + + BeforeEach(func() { + By("port-forward to control api service") + var err error + controlAPIClient, err = s.ControlAPIClient() + Expect(err).NotTo(HaveOccurred(), "create control api client") + }) + + Context("Benchmark ApisixRoute", func() { + const ingressClassYaml = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: apisix +spec: + controller: "%s" + parameters: + apiGroup: "apisix.apache.org" + kind: "GatewayProxy" + name: "apisix-proxy-config" + namespace: %s + scope: "Namespace" +` + + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /get + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + var apisixRouteSpecHeaders = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /headers + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + + var apisixUpstreamSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-service-e2e-test +spec: + ingressClassName: apisix + scheme: https +` + var apisixRouteSpecKeyAuth = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: key-auth +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /get + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + authentication: + enable: true + type: keyAuth +` + var keyAuth = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixConsumer +metadata: + name: %s +spec: + ingressClassName: apisix + authParameter: + keyAuth: + value: + key: %s +` + + getRouteName := func(i int) string { + return fmt.Sprintf("test-route-%04d", i) + } + + createBatchApisixRoutes := func(number int) string { + var buf bytes.Buffer + for i := 0; i < number; i++ { + name := getRouteName(i) + fmt.Fprintf(&buf, apisixRouteSpec, name, name) + buf.WriteString("\n---\n") + } + return buf.String() + } + getConsumerName := func(i int) string { + return fmt.Sprintf("consumer-%04d", i) + } + createBatchConsumers := func(number int) string { + var buf bytes.Buffer + for i := 0; i < number; i++ { + name := getConsumerName(i) + fmt.Fprintf(&buf, keyAuth, name, name) + buf.WriteString("\n---\n") + } + return buf.String() + } + + benchmark := func(scenario string) { + s.Deployer.ScaleIngress(0) + By(fmt.Sprintf("prepare %d ApisixRoutes", totalRoutes)) + err := s.CreateResourceFromString(createBatchApisixRoutes(totalRoutes)) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes") + s.Deployer.ScaleIngress(1) + + now := time.Now() + By(fmt.Sprintf("start cale time for applying %d ApisixRoutes to take effect", totalRoutes)) + err = s.EnsureNumService(controlAPIClient, func(actual int) bool { return actual == totalRoutes }) + Expect(err).ShouldNot(HaveOccurred()) + costTime := time.Since(now) + report.Add(scenario, fmt.Sprintf("Apply %d ApisixRoutes", totalRoutes), costTime) + + By("Test the time required for an ApisixRoute update to take effect") + name := getRouteName(int(time.Now().Unix())) + err = s.CreateResourceFromString(fmt.Sprintf(apisixRouteSpecHeaders, name, name)) + Expect(err).NotTo(HaveOccurred()) + now = time.Now() + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode + }).WithTimeout(15 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + report.Add(scenario, fmt.Sprintf("Update a single ApisixRoute base on %d ApisixRoutes", totalRoutes), time.Since(now)) + + By("Test the time required for a service endpoint change to take effect") + err = s.ScaleHTTPBIN(2) + Expect(err).NotTo(HaveOccurred(), "scale httpbin deployment") + now = time.Now() + err = s.EnsureNumUpstreamNodes(controlAPIClient, "", 2) + Expect(err).ShouldNot(HaveOccurred()) + costTime = time.Since(now) + report.Add(scenario, fmt.Sprintf("Service endpoint change base on %d ApisixRoutes", totalRoutes), costTime) + + By("Test the time required for an ApisixUpstream update to take effect") + err = s.CreateResourceFromString(apisixUpstreamSpec) + Expect(err).NotTo(HaveOccurred(), "creating ApisixUpstream") + now = time.Now() + err = s.ExpectUpstream(controlAPIClient, "", func(upstream adctypes.Upstream) bool { + if upstream.Scheme != "https" { + log.Warnf("expect upstream: [%s] scheme to be https, but got [%s]", upstream.Name, upstream.Scheme) + return false + } + return true + }) + Expect(err).ShouldNot(HaveOccurred()) + costTime = time.Since(now) + report.Add(scenario, fmt.Sprintf("Update ApisixUpstream base on %d ApisixRoutes", totalRoutes), costTime) + } + + BeforeEach(func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, s.GetControllerName(), s.Namespace()), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass") + time.Sleep(5 * time.Second) + }) + It("benchmark ApisixRoute", func() { + benchmark("ApisixRoute Benchmark") + }) + It("10 apisix-standalone pod scale benchmark", func() { + if framework.ProviderType != framework.ProviderTypeAPISIXStandalone { + Skip("only apisix-standalone support scale benchmark") + } + s.Deployer.ScaleDataplane(10) + benchmark("ApisixRoute Benchmark with 10 apisix-standalone pods") + }) + It("ApisixRoute With Consumers benchmark", func() { + s.Deployer.ScaleIngress(0) + By(fmt.Sprintf("prepare %d ApisixConsumers", totalRoutes)) + err := s.CreateResourceFromString(createBatchConsumers(totalRoutes)) + Expect(err).NotTo(HaveOccurred(), "creating ApisixConsumers") + err = s.CreateResourceFromString(apisixRouteSpecKeyAuth) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute with KeyAuth") + s.Deployer.ScaleIngress(1) + + now := time.Now() + Eventually(func() error { + consumer, err := s.DefaultDataplaneResource().Consumer().List(context.Background()) + if err != nil { + return err + } + if len(consumer) != totalConsumers { + return fmt.Errorf("expect %d consumers, but got %d", totalConsumers, len(consumer)) + } + return nil + }).WithTimeout(15*time.Minute).ProbeEvery(1*time.Second).ShouldNot(HaveOccurred(), "waiting for all consumers to be synced to APISIX") + costTime := time.Since(now) + report.AddResult(TestResult{ + Scenario: "ApisixRoute With Consumers Benchmark", + CaseName: fmt.Sprintf("Apply %d ApisixConsumers and ApisixRoute with KeyAuth", totalConsumers), + CostTime: costTime, + IsRequestGateway: true, + }) + }) + }) + + Context("Benchmark HTTPRoute", func() { + const httpRouteSpec = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: %s +spec: + parentRefs: + - name: %s + rules: + - matches: + - path: + type: Exact + value: /get + headers: + - type: Exact + name: X-Route-Name + value: %s + # name: get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + const httpRouteSpecHeaders = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: %s +spec: + parentRefs: + - name: %s + rules: + - matches: + - path: + type: Exact + value: /headers + headers: + - type: Exact + name: X-Route-Name + value: %s + # name: get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + createBatchHTTPRoutes := func(number int, parentGateway string) string { + var buf bytes.Buffer + for i := 0; i < number; i++ { + name := getRouteName(i) + fmt.Fprintf(&buf, httpRouteSpec, name, parentGateway, name) + buf.WriteString("\n---\n") + } + return buf.String() + } + + BeforeEach(func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create GatewayClass") + Expect(s.CreateResourceFromString(s.GetGatewayClassYaml())).NotTo(HaveOccurred(), "creating GatewayClass") + + By("create Gateway") + Expect(s.CreateResourceFromString(s.GetGatewayYaml())).NotTo(HaveOccurred(), "creating Gateway") + time.Sleep(5 * time.Second) + }) + + It("benchmark HTTPRoute", func() { + s.Deployer.ScaleIngress(0) + By(fmt.Sprintf("prepare %d HTTPRoute", totalRoutes)) + err := s.CreateResourceFromString(createBatchHTTPRoutes(totalRoutes, s.Namespace())) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") + s.Deployer.ScaleIngress(1) + + now := time.Now() + By(fmt.Sprintf("start cale time for applying %d HTTPRoute to take effect", totalRoutes)) + err = s.EnsureNumService(controlAPIClient, func(actual int) bool { return actual == totalRoutes }) + Expect(err).ShouldNot(HaveOccurred()) + costTime := time.Since(now) + report.Add("HTTPRoute Benchmark", fmt.Sprintf("Apply %d HTTPRoute", totalRoutes), costTime) + + By("Test the time required for an HTTPRoute update to take effect") + name := getRouteName(int(time.Now().Unix())) + err = s.CreateResourceFromString(fmt.Sprintf(httpRouteSpecHeaders, name, s.Namespace(), name)) + Expect(err).NotTo(HaveOccurred()) + now = time.Now() + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode + }).WithTimeout(5 * time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + report.AddResult(TestResult{ + Scenario: "HTTPRoute Benchmark", + CaseName: fmt.Sprintf("Update a single HTTPRoute base on %d HTTPRoute", totalRoutes), + CostTime: time.Since(now), + }) + + By("Test the time required for a service endpoint change to take effect") + err = s.ScaleHTTPBIN(2) + Expect(err).NotTo(HaveOccurred(), "scale httpbin deployment") + now = time.Now() + err = s.EnsureNumUpstreamNodes(controlAPIClient, "", 2) + Expect(err).ShouldNot(HaveOccurred()) + costTime = time.Since(now) + report.Add("HTTPRoute Benchmark", fmt.Sprintf("Service endpoint change base on %d HTTPRoute", totalRoutes), costTime) + }) + }) +}) + +func getRouteName(i int) string { + return fmt.Sprintf("test-route-%04d", i) +} diff --git a/test/benchmark/suite_test.go b/test/benchmark/suite_test.go new file mode 100644 index 0000000000..32f60af83d --- /dev/null +++ b/test/benchmark/suite_test.go @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package benchmark + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +// Run long-term-stability tests using Ginkgo runner. +func TestBenchmark(t *testing.T) { + RegisterFailHandler(Fail) + var f = framework.NewFramework() + _ = f + + scaffold.NewDeployer = scaffold.NewAPISIXDeployer + + _, _ = fmt.Fprintf(GinkgoWriter, "Starting Benchmark Tests\n") + RunSpecs(t, "Benchmark Tests Suite") +} diff --git a/test/benchmark/utis.go b/test/benchmark/utis.go new file mode 100644 index 0000000000..35de0b908b --- /dev/null +++ b/test/benchmark/utis.go @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package benchmark + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/api7/gopkg/pkg/log" + "github.com/olekukonko/tablewriter" + "go.uber.org/zap" +) + +type TestResult struct { + Scenario string `json:"scenario"` + CaseName string `json:"case_name"` + CostTime time.Duration `json:"cost_time"` + IsRequestGateway bool `json:"is_request_gateway,omitempty"` +} + +type BenchmarkReport struct { + Results []TestResult +} + +func (r *BenchmarkReport) PrintTable() { + table := tablewriter.NewWriter(os.Stdout) + table.Header([]string{"Scenario", "Case", "Cost", "IsRequestGateway"}) + + for _, res := range r.Results { + if err := table.Append([]any{ + res.Scenario, + res.CaseName, + res.CostTime.String(), + res.IsRequestGateway, + }); err != nil { + log.Errorw("failed to append row to table", zap.Error(err)) + } + } + if err := table.Render(); err != nil { + log.Errorw("failed to render table", zap.Error(err)) + } +} + +func (r *BenchmarkReport) PrintJSON() { + b, _ := json.MarshalIndent(r.Results, "", " ") + fmt.Println(string(b)) +} + +func (r *BenchmarkReport) AddResult(result TestResult) { + r.Results = append(r.Results, result) +} + +func (r *BenchmarkReport) Add(scenario, caseName string, cost time.Duration) { + r.Results = append(r.Results, TestResult{ + Scenario: scenario, + CaseName: caseName, + CostTime: cost, + }) +} diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml deleted file mode 100644 index 0eda2bc8af..0000000000 --- a/test/e2e/framework/manifests/apisix-standalone.yaml +++ /dev/null @@ -1,153 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: apisix-conf -data: - config.yaml: | - deployment: - role: traditional - role_traditional: - config_provider: yaml - admin: - allow_admin: - - 0.0.0.0/0 - admin_key: - - key: {{ .AdminKey }} - name: admin - role: admin - nginx_config: - worker_processes: 2 - error_log_level: info - apisix: - proxy_mode: http&stream - stream_proxy: # TCP/UDP proxy - tcp: # TCP proxy port list - - 9100 - - addr: 9110 - tls: true - udp: # UDP proxy port list - - 9200 - discovery: - dns: - servers: - - "10.96.0.10:53" # use the real address of your dns server. - # currently we use KIND as the standard test environment, so here we can hard-code the default DNS address first. ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apisix - labels: - app.kubernetes.io/name: apisix -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: apisix - template: - metadata: - labels: - app.kubernetes.io/name: apisix - spec: - initContainers: - - name: config-setup - image: apache/apisix:dev - command: - - sh - - -c - - | - echo "Copying default config directory to writable volume" - cp -r /usr/local/apisix/conf/* /tmp/apisix-conf/ - echo "Overwriting config.yaml with custom configuration" - cp /tmp/config-source/config.yaml /tmp/apisix-conf/config.yaml - echo "Config setup completed successfully" - ls -la /tmp/apisix-conf/ - volumeMounts: - - name: config-source - mountPath: /tmp/config-source - - name: config-writable - mountPath: /tmp/apisix-conf - containers: - - name: apisix - image: apache/apisix:dev - ports: - - name: http - containerPort: 9080 - protocol: TCP - - name: https - containerPort: 9443 - protocol: TCP - - name: admin - containerPort: 9180 - protocol: TCP - - name: tcp - containerPort: 9100 - protocol: TCP - - name: udp - containerPort: 9200 - protocol: UDP - - name: tls - containerPort: 9110 - protocol: TCP - volumeMounts: - - name: config-writable - mountPath: /usr/local/apisix/conf - volumes: - - name: config-source - configMap: - name: apisix-conf - - name: config-writable - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ .ServiceName }} - labels: - app.kubernetes.io/name: apisix -spec: - ports: - - port: {{ .ServiceHTTPPort }} - name: http - protocol: TCP - targetPort: 9080 - - port: {{ .ServiceHTTPSPort }} - name: https - protocol: TCP - targetPort: 9443 - - port: 9180 - name: admin - protocol: TCP - targetPort: 9180 - - name: tcp - port: 9100 - protocol: TCP - targetPort: 9100 - - name: udp - port: 9200 - protocol: UDP - targetPort: 9200 - - name: tls - port: 9110 - protocol: TCP - targetPort: 9110 - selector: - app.kubernetes.io/name: apisix - type: {{ .ServiceType | default "NodePort" }} diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index 31581bcc2b..eada504150 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -42,6 +42,9 @@ data: nginx_config: worker_processes: 2 error_log_level: info + meta: + lua_shared_dict: + standalone-config: 50m apisix: proxy_mode: http&stream stream_proxy: # TCP/UDP proxy @@ -168,3 +171,20 @@ spec: selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} + +--- + +apiVersion: v1 +kind: Service +metadata: + name: apisix-control-api + labels: + app.kubernetes.io/name: apisix-control-api +spec: + ports: + - port: 9090 + name: control + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/name: apisix diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index c23acd6a87..799cc798c1 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -20,11 +20,13 @@ package scaffold import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "net/url" "os" "strings" + "time" "github.com/api7/gopkg/pkg/log" "github.com/gavv/httpexpect/v2" @@ -32,10 +34,13 @@ import ( "github.com/gruntwork-io/terratest/modules/testing" . "github.com/onsi/ginkgo/v2" //nolint:staticcheck . "github.com/onsi/gomega" //nolint:staticcheck + "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" + adctypes "github.com/apache/apisix-ingress-controller/api/adc" apiv2 "github.com/apache/apisix-ingress-controller/api/v2" "github.com/apache/apisix-ingress-controller/test/e2e/framework" ) @@ -556,3 +561,104 @@ func (s *Scaffold) GetMetricsEndpoint() string { s.addFinalizers(tunnel.Close) return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint()) } + +func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) { + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090) + if err := tunnel.ForwardPortE(s.t); err != nil { + return nil, err + } + s.addFinalizers(tunnel.Close) + + return &controlAPI{ + client: NewClient("http", tunnel.Endpoint()), + }, nil +} + +func (s *Scaffold) EnsureNumService(controlAPIClient ControlAPIClient, matcher func(result int) bool) error { + times := 0 + return wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + times++ + results, _, err := controlAPIClient.ListServices() + if err != nil { + log.Errorw("failed to ListServices", zap.Error(err)) + return false, nil + } + if !matcher(len(results)) { + log.Debugw("number of effective services", zap.Int("number", len(results)), zap.Int("times", times)) + return false, nil + } + return true, nil + }) +} + +func (s *Scaffold) ExpectUpstream(controlAPIClient ControlAPIClient, name string, matcher func(upstream adctypes.Upstream) bool) error { + times := 0 + return wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + times++ + upstreams, _, err := controlAPIClient.ListUpstreams() + if err != nil { + log.Errorw("failed to ListServices", zap.Error(err)) + return false, nil + } + for _, upstream := range upstreams { + upsValue := upstream.(map[string]any) + data, err := json.Marshal(upsValue["value"]) + if err != nil { + return false, fmt.Errorf("failed to marshal upstream: %v", err) + } + + var ups adctypes.Upstream + if err := json.Unmarshal(data, &ups); err != nil { + return false, fmt.Errorf("failed to unmarshal upstream: %v", err) + } + if name != "" && ups.Name != name { + continue + } + if ok := matcher(ups); !ok { + return false, nil + } + } + return true, nil + }) +} + +func (s *Scaffold) EnsureNumUpstreamNodes(controlAPIClient ControlAPIClient, name string, number int) error { + return s.ExpectUpstream(controlAPIClient, name, func(upstream adctypes.Upstream) bool { + if len(upstream.Nodes) != number { + log.Warnf("expect upstream: [%s] nodes num to be %d, but got %d", upstream.Name, number, len(upstream.Nodes)) + return false + } + return true + }) +} + +type ControlAPIClient interface { + ListServices() ([]any, int64, error) + ListUpstreams() ([]any, int64, error) +} + +type controlAPI struct { + client *httpexpect.Expect +} + +func (c *controlAPI) ListUpstreams() (result []any, total int64, err error) { + resp := c.client.Request(http.MethodGet, "/v1/upstreams").Expect() + if resp.Raw().StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw()) + } + if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err) + } + return result, int64(len(result)), err +} + +func (c *controlAPI) ListServices() (result []any, total int64, err error) { + resp := c.client.Request(http.MethodGet, "/v1/services").Expect() + if resp.Raw().StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw()) + } + if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err) + } + return result, int64(len(result)), err +}