diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c3e9f5b9a6..e5edaad3a57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,19 +17,20 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: get rocksdb sha - id: rocksdb-sha - run: | - ROCKSDB_SHA=$(cd deps/rocksdb && git rev-parse HEAD) - echo "ROCKSDB_SHA is $ROCKSDB_SHA" - echo "ROCKSDB_SHA=$ROCKSDB_SHA" >> $GITHUB_ENV - shell: bash - - name: cache rocksdb - id: cache-rocksdb - uses: actions/cache@v4 - with: - path: deps/rocksdb - key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} + # Temporarily disable rocksdb cache to test full rebuild + # - name: get rocksdb sha + # id: rocksdb-sha + # run: | + # ROCKSDB_SHA=$(cd deps/rocksdb && git rev-parse HEAD) + # echo "ROCKSDB_SHA is $ROCKSDB_SHA" + # echo "ROCKSDB_SHA=$ROCKSDB_SHA" >> $GITHUB_ENV + # shell: bash + # - name: cache rocksdb + # id: cache-rocksdb + # uses: actions/cache@v4 + # with: + # path: deps/rocksdb + # key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} - name: install dependencies run: | sudo apt-get update @@ -82,25 +83,28 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: get rocksdb sha - id: rocksdb-sha - run: | - ROCKSDB_SHA=$(cd deps/rocksdb && git rev-parse HEAD) - echo "ROCKSDB_SHA is $ROCKSDB_SHA" - echo "ROCKSDB_SHA=$ROCKSDB_SHA" >> $GITHUB_ENV - shell: bash - - name: cache rocksdb - id: cache-rocksdb - uses: actions/cache@v4 - with: - path: deps/rocksdb - key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} + # Temporarily disable rocksdb cache to test full rebuild + # - name: get rocksdb sha + # id: rocksdb-sha + # run: | + # ROCKSDB_SHA=$(cd deps/rocksdb && git rev-parse HEAD) + # echo "ROCKSDB_SHA is $ROCKSDB_SHA" + # echo "ROCKSDB_SHA=$ROCKSDB_SHA" >> $GITHUB_ENV + # shell: bash + # - name: cache rocksdb + # id: cache-rocksdb + # uses: actions/cache@v4 + # with: + # path: deps/rocksdb + # key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} - name: install dependencies run: | sudo apt-get update sudo apt-get -y install libsnappy-dev zlib1g-dev libstdc++6 liburing-dev libevent-dev libunwind-dev libgoogle-glog-dev - - name: make test - run: LD_LIBRARY_PATH=/home/runner/work/Redis-On-Rocks/Redis-On-Rocks/deps/rocksdb/_build_folly/libs/glog/lib:/home/runner/work/Redis-On-Rocks/Redis-On-Rocks/deps/rocksdb/_build_folly/libs/libevent/lib make SWAP=1 test -j8 + - name: make + run: make SWAP=1 -j8 + - name: test + run: LD_LIBRARY_PATH=/home/runner/work/Redis-On-Rocks/Redis-On-Rocks/deps/rocksdb/_build_folly/libs/glog/lib:/home/runner/work/Redis-On-Rocks/Redis-On-Rocks/deps/rocksdb/_build_folly/libs/libevent/lib make SWAP=1 test # SWAP mode with Address Sanitizer swap-asan: @@ -112,19 +116,20 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: get rocksdb sha - id: rocksdb-sha - run: | - ROCKSDB_SHA=$(cd deps/rocksdb && git rev-parse HEAD) - echo "ROCKSDB_SHA is $ROCKSDB_SHA" - echo "ROCKSDB_SHA=$ROCKSDB_SHA" >> $GITHUB_ENV - shell: bash - - name: cache rocksdb - id: cache-rocksdb - uses: actions/cache@v4 - with: - path: deps/rocksdb - key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} + # Temporarily disable rocksdb cache to test full rebuild + # - name: get rocksdb sha + # id: rocksdb-sha + # run: | + # ROCKSDB_SHA=$(cd deps/rocksdb && git rev-parse HEAD) + # echo "ROCKSDB_SHA is $ROCKSDB_SHA" + # echo "ROCKSDB_SHA=$ROCKSDB_SHA" >> $GITHUB_ENV + # shell: bash + # - name: cache rocksdb + # id: cache-rocksdb + # uses: actions/cache@v4 + # with: + # path: deps/rocksdb + # key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_SHA }} - name: install dependencies run: | sudo apt-get update diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 4395d62f80f..00000000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: "Codecov" - -# Enabling on each push is to display the coverage changes in every PR, -# where each PR needs to be compared against the coverage of the head commit -on: [push, pull_request] - -jobs: - code-coverage: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - - - name: Install dependencies and lcov - run: | - sudo apt-get update - sudo apt-get install -y lcov libsnappy-dev zlib1g-dev libstdc++6 liburing-dev libevent-dev libunwind-dev libgoogle-glog-dev - - - name: Run test with coverage - run: make lcov - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./src/redis.info diff --git a/.github/workflows/external.yml b/.github/workflows/external.yml deleted file mode 100644 index 0cc863a6913..00000000000 --- a/.github/workflows/external.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: External Server Tests - -on: - pull_request: - push: - schedule: - - cron: '0 0 * * *' - -jobs: - test-external-standalone: - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: install dependencies - run: | - sudo apt-get update - sudo apt-get -y install tcl8.6 tclx libsnappy-dev zlib1g-dev libstdc++6 liburing-dev libevent-dev libgoogle-glog-dev - - name: Build - run: make -j8 - - name: Start redis-server - run: | - ./src/redis-server --daemonize yes --save "" --logfile external-redis.log \ - --enable-protected-configs yes --enable-debug-command yes --enable-module-command yes - - name: Run external test - run: | - ./runtest \ - --host 127.0.0.1 --port 6379 \ - --verbose \ - --tags -slow - - name: Archive redis log - if: ${{ failure() }} - uses: actions/upload-artifact@v4 - with: - name: test-external-redis-log - path: external-redis.log - - test-external-cluster: - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: install dependencies - run: | - sudo apt-get update - sudo apt-get -y install tcl8.6 tclx libsnappy-dev zlib1g-dev libstdc++6 liburing-dev libevent-dev libgoogle-glog-dev - - name: Build - run: make -j8 - - name: Start redis-server - run: | - ./src/redis-server --cluster-enabled yes --daemonize yes --save "" --logfile external-redis-cluster.log \ - --enable-protected-configs yes --enable-debug-command yes --enable-module-command yes - - name: Create a single node cluster - run: ./src/redis-cli cluster addslots $(for slot in {0..16383}; do echo $slot; done); sleep 5 - - name: Run external test - run: | - ./runtest \ - --host 127.0.0.1 --port 6379 \ - --verbose \ - --cluster-mode \ - --tags -slow - - name: Archive redis log - if: ${{ failure() }} - uses: actions/upload-artifact@v4 - with: - name: test-external-cluster-log - path: external-redis-cluster.log - - test-external-nodebug: - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || github.repository == 'redis/redis' - timeout-minutes: 14400 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: install dependencies - run: | - sudo apt-get update - sudo apt-get -y install tcl8.6 tclx libsnappy-dev zlib1g-dev libstdc++6 liburing-dev libevent-dev libgoogle-glog-dev - - name: Build - run: make -j8 - - name: Start redis-server - run: | - ./src/redis-server --daemonize yes --save "" --logfile external-redis-nodebug.log - - name: Run external test - run: | - ./runtest \ - --host 127.0.0.1 --port 6379 \ - --verbose \ - --tags "-slow -needs:debug" - - name: Archive redis log - if: ${{ failure() }} - uses: actions/upload-artifact@v4 - with: - name: test-external-redis-nodebug-log - path: external-redis-nodebug.log diff --git a/deps/Makefile b/deps/Makefile index 3edeecb4ed1..63ebe853bad 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -160,43 +160,48 @@ ROCKSDB_CFLAGS = $(CFLAGS) ROCKSDB_BUILD_FOLLY_FLAGS=ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 ROCKSDB_DISABLE_MALLOC_USABLE_SIZE=1 ROCKSDB_DISABLE_MEMKIND=1 PORTABLE=1 ROCKSDB_USE_IO_URING=1 ROCKSDB_BUILD_FLAGS=USE_COROUTINES=1 USE_FOLLY=1 $(ROCKSDB_BUILD_FOLLY_FLAGS) -./rocksdb/_build_folly/libs/folly/lib/libfolly.a: - if [ ! -f rocksdb/_build_folly/installed/folly/lib/libfolly.a ]; then \ +./rocksdb/_build_folly/folly/lib/libfolly.a: + @if [ ! -f rocksdb/_build_folly/installed/folly/lib/libfolly.a ]; then \ + echo "Building folly for the first time..."; \ cd rocksdb && ${ROCKSDB_BUILD_FOLLY_FLAGS} ${MAKE} checkout_folly && ${ROCKSDB_BUILD_FOLLY_FLAGS} ${MAKE} build_folly; \ fi - rm -rf ./rocksdb/_build_folly/libs - mkdir -p ./rocksdb/_build_folly/libs - FBUILD_PATH=`ls -d ./rocksdb/_build_folly*`; \ - BOOST_PATH=`ls -d $${FBUILD_PATH}/installed/boost*`; \ - DBL_CONV_PATH=`ls -d $${FBUILD_PATH}/installed/double-conversion*`; \ - FMT_PATH=`ls -d $${FBUILD_PATH}/installed/fmt*`; \ - GLOG_PATH=`ls -d $${FBUILD_PATH}/installed/glog*`; \ - GFLAGS_PATH=`ls -d $${FBUILD_PATH}/installed/gflags*`; \ - LIBEVENT_PATH=`ls -d $${FBUILD_PATH}/installed/libevent*`; \ - GLOG_LIB_PATH=`ls -d $${GLOG_PATH}/lib*`; \ - FMT_LIB_PATH=`ls -d $${FMT_PATH}/lib*`; \ - cp -r $${BOOST_PATH} ./rocksdb/_build_folly/libs/boost; \ - cp -r $${DBL_CONV_PATH} ./rocksdb/_build_folly/libs/double-conversion; \ - mkdir -p ./rocksdb/_build_folly/libs/fmt; \ - cp -r $${FMT_PATH}/include ./rocksdb/_build_folly/libs/fmt/include; \ - cp -r $${FMT_LIB_PATH} ./rocksdb/_build_folly/libs/fmt/lib; \ - mkdir -p ./rocksdb/_build_folly/libs/glog; \ - cp -r $${GLOG_PATH}/include ./rocksdb/_build_folly/libs/glog/include; \ - cp -r $${GLOG_LIB_PATH} ./rocksdb/_build_folly/libs/glog/lib; \ - mkdir -p ./rocksdb/_build_folly/libs/libiberty; \ - cp -r $${GFLAGS_PATH} ./rocksdb/_build_folly/libs/gflags; \ - cp -r $${LIBEVENT_PATH} ./rocksdb/_build_folly/libs/libevent; \ - cp -r $${FBUILD_PATH}/installed/folly ./rocksdb/_build_folly/libs/folly - - - -rocksdb: rocksdb/librocksdb.a ./rocksdb/_build_folly/libs/folly/lib/libfolly.a - -rocksdb/librocksdb.a: ./rocksdb/_build_folly/libs/folly/lib/libfolly.a - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)rocksdb$(ENDCOLOR) + @if [ ! -f ./rocksdb/_build_folly/libs/folly/lib/libfolly.a ]; then \ + echo "Copying libraries from installed to libs directory..."; \ + rm -rf ./rocksdb/_build_folly/libs; \ + mkdir -p ./rocksdb/_build_folly/libs; \ + FBUILD_PATH=`ls -d ./rocksdb/_build_folly*`; \ + BOOST_PATH=`ls -d $${FBUILD_PATH}/installed/boost*`; \ + DBL_CONV_PATH=`ls -d $${FBUILD_PATH}/installed/double-conversion*`; \ + FMT_PATH=`ls -d $${FBUILD_PATH}/installed/fmt*`; \ + GLOG_PATH=`ls -d $${FBUILD_PATH}/installed/glog*`; \ + GFLAGS_PATH=`ls -d $${FBUILD_PATH}/installed/gflags*`; \ + LIBEVENT_PATH=`ls -d $${FBUILD_PATH}/installed/libevent*`; \ + GLOG_LIB_PATH=`ls -d $${GLOG_PATH}/lib*`; \ + FMT_LIB_PATH=`ls -d $${FMT_PATH}/lib*`; \ + cp -r $${BOOST_PATH} ./rocksdb/_build_folly/libs/boost; \ + cp -r $${DBL_CONV_PATH} ./rocksdb/_build_folly/libs/double-conversion; \ + mkdir -p ./rocksdb/_build_folly/libs/fmt; \ + cp -r $${FMT_PATH}/include ./rocksdb/_build_folly/libs/fmt/include; \ + cp -r $${FMT_LIB_PATH} ./rocksdb/_build_folly/libs/fmt/lib; \ + mkdir -p ./rocksdb/_build_folly/libs/glog; \ + cp -r $${GLOG_PATH}/include ./rocksdb/_build_folly/libs/glog/include; \ + cp -r $${GLOG_LIB_PATH} ./rocksdb/_build_folly/libs/glog/lib; \ + mkdir -p ./rocksdb/_build_folly/libs/libiberty; \ + cp -r $${GFLAGS_PATH} ./rocksdb/_build_folly/libs/gflags; \ + cp -r $${LIBEVENT_PATH} ./rocksdb/_build_folly/libs/libevent; \ + cp -r $${FBUILD_PATH}/installed/folly ./rocksdb/_build_folly/libs/folly; \ + echo "Libraries copied successfully."; \ + else \ + echo "libs directory already exists, skipping copy to preserve timestamps."; \ + fi + + + +rocksdb: ./rocksdb/_build_folly/folly/lib/libfolly.a .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) cd rocksdb && $(ROCKSDB_BUILD_FLAGS) $(MAKE) CFLAGS="$(ROCKSDB_CFLAGS)" static_lib -# Remove .PHONY for rocksdb to enable incremental builds +.PHONY: rocksdb XREDIS_GTID_CFLAGS= -std=gnu99 -Wall -pipe -g3 -O3 -funroll-loops $(CFLAGS) XREDIS_GTID_LDFLAGS= $(LDFLAGS) diff --git a/src/Makefile b/src/Makefile index dcd8a5facbc..a764b5f1471 100644 --- a/src/Makefile +++ b/src/Makefile @@ -36,10 +36,10 @@ ifneq ($(OPTIMIZATION),-O0) OPTIMIZATION+=-fno-omit-frame-pointer endif DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv fast_float xredis-gtid -NODEPS:=clean distclean ifdef SWAP DEPENDENCY_TARGETS+=tdigest endif +NODEPS:=clean distclean # Default settings STD=-pedantic -DREDIS_STATIC='' @@ -414,7 +414,7 @@ all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCH @echo "Hint: It's a good idea to run 'make test' ;)" @echo "" -Makefile.dep: .make-compile-prerequisites +Makefile.dep: .make-prerequisites -$(REDIS_CC) -MM $(ALL_SOURCES) > Makefile.dep 2> /dev/null || true ifeq (0, $(words $(findstring $(MAKECMDGOALS), $(NODEPS)))) @@ -436,42 +436,24 @@ persist-settings: echo REDIS_LDFLAGS=$(REDIS_LDFLAGS) >> .make-settings echo PREV_FINAL_CFLAGS=$(FINAL_CFLAGS) >> .make-settings echo PREV_FINAL_LDFLAGS=$(FINAL_LDFLAGS) >> .make-settings - @(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS) > /dev/null 2>&1) + @echo "==================== Building dependencies ====================" + cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS) || { echo "ERROR: Dependency build failed!"; exit 1; } .PHONY: persist-settings -# Prerequisites target - separated for compile-time and link-time deps -# Always build dependencies on first run (when .make-compile-prerequisites doesn't exist) -.make-compile-prerequisites: - @if [ ! -f $@ ]; then \ - echo "Building compile dependencies for first time..."; \ - touch $@; \ - echo STD=$(STD) > .make-settings; \ - echo WARN=$(WARN) >> .make-settings; \ - echo OPT=$(OPT) >> .make-settings; \ - echo MALLOC=$(MALLOC) >> .make-settings; \ - echo BUILD_TLS=$(BUILD_TLS) >> .make-settings; \ - echo USE_SYSTEMD=$(USE_SYSTEMD) >> .make-settings; \ - echo CFLAGS=$(CFLAGS) >> .make-settings; \ - echo LDFLAGS=$(LDFLAGS) >> .make-settings; \ - echo REDIS_CFLAGS=$(REDIS_CFLAGS) >> .make-settings; \ - echo REDIS_LDFLAGS=$(REDIS_LDFLAGS) >> .make-settings; \ - echo PREV_FINAL_CFLAGS=$(FINAL_CFLAGS) >> .make-settings; \ - echo PREV_FINAL_LDFLAGS=$(FINAL_LDFLAGS) >> .make-settings; \ - (cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS) > /dev/null 2>&1); \ - else \ - touch $@; \ - fi - -# Clean everything, persist settings and build compile dependencies if anything changed -# Note: These conditional dependencies have been removed to prevent circular dependency issues -# They were originally designed to trigger rebuilds when CFLAGS/LDFLAGS changed - -# Build link dependencies separately -.make-link-prerequisites: .make-compile-prerequisites +# Prerequisites target +.make-prerequisites: @touch $@ -# Build rocksdb independently to support caching +# Clean everything, persist settings and build dependencies if anything changed +ifneq ($(strip $(PREV_FINAL_CFLAGS)), $(strip $(FINAL_CFLAGS))) +.make-prerequisites: persist-settings +endif + +ifneq ($(strip $(PREV_FINAL_LDFLAGS)), $(strip $(FINAL_LDFLAGS))) +.make-prerequisites: persist-settings +endif + ../deps/rocksdb/librocksdb.a: cd ../deps && $(MAKE) rocksdb @@ -495,7 +477,15 @@ SWAP_LIBS= ../deps/rocksdb/librocksdb.a ../deps/tdigest/tdigest.o endif # redis-server -$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_VEC_SETS_OBJ) .make-link-prerequisites +$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_VEC_SETS_OBJ) .make-prerequisites + @echo "==================== Checking required libraries before linking ====================" + @for lib in ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/fast_float/libfast_float.a ../deps/xredis-gtid/lib/libgtid.a $(SWAP_LIBS); do \ + if [ ! -f "$$lib" ]; then \ + echo "ERROR: Required library not found: $$lib"; \ + exit 1; \ + fi; \ + done + @echo "All required libraries found. Proceeding with linking..." $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/fast_float/libfast_float.a ../deps/xredis-gtid/lib/libgtid.a $(SWAP_LIBS) $(FINAL_LIBS) # redis-sentinel @@ -515,12 +505,22 @@ $(TLS_MODULE_NAME): $(REDIS_SERVER_NAME) $(QUIET_CC)$(CC) -o $@ tls.c -shared -fPIC $(TLS_MODULE_CFLAGS) $(TLS_CLIENT_LIBS) # redis-cli +ifdef SWAP +$(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) ../deps/rocksdb/librocksdb.a + $(REDIS_LD) -o $@ $(REDIS_CLI_OBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS) +else $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS) +endif # redis-benchmark +ifdef SWAP +$(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) ../deps/rocksdb/librocksdb.a + $(REDIS_LD) -o $@ $(REDIS_BENCHMARK_OBJ) ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS) +else $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS) +endif DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_VEC_SETS_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) -include $(DEP) @@ -528,11 +528,11 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_VEC_SETS_OBJ:%.o=%.d) $(REDIS_CLI_OBJ: # Because the jemalloc.h header is generated as a part of the jemalloc build, # building it should complete before building any other object. Instead of # depending on a single artifact, build all compile dependencies first. -%.o: %.c .make-compile-prerequisites +%.o: %.c .make-prerequisites $(REDIS_CC) -MMD -o $@ -c $< # commands.o specifically depends on the generated command definition file -commands.o: commands.c $(COMMANDS_DEF_FILENAME).def .make-compile-prerequisites +commands.o: commands.c $(COMMANDS_DEF_FILENAME).def .make-prerequisites $(REDIS_CC) -MMD -o $@ -c $< # The following files are checked in and don't normally need to be rebuilt. They diff --git a/src/networking.c b/src/networking.c index 4240d9c96ee..907687bf063 100644 --- a/src/networking.c +++ b/src/networking.c @@ -5015,7 +5015,9 @@ void evictClients(void) { size_t client_eviction_limit = getClientEvictionLimit(); if (client_eviction_limit == 0) return; - while (server.stat_clients_type_memory[CLIENT_TYPE_TRACKING] >= client_eviction_limit) { + while (server.stat_clients_type_memory[CLIENT_TYPE_TRACKING] + + server.stat_clients_type_memory[CLIENT_TYPE_PUBSUB] + + server.stat_clients_type_memory[CLIENT_TYPE_NORMAL] >= client_eviction_limit) { listNode *ln = listNext(&bucket_iter); if (ln) { client *c = ln->value; diff --git a/src/server.c b/src/server.c index 09285960b07..58795faf3eb 100644 --- a/src/server.c +++ b/src/server.c @@ -1023,7 +1023,7 @@ int clientEvictionAllowed(client *c) { return 0; } int type = getClientType(c); - return (type == CLIENT_TYPE_TRACKING); + return (type == CLIENT_TYPE_TRACKING || type == CLIENT_TYPE_NORMAL || type == CLIENT_TYPE_PUBSUB); } diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl index 23a9daab89c..5ebd3b7584e 100644 --- a/tests/unit/maxmemory.tcl +++ b/tests/unit/maxmemory.tcl @@ -1,4 +1,4 @@ -start_server {tags {"maxmemory" "external:skip"}} { +start_server {tags {"maxmemory" "external:skip" "memonly"}} { r config set maxmemory 11mb r config set maxmemory-policy allkeys-lru set server_pid [s process_id] @@ -147,7 +147,7 @@ start_server {tags {"maxmemory" "external:skip"}} { } -start_server {tags {"maxmemory external:skip"}} { +start_server {tags {"maxmemory external:skip" "memonly"}} { foreach policy { allkeys-random allkeys-lru allkeys-lfu volatile-lru volatile-lfu volatile-random volatile-ttl @@ -283,8 +283,8 @@ proc slave_query_buffer {srv} { proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline} { start_server {tags {"maxmemory external:skip"}} { - start_server {} { - set slave_pid [s process_id] + start_server {tags {memonly} } { + set slave_pid [s process_id] test "$test_name" { set slave [srv 0 client] set slave_host [srv 0 host] @@ -395,7 +395,7 @@ test_slave_buffers {slave buffer are counted correctly} 1000000 10 0 1 # test again with fewer (and bigger) commands without pipeline, but with eviction test_slave_buffers "replica buffer don't induce eviction" 100000 100 1 0 -start_server {tags {"maxmemory external:skip"}} { +start_server {tags {"maxmemory external:skip" "memonly"}} { test {Don't rehash if used memory exceeds maxmemory after rehash} { r config set latency-tracking no r config set maxmemory 0 diff --git a/tests/unit/obuf-limits.tcl b/tests/unit/obuf-limits.tcl index 507affee635..2484c7d837a 100644 --- a/tests/unit/obuf-limits.tcl +++ b/tests/unit/obuf-limits.tcl @@ -21,7 +21,7 @@ start_server {tags {"obuf-limits external:skip logreqres:skip"}} { r config set client-output-buffer-limit "normal 1mb 2mb 60 replica 3mb 4mb 70 pubsub 5mb 6mb 80" set res [lindex [r config get client-output-buffer-limit] 1] - assert_equal $res "normal 1048576 2097152 60 slave 3145728 4194304 70 pubsub 5242880 6291456 80" + assert_equal $res "normal 1048576 2097152 60 slave 3145728 4194304 70 pubsub 5242880 6291456 80 tracking 33554432 8388608 60" # Set back to the original value. r config set client-output-buffer-limit $oldval @@ -267,4 +267,36 @@ start_server {tags {"obuf-limits external:skip logreqres:skip"}} { set fd [$rd channel] assert_equal {} [read $fd] } + + test {No response for tracking client if output buffer hard limit is enforced} { + r config set client-output-buffer-limit {tracking 100000 0 0} + # Total size of all items must be more than 100k + set item [string repeat "x" 1000] + for {set i 0} {$i < 150} {incr i} { + r lpush mylist $item + } + set orig_mem [s used_memory] + # Set client name and get all items + set rd [redis_deferring_client] + $rd client setname mybiglist + assert {[$rd read] eq "OK"} + + $rd client tracking on + after 100 + assert {[$rd read] eq "OK"} + + $rd lrange mylist 0 -1 + after 100 + + # Before we read reply, redis will close this client. + set clients [r client list] + assert_no_match "*name=mybiglist*" $clients + set cur_mem [s used_memory] + # 10k just is a deviation threshold + assert {$cur_mem < 10000 + $orig_mem} + + # Read nothing + set fd [$rd channel] + assert_equal {} [read $fd] + } }