sharedTags and sharedTagsTtl in Redis never expire and grow indefinitely if revalidate is not called explicitly

[sad-scope]

Hello!

When using @fortedigital/nextjs-cache-handler/redis-strings, Redis not only stores the actual cache entries but also creates sharedTags and sharedTagsTtl keys.
Both sharedTags and sharedTagsTtl are created with TTL = -1, meaning they never expire.

Entries inside these structures are never cleaned up and keep accumulating over time.
This leads to unbounded growth in Redis usage, which can reach gigabytes and significantly impact performance and storage costs.

The cache lifetime is 300 seconds. And redis clears the keys when ttl ends.

Steps to reproduce:
Configure caching with @fortedigital/nextjs-cache-handler/redis-strings.
Request multiple pages so they are cached.
Inspect Redis:
sharedTags and sharedTagsTtl keys exist,
Both have TTL = -1,
Their size keeps growing with more cached entries.
Expected behavior:
sharedTags and sharedTagsTtl should either:
Expire together with the cache entries they reference, or
Be periodically cleaned up to prevent unbounded growth.
Actual behavior:
sharedTags and sharedTagsTtl never expire.
They continuously accumulate data and can consume large amounts of memory.
And the records inside these structures (sharedTags and sharedTagsTtl) are not cleared

Environment:
Package: @fortedigital/nextjs-cache-handler: 2.0.3
Redis version: 5.8.2
Node.js version: 22.19.0
Next.js version: 15.3.4

[AyronK]

Hey, thanks for reporting. This issue comes back from the original neshca package and we actually made a fix as one of the first things done in this repository. It's definitely surprising it comes back as I have not noticed it in our projects - I'll double check.

Would you mind sharing your cache handler configuration?

[sad-scope]

Hello! Here code of the cache handler

const { createClient } = require('redis');
const { PHASE_PRODUCTION_BUILD } = require('next/constants');

// @fortedigital/nextjs-cache-handler DEPENDENCIES
const createLruHandler =
  require('@fortedigital/nextjs-cache-handler/local-lru').default;
const createCompositeHandler =
  require('@fortedigital/nextjs-cache-handler/composite').default;
const createRedisHandler =
  require('@fortedigital/nextjs-cache-handler/redis-strings').default;
const { CacheHandler } = require('@fortedigital/nextjs-cache-handler');

const withTimeout = (promise, ms, errorMessage) => {
  if (typeof ms !== 'number' || ms <= 0 || Number.isNaN(ms)) {
    return Promise.reject(new Error('Timeout value must be a positive number'));
  }

  const timeout = new Promise((_, reject) => {
    const rejectTimeout = () => {
      reject(new Error(errorMessage || 'Operation timed out'));
    };

    setTimeout(rejectTimeout, ms);
  });

  return Promise.race([promise, timeout]);
};

CacheHandler.onCreation(() => {
  // CONDITIONALS
  if (global.cacheHandlerConfig) return global.cacheHandlerConfig;
  if (global.cacheHandlerConfigPromise) return global.cacheHandlerConfigPromise;

  // MAIN PROMISE
  global.cacheHandlerConfigPromise = (async () => {
    let redisClient = null;
    if (PHASE_PRODUCTION_BUILD !== process.env.NEXT_PHASE) {
      try {
        redisClient = createClient({
          url: `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT}`,
          database: process.env.REDIS_DB,
        });
        redisClient.on('error', (e) => {
          console.warn('Redis error', e);
          global.cacheHandlerConfig = null;
          global.cacheHandlerConfigPromise = null;
        });
      } catch (error) {
        console.warn('Failed to create Redis client:', error);
      }
    }

    if (redisClient) {
      try {
        console.info('Connecting Redis client...');
        await withTimeout(
          redisClient.connect(),
          5000,
          'Redis connection timed out after 5 seconds',
        );
        console.info('Redis client connected.');
      } catch (error) {
        console.warn('Failed to connect Redis client:', error);

        console.info('Attempting to disconnect Redis client...');
        try {
          await withTimeout(
            redisClient.disconnect(),
            2000,
            'Redis disconnection timed out after 2 seconds',
          );
        } catch (disconnectError) {
          console.warn(
            'Failed to disconnect Redis client:',
            disconnectError.message,
          );
        }
      }
    }
    const lruCache = createLruHandler();

    if (!redisClient?.isReady) {
      console.error('Failed to initialize caching layer.');
      global.cacheHandlerConfigPromise = null;
      global.cacheHandlerConfig = { handlers: [lruCache] };

      return global.cacheHandlerConfig;
    }

    let redisCacheHandler;

    try {
      redisCacheHandler = await withTimeout(
        createRedisHandler({
          client: redisClient,
          keyPrefix: `nextCache:`,
          timeoutMs: 1000,
          sharedTagsKey: '__sharedTags__',
          sharedTagsTtlKey: '__sharedTagsTtl__',
        }),
        3000,
        'Redis handler creation timed out after 3 seconds',
      );

      console.info('Redis handler created successfully.');
    } catch (error) {
      console.warn('Failed to create Redis handler:', error.message);
    }

    global.cacheHandlerConfigPromise = null;
    global.cacheHandlerConfig = {
      handlers: [
        createCompositeHandler({
          handlers: [lruCache, redisCacheHandler].filter(Boolean),
          setStrategy: (ctx) => (ctx?.tags.includes('memory-cache') ? 0 : 1),
        }),
      ],
    };

    return global.cacheHandlerConfig;
  })();

  return global.cacheHandlerConfigPromise;
});

module.exports = CacheHandler;

[m-sanders]

I faced a similar issue, it is further exacerbated by using a keyPrefix that is scoped to the application version. I worked around the issue by configuring redis to use an allkeys-lru eviction policy and limiting the size of the shared tags to a sensible max size.

[AyronK]

I double checked on our production application and we don't have this issue, both sharedTags and sharedTagsTtl is revisited and cleaned up properly. I will try to reproduce your issue on a minimal setup project next week. I had other priorities last two weeks and didn't have time to check properly, I'll follow up on this.

Just to clarify the way the algorithm works sharedTags and sharedTagsTtl are expected to be never expiring keys, there are maps where we store tags and their expiration time for tag-based revalidation in Next.js. The package contains a mechanism clearing expired tags and their ttls on expiration, but I believe this will only work for keys with defined TTL. Once again sharedTags and sharedTagsTtl are expected to stay forever, they should be cleared periodically by the package so if you report the forever increasing size of these maps I expect there might be a bug in that code. I'll put my hands on it next week, thanks for the patience.

[FrederikAF]

Found a fix that works in our setup. Looks like the sharedTags maps are only cleaned up when calling revalidateTag, so we added a function that revalidates a non-existing tag every x-minutes to our cache-handler.mjs.

const startCleanupInterval = (handler) => {
    // Use a global flag to ensure this interval is only started once per process.
    if (globalThis.cacheCleanupIntervalStarted) {
        return;
    }
    globalThis.cacheCleanupIntervalStarted = true;

    const DUMMY_CLEANUP_TAG = 'internal-periodic-cache-cleanup-trigger';
    const CLEANUP_INTERVAL_MS = 1000 * 60 * 60 * 6; // Clean up once every 6 hours.

    console.info(`CACHE HANDLER: Starting automatic cleanup interval every ${CLEANUP_INTERVAL_MS / 1000 / 60} minutes.`);
    setInterval(() => {
        console.log('CACHE HANDLER: Triggering automatic cleanup of expired tag metadata...');
        // By calling revalidateTag with a dummy tag, we force the handler to run
        // its internal cleanup logic without affecting any real cached data.
        handler.revalidateTag(DUMMY_CLEANUP_TAG)
            .catch(err => console.error("CACHE HANDLER: Automatic cleanup failed.", err));
    }, CLEANUP_INTERVAL_MS);
};

...

async function createCacheConfig() {
    const redisClient = await setupRedisClient();
    let activeHandler;

    if (redisClient?.isReady) {
        console.info("CACHE HANDLER: Using Forte Digital Redis handler.");
        activeHandler = createRedisHandler({
            client: redisClient,
            timeoutMs: 1000,
            keyExpirationStrategy: "EXAT",
            keyPrefix: "nextjs-cache:",
            revalidateTagQuerySize: 10000,
        });
        startCleanupInterval(activeHandler);
    } else {
        console.warn("CACHE HANDLER: Redis client not ready. Falling back to LRU handler.");
        activeHandler = createLruHandler({
            maxItemsNumber: 10000,
            maxItemSizeBytes: 1024 * 1024 * 50, // 50MB
        });
    }

    return {
        handlers: [activeHandler],
        ttl: {
            defaultStaleAge: 30,
            estimateExpireAge: () => 3600,
        },
    };
}

[AyronK]

That's a very important detail and explains why I could not reproduce it or spot it in our projects! Unfortunately you're right, there is no background process running cleanups. This "never expire and grow indefinitely" issue was one of the biggest issues we had with the original neshca package, and that's why we introduced sharedTagsTtl hashmap and our own cleanup mechanism. It assumed people use revalidate, which isn't true in all situation as I can see. I'll try to integrate some kind of background cleanup in the future releases.