Use of prefetching conditionally.

Using the prefetch buffer only helps if there are enough objects.  Use
the long-lived count to decide if it's worth enabling.  If not, fallback
to the previous method of marking objects alive (dereference object
pointers as we encounter them).

Improve code by adding some additional helper functions, adding comments
and general tidying.  The buffer logic has been changed to use a mask
for size rather than the % operator.  Some other small optimizations
that only help a little.

Author: nascheme

Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-22-34.gh-issue-129201.wiZzEb.rst

@@ -1,4 +1,5 @@
-The free-threaded version of the cyclic garbage collector has been optimized
-to use CPU prefetch instructions during the collection.  On large object
-graphs, this can reduce collection times by making it more likely that data
-is in the CPU cache when its needed.
+The free-threaded version of the cyclic garbage collector has been optimized to
+conditionally use CPU prefetch instructions during the collection.  This can
+reduce collection times by making it more likely that data is in the CPU cache
+when it is needed.  The prefetch instructions are enabled if the number of
+long-lived objects (objects surviving a full collection) exceeds a threshold.

Python/gc_free_threading.c

@@ -485,9 +485,10 @@ gc_maybe_untrack(PyObject *op)
 // enough time between the enqueue and dequeue so that the needed memory
 // for the object, most importantly ob_gc_bits and ob_type words, will
 // already be in the CPU cache.
-#define BUFFER_SIZE 256
+#define BUFFER_SIZE 256  // this must be a power of 2
 #define BUFFER_HI 16
 #define BUFFER_LO 8
+#define BUFFER_MASK (BUFFER_SIZE - 1)
 
 // Prefetch intructions will fetch the line of data from memory that
 // contains the byte specified with the source operand to a location in
@@ -554,15 +555,63 @@ typedef struct {
 } gc_span_stack_t;
 
 typedef struct {
-    Py_ssize_t enqueued;
-    Py_ssize_t dequeued;
+    unsigned int in;
+    unsigned int out;
     _PyObjectStack stack;
     gc_span_stack_t spans;
     PyObject *buffer[BUFFER_SIZE];
+    bool use_prefetch;
 } gc_mark_args_t;
 
-// Called when we run out of space in the buffer.  The object will be added
-// to gc_mark_args.stack instead.
+
+// Returns number of entries in buffer
+static inline unsigned int
+gc_mark_buffer_len(gc_mark_args_t *args)
+{
+    return args->in - args->out;
+}
+
+// Returns number of free entry slots in buffer
+static inline unsigned int
+gc_mark_buffer_avail(gc_mark_args_t *args)
+{
+    return BUFFER_SIZE - gc_mark_buffer_len(args);
+}
+
+static inline bool
+gc_mark_buffer_is_empty(gc_mark_args_t *args)
+{
+    return args->in == args->out;
+}
+
+static inline bool
+gc_mark_buffer_is_full(gc_mark_args_t *args)
+{
+    return gc_mark_buffer_len(args) == BUFFER_SIZE;
+}
+
+static inline PyObject *
+gc_mark_buffer_pop(gc_mark_args_t *args)
+{
+    assert(!gc_mark_buffer_is_empty(args));
+    PyObject *op = args->buffer[args->out & BUFFER_MASK];
+    args->out++;
+    return op;
+}
+
+// Called when there is space in the buffer for the object.  Issue the
+// prefetch instruction and add it to the end of the buffer.
+static inline void
+gc_mark_buffer_push(PyObject *op, gc_mark_args_t *args)
+{
+    assert(!gc_mark_buffer_is_full(args));
+    prefetch(op);
+    args->buffer[args->in & BUFFER_MASK] = op;
+    args->in++;
+}
+
+// Called when we run out of space in the buffer or if the prefetching
+// is disabled. The object will be pushed on the gc_mark_args.stack.
 static int
 gc_mark_stack_push(_PyObjectStack *ms, PyObject *op)
 {
@@ -575,6 +624,9 @@ gc_mark_stack_push(_PyObjectStack *ms, PyObject *op)
 static int
 gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end)
 {
+    if (start == end) {
+        return 0;
+    }
     if (ss->size >= ss->capacity) {
         if (ss->capacity == 0) {
             ss->capacity = 256;
@@ -594,27 +646,36 @@ gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end)
     return 0;
 }
 
-// Called when there is space in the buffer for the object.  Add it to the end
-// of the buffer and issue the prefetch instruction.
-static void
-gc_mark_buffer_push(PyObject *op, gc_mark_args_t *args)
+static int
+gc_mark_enqueue_no_buffer(PyObject *op, gc_mark_args_t *args)
 {
-#ifdef Py_DEBUG
-    Py_ssize_t buf_used = args->enqueued - args->dequeued;
-    assert(buf_used < BUFFER_SIZE);
-#endif
-    prefetch(op);
-    args->buffer[args->enqueued % BUFFER_SIZE] = op;
-    args->enqueued++;
+    if (op == NULL) {
+        return 0;
+    }
+    if (!gc_has_bit(op,  _PyGC_BITS_TRACKED)) {
+        return 0;
+    }
+    if (gc_is_alive(op)) {
+        return 0; // already visited this object
+    }
+    if (gc_maybe_untrack(op)) {
+        return 0; // was untracked, don't visit it
+    }
+
+    // Need to call tp_traverse on this object. Add to stack and mark it
+    // alive so we don't traverse it a second time.
+    gc_set_alive(op);
+    if (_PyObjectStack_Push(&args->stack, op) < 0) {
+        return -1;
+    }
+    return 0;
 }
 
-// Called when we find an object that needs to be marked alive (either from a
-// root or from calling tp_traverse).
 static int
-gc_mark_enqueue(PyObject *op, gc_mark_args_t *args)
+gc_mark_enqueue_buffer(PyObject *op, gc_mark_args_t *args)
 {
     assert(op != NULL);
-    if (args->enqueued - args->dequeued < BUFFER_SIZE) {
+    if (!gc_mark_buffer_is_full(args)) {
         gc_mark_buffer_push(op, args);
         return 0;
     }
@@ -623,12 +684,31 @@ gc_mark_enqueue(PyObject *op, gc_mark_args_t *args)
     }
 }
 
+// Called when we find an object that needs to be marked alive (either from a
+// root or from calling tp_traverse).
+static int
+gc_mark_enqueue(PyObject *op, gc_mark_args_t *args)
+{
+    if (args->use_prefetch) {
+        return gc_mark_enqueue_buffer(op, args);
+    }
+    else {
+        return gc_mark_enqueue_no_buffer(op, args);
+    }
+}
+
+// Called when we have a contigous sequence of PyObject pointers, either
+// a tuple or list object.  This will add the items to the buffer if there
+// is space for them all otherwise push a new "span" on the span stack.  Using
+// spans has the advantage of not creating a deep _PyObjectStack stack when
+// dealing with long sequences.  Those sequences will be processed in smaller
+// chunks by the gc_prime_from_spans() function.
 static int
 gc_mark_enqueue_span(PyObject **item, Py_ssize_t size, gc_mark_args_t *args)
 {
-    Py_ssize_t used = args->enqueued - args->dequeued;
+    Py_ssize_t used = gc_mark_buffer_len(args);
     Py_ssize_t free = BUFFER_SIZE - used;
-    if (free > size) {
+    if (free >= size) {
         for (Py_ssize_t i = 0; i < size; i++) {
             PyObject *op = item[i];
             if (op == NULL) {
@@ -694,9 +774,9 @@ gc_abort_mark_alive(PyInterpreterState *interp,
                     struct collection_state *state,
                     gc_mark_args_t *args)
 {
-    // We failed to allocate memory for "stack" while doing the "mark
-    // alive" phase.  In that case, free the object stack and make sure
-    // that no objects have the alive bit set.
+    // We failed to allocate memory while doing the "mark alive" phase.
+    // In that case, free the memory used for marking state and make
+    // sure that no objects have the alive bit set.
     _PyObjectStack_Clear(&args->stack);
     if (args->spans.stack != NULL) {
         PyMem_Free(args->spans.stack);
@@ -1089,24 +1169,26 @@ move_legacy_finalizer_reachable(struct collection_state *state);
 static void
 gc_prime_from_spans(gc_mark_args_t *args)
 {
-    Py_ssize_t space = BUFFER_HI - (args->enqueued - args->dequeued);
-    assert(space >= 1); // needed to make progress
+    Py_ssize_t space = BUFFER_HI - gc_mark_buffer_len(args);
+    // there should always be at least this amount of space
+    assert(space <= gc_mark_buffer_avail(args));
+    assert(space > 0);
     gc_span_t entry = args->spans.stack[--args->spans.size];
-    while (entry.start < entry.end) {
+    // spans on the stack should always have one or more elements
+    assert(entry.start < entry.end);
+    do {
         PyObject *op = *entry.start;
+        entry.start++;
         if (op != NULL) {
-            if (space > 0) {
-                gc_mark_buffer_push(op, args);
-                space--;
-            }
-            else {
-                // no more space in buffer, push remaining
+            gc_mark_buffer_push(op, args);
+            space--;
+            if (space == 0) {
+                // buffer is as full was we want and not done with span
                 gc_mark_span_push(&args->spans, entry.start, entry.end);
-                break;
+                return;
             }
         }
-        entry.start++;
-    }
+    } while (entry.start < entry.end);
 }
 
 static void
@@ -1120,36 +1202,36 @@ gc_prime_buffer(gc_mark_args_t *args)
         // likely cause the stack to be used shortly after when it
         // fills. We want to use the buffer as much as possible and so
         // we only fill to BUFFER_HI, not BUFFER_SIZE.
-        Py_ssize_t space = BUFFER_HI - (args->enqueued - args->dequeued);
-        while (space > 0) {
+        Py_ssize_t space = BUFFER_HI - gc_mark_buffer_len(args);
+        assert(space > 0);
+        do {
             PyObject *op = _PyObjectStack_Pop(&args->stack);
             if (op == NULL) {
                 return;
             }
             gc_mark_buffer_push(op, args);
             space--;
-        }
+        } while (space > 0);
     }
 }
 
 static int
-gc_propagate_alive(gc_mark_args_t *args)
+gc_propagate_alive_prefetch(gc_mark_args_t *args)
 {
     for (;;) {
-        Py_ssize_t buf_used = args->enqueued - args->dequeued;
+        Py_ssize_t buf_used = gc_mark_buffer_len(args);
         if (buf_used <= BUFFER_LO) {
             // The mark buffer is getting empty.  If it's too empty
             // then there will not be enough delay between issuing
             // the prefetch and when the object is actually accessed.
             // Prime the buffer with object pointers from the stack or
             // from the spans, if there are any available.
             gc_prime_buffer(args);
-            if (args->enqueued == args->dequeued) {
-                return 0; // buffer empty, done
+            if (gc_mark_buffer_is_empty(args)) {
+                return 0;
             }
         }
-        PyObject *op = args->buffer[args->dequeued % BUFFER_SIZE];
-        args->dequeued++;
+        PyObject *op = gc_mark_buffer_pop(args);
 
         if (!gc_has_bit(op, _PyGC_BITS_TRACKED)) {
             continue;
@@ -1174,12 +1256,35 @@ gc_propagate_alive(gc_mark_args_t *args)
                 return -1;
             }
         }
-        else if (traverse(op, (visitproc)&gc_mark_enqueue, args) < 0) {
+        else if (traverse(op, (visitproc)&gc_mark_enqueue_buffer, args) < 0) {
             return -1;
         }
     }
 }
 
+static int
+gc_propagate_alive(gc_mark_args_t *args)
+{
+    if (args->use_prefetch) {
+        return gc_propagate_alive_prefetch(args);
+    }
+    else {
+        for (;;) {
+            PyObject *op = _PyObjectStack_Pop(&args->stack);
+            if (op == NULL) {
+                break;
+            }
+            assert(_PyObject_GC_IS_TRACKED(op));
+            assert(gc_is_alive(op));
+            traverseproc traverse = Py_TYPE(op)->tp_traverse;
+            if (traverse(op, (visitproc)&gc_mark_enqueue_no_buffer, args) < 0) {
+                return -1;
+            }
+        }
+        return 0;
+    }
+}
+
 // Using tp_traverse, mark everything reachable from known root objects
 // (which must be non-garbage) as alive (_PyGC_BITS_ALIVE is set).  In
 // most programs, this marks nearly all objects that are not actually
@@ -1206,6 +1311,14 @@ gc_mark_alive_from_roots(PyInterpreterState *interp,
 #endif
     gc_mark_args_t mark_args = { 0 };
 
+    // Using prefetch instructions is only a win if the set of objects being
+    // examined by the GC does not fit into CPU caches.  Otherwise, using the
+    // buffer and prefetch instructions is just overhead.  Using the long lived
+    // object count seems a good estimate of if things will fit in the cache.
+    // On 64-bit platforms, the minimum object size is 32 bytes.  A 4MB L2 cache
+    // would hold about 130k objects.
+    mark_args.use_prefetch = interp->gc.long_lived_total > 200000;
+
     #define MARK_ENQUEUE(op) \
         if (op != NULL ) { \
             if (gc_mark_enqueue(op, &mark_args) < 0) { \