docs: update documentation for DataFrame iteration and RecordBatch usage

Author: kosiew

docs/source/user-guide/dataframe/index.rst

@@ -131,13 +131,13 @@ Terminal Operations
 -------------------
 
 To materialize the results of your DataFrame operations, call a terminal method or iterate over the
-``DataFrame`` to consume :py:class:`datafusion.record_batch.RecordBatch` objects lazily:
+``DataFrame`` to consume :py:class:`~datafusion.record_batch.RecordBatch` objects lazily:
 
 .. code-block:: python
 
     # Iterate over the DataFrame to stream record batches
     for batch in df:
-        ...  # batch is a datafusion.record_batch.RecordBatch
+        ...  # each batch is a RecordBatch (use batch.to_pyarrow() for PyArrow)
 
     # Collect all data as PyArrow RecordBatches
     result_batches = df.collect()

python/datafusion/dataframe.py

@@ -295,7 +295,7 @@ class DataFrame:
     up the plan without executing it, and results are only materialized during a
     terminal operation (for example, :py:meth:`collect`, :py:meth:`show`, or
     :py:meth:`to_pandas`) or when iterating over the DataFrame, which yields
-    :class:`pyarrow.RecordBatch` objects lazily.
+    :py:class:`~datafusion.record_batch.RecordBatch` objects lazily.
 
     See :ref:`user_guide_concepts` in the online documentation for more information.
     """
@@ -1134,25 +1134,16 @@ def __arrow_c_stream__(self, requested_schema: object | None = None) -> object:
         return self.df.__arrow_c_stream__(requested_schema)
 
     def __iter__(self) -> Iterator[RecordBatch]:
-        """Yield :class:`datafusion.record_batch.RecordBatch` objects lazily.
+        """Yield record batches from this DataFrame lazily.
 
-        This delegates to :py:meth:`to_stream` without converting each batch to a
-        :class:`pyarrow.RecordBatch`. Use
-        :py:meth:`datafusion.record_batch.RecordBatch.to_pyarrow` when a
-        :class:`pyarrow.RecordBatch` is required.
+        This delegates to :py:meth:`to_stream` without eagerly materializing the
+        entire result set.
         """
-        for batch in self.to_stream():
-            yield batch
+        return iter(self.to_stream())
 
     def __aiter__(self) -> AsyncIterator[RecordBatch]:
-        """Asynchronously yield :class:`datafusion.record_batch.RecordBatch` objects lazily."""
-        stream = self.to_stream()
-
-        async def iterator() -> AsyncIterator[RecordBatch]:
-            async for batch in stream:
-                yield batch
-
-        return iterator()
+        """Asynchronously yield record batches from this DataFrame lazily."""
+        return self.to_stream()
 
     def transform(self, func: Callable[..., DataFrame], *args: Any) -> DataFrame:
         """Apply a function to the current DataFrame which returns another DataFrame.