Merge remote-tracking branch 'upstream/main' into read-csv-from-directory

Author: fangchenli

.github/workflows/docbuild-and-upload.yml

@@ -57,8 +57,6 @@ jobs:
       run: python web/pandas_web.py web/pandas --target-path=web/build
 
     - name: Build documentation
-      # TEMP don't let errors fail the build until all string dtype changes are fixed
-      continue-on-error: true
       run: doc/make.py --warnings-are-errors
 
     - name: Build the interactive terminal

ci/code_checks.sh

@@ -74,6 +74,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.ordinal GL08" \
+        -i "pandas.errors.IncompatibleFrequency SA01,SS06,EX01" \
         -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \

doc/source/reference/testing.rst

@@ -36,6 +36,7 @@ Exceptions and warnings
    errors.DuplicateLabelError
    errors.EmptyDataError
    errors.IncompatibilityWarning
+   errors.IncompatibleFrequency
    errors.IndexingError
    errors.InvalidColumnName
    errors.InvalidComparison

doc/source/user_guide/basics.rst

@@ -590,7 +590,7 @@ arguments. The special value ``all`` can also be used:
 
 .. ipython:: python
 
-    frame.describe(include=["object"])
+    frame.describe(include=["str"])
     frame.describe(include=["number"])
     frame.describe(include="all")
 

doc/source/whatsnew/v0.13.0.rst

@@ -184,7 +184,7 @@ API changes
   .. ipython:: python
      :okwarning:
 
-     dfc.loc[0]['A'] = 1111
+     dfc.loc[0]['B'] = 1111
 
   ::
 
@@ -198,7 +198,7 @@ API changes
 
   .. ipython:: python
 
-     dfc.loc[0, 'A'] = 11
+     dfc.loc[0, 'B'] = 1111
      dfc
 
 - ``Panel.reindex`` has the following call signature ``Panel.reindex(items=None, major_axis=None, minor_axis=None, **kwargs)``

doc/source/whatsnew/v0.15.0.rst

@@ -1025,20 +1025,49 @@ Other:
 - :func:`describe` on mixed-types DataFrames is more flexible. Type-based column filtering is now possible via the ``include``/``exclude`` arguments.
   See the :ref:`docs <basics.describe>` (:issue:`8164`).
 
-  .. ipython:: python
+  .. code-block:: python
 
-    df = pd.DataFrame({'catA': ['foo', 'foo', 'bar'] * 8,
-                       'catB': ['a', 'b', 'c', 'd'] * 6,
-                       'numC': np.arange(24),
-                       'numD': np.arange(24.) + .5})
-    df.describe(include=["object"])
-    df.describe(include=["number", "object"], exclude=["float"])
+    >>> df = pd.DataFrame({'catA': ['foo', 'foo', 'bar'] * 8,
+    ...                    'catB': ['a', 'b', 'c', 'd'] * 6,
+    ...                    'numC': np.arange(24),
+    ...                    'numD': np.arange(24.) + .5})
+    >>> df.describe(include=["object"])
+           catA catB
+    count    24   24
+    unique    2    4
+    top     foo    a
+    freq     16    6
+    >>> df.describe(include=["number", "object"], exclude=["float"])
+           catA catB       numC
+    count    24   24  24.000000
+    unique    2    4        NaN
+    top     foo    a        NaN
+    freq     16    6        NaN
+    mean    NaN  NaN  11.500000
+    std     NaN  NaN   7.071068
+    min     NaN  NaN   0.000000
+    25%     NaN  NaN   5.750000
+    50%     NaN  NaN  11.500000
+    75%     NaN  NaN  17.250000
+    max     NaN  NaN  23.000000
 
   Requesting all columns is possible with the shorthand 'all'
 
-  .. ipython:: python
+  .. code-block:: python
 
-    df.describe(include='all')
+    >>> df.describe(include='all')
+           catA catB       numC       numD
+    count    24   24  24.000000  24.000000
+    unique    2    4        NaN        NaN
+    top     foo    a        NaN        NaN
+    freq     16    6        NaN        NaN
+    mean    NaN  NaN  11.500000  12.000000
+    std     NaN  NaN   7.071068   7.071068
+    min     NaN  NaN   0.000000   0.500000
+    25%     NaN  NaN   5.750000   6.250000
+    50%     NaN  NaN  11.500000  12.000000
+    75%     NaN  NaN  17.250000  17.750000
+    max     NaN  NaN  23.000000  23.500000
 
   Without those arguments, ``describe`` will behave as before, including only numerical columns or, if none are, only categorical columns. See also the :ref:`docs <basics.describe>`
 

doc/source/whatsnew/v3.0.0.rst

@@ -414,6 +414,8 @@ Other API changes
 - Index set operations (like union or intersection) will now ignore the dtype of
   an empty ``RangeIndex`` or empty ``Index`` with object dtype when determining
   the dtype of the resulting Index (:issue:`60797`)
+- :class:`IncompatibleFrequency` now subclasses ``TypeError`` instead of ``ValueError``. As a result, joins with mismatched frequencies now cast to object like other non-comparable joins, and arithmetic with indexes with mismatched frequencies align (:issue:`55782`)
+- Comparison operations between :class:`Index` and :class:`Series` now consistently return :class:`Series` regardless of which object is on the left or right (:issue:`36759`)
 - Numpy functions like ``np.isinf`` that return a bool dtype when called on a :class:`Index` object now return a bool-dtype :class:`Index` instead of ``np.ndarray`` (:issue:`52676`)
 
 .. ---------------------------------------------------------------------------
@@ -693,6 +695,7 @@ Datetimelike
 - Bug in :attr:`is_year_start` where a DateTimeIndex constructed via a date_range with frequency 'MS' wouldn't have the correct year or quarter start attributes (:issue:`57377`)
 - Bug in :class:`DataFrame` raising ``ValueError`` when ``dtype`` is ``timedelta64`` and ``data`` is a list containing ``None`` (:issue:`60064`)
 - Bug in :class:`Timestamp` constructor failing to raise when ``tz=None`` is explicitly specified in conjunction with timezone-aware ``tzinfo`` or data (:issue:`48688`)
+- Bug in :class:`Timestamp` constructor failing to raise when given a ``np.datetime64`` object with non-standard unit (:issue:`25611`)
 - Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`)
 - Bug in :func:`date_range` where using a negative frequency value would not include all points between the start and end values (:issue:`56147`)
 - Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`)

pandas/_libs/tslibs/conversion.pyx

@@ -5,6 +5,7 @@ import numpy as np
 cimport numpy as cnp
 from libc.math cimport log10
 from numpy cimport (
+    PyDatetimeScalarObject,
     float64_t,
     int32_t,
     int64_t,
@@ -358,6 +359,7 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
     cdef:
         _TSObject obj
         NPY_DATETIMEUNIT reso
+        int64_t num
 
     obj = _TSObject()
 
@@ -367,6 +369,13 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
     if checknull_with_nat_and_na(ts):
         obj.value = NPY_NAT
     elif cnp.is_datetime64_object(ts):
+        num = (<PyDatetimeScalarObject*>ts).obmeta.num
+        if num != 1:
+            raise ValueError(
+                # GH#25611
+                "np.datetime64 objects with units containing a multiplier are "
+                "not supported"
+            )
         reso = get_supported_reso(get_datetime64_unit(ts))
         obj.creso = reso
         obj.value = get_datetime64_nanos(ts, reso)

pandas/_libs/tslibs/period.pyi

@@ -15,7 +15,7 @@ from pandas._typing import (
 INVALID_FREQ_ERR_MSG: str
 DIFFERENT_FREQ: str
 
-class IncompatibleFrequency(ValueError): ...
+class IncompatibleFrequency(TypeError): ...
 
 def periodarr_to_dt64arr(
     periodarr: npt.NDArray[np.int64],  # const int64_t[:]

pandas/_libs/tslibs/period.pyx

@@ -1625,7 +1625,11 @@ DIFFERENT_FREQ = ("Input has different freq={other_freq} "
                   "from {cls}(freq={own_freq})")
 
 
-class IncompatibleFrequency(ValueError):
+class IncompatibleFrequency(TypeError):
+    """
+    Raised when trying to compare or operate between Periods with different
+    frequencies.
+    """
     pass