[backport] Avoid overflow in rounding estimation. (dmlc#11910) (dmlc#11911)

Author: trivialfis

src/common/deterministic.cuh

@@ -1,11 +1,11 @@
 /**
- * Copyright 2020-2023 by XGBoost Contributors
+ * Copyright 2020-2026, XGBoost Contributors
  */
 #ifndef XGBOOST_COMMON_DETERMINISTIC_CUH_
 #define XGBOOST_COMMON_DETERMINISTIC_CUH_
 
-#include <cmath>
-#include <limits>          // std::numeric_limits
+#include <cmath>   // for frexp, ldexp
+#include <limits>  // for numeric_limits
 
 #include "xgboost/base.h"  // XGBOOST_DEVICE
 
@@ -22,7 +22,7 @@ namespace common {
  * Summation' by Demmel and Nguyen.
  */
 template <typename T>
-XGBOOST_DEVICE T CreateRoundingFactor(T max_abs, int n) {
+XGBOOST_DEVICE T CreateRoundingFactor(T max_abs, bst_idx_t n) {
   T delta = max_abs / (static_cast<T>(1.0) -
                        static_cast<T>(2.0) * static_cast<T>(n) * std::numeric_limits<T>::epsilon());
 

src/tree/gpu_hist/histogram.cu

@@ -17,8 +17,8 @@
 namespace xgboost::tree {
 namespace {
 struct Pair {
-  GradientPair first;
-  GradientPair second;
+  GradientPairPrecise first;
+  GradientPairPrecise second;
 };
 __host__ XGBOOST_DEV_INLINE Pair operator+(Pair const& lhs, Pair const& rhs) {
   return {lhs.first + rhs.first, lhs.second + rhs.second};
@@ -53,7 +53,7 @@ struct Clip {
     auto ng = Nclip(x.GetGrad());
     auto nh = Nclip(x.GetHess());
 
-    return {GradientPair{pg, ph}, GradientPair{ng, nh}};
+    return {GradientPairPrecise{pg, ph}, GradientPairPrecise{ng, nh}};
   }
 };
 
@@ -81,7 +81,7 @@ GradientQuantiser::GradientQuantiser(Context const* ctx, common::Span<GradientPa
   auto rc = collective::GlobalSum(ctx, info, linalg::MakeVec(reinterpret_cast<ReduceT*>(&p), 4));
   collective::SafeColl(rc);
 
-  GradientPair positive_sum{p.first}, negative_sum{p.second};
+  GradientSumT positive_sum{p.first}, negative_sum{p.second};
 
   std::size_t total_rows = gpair.size();
   rc = collective::GlobalSum(ctx, info, linalg::MakeVec(&total_rows, 1));
@@ -111,8 +111,8 @@ GradientQuantiser::GradientQuantiser(Context const* ctx, common::Span<GradientPa
    * rounding is calcuated as exp(m), see the rounding factor calcuation for
    * details.
    */
-  to_fixed_point_ = GradientSumT(static_cast<T>(1) / to_floating_point_.GetGrad(),
-                                 static_cast<T>(1) / to_floating_point_.GetHess());
+  to_fixed_point_ = GradientSumT{static_cast<T>(1) / to_floating_point_.GetGrad(),
+                                 static_cast<T>(1) / to_floating_point_.GetHess()};
 }
 
 XGBOOST_DEV_INLINE void AtomicAddGpairShared(xgboost::GradientPairInt64* dest,