Skip to content

Commit 13efc1f

Browse files
authored
fix: http handler cut block to pages using memory size after gc. (#19071)
* Datablock add fn memory_size_with_options * fix: http handler cut block to pages using memory size after gc. * Column::memory_size add arg gc. * Column::memory_size add arg gc.
1 parent 6fbb28e commit 13efc1f

File tree

29 files changed

+74
-62
lines changed

29 files changed

+74
-62
lines changed

src/common/column/src/binview/mod.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -314,8 +314,12 @@ impl<T: ViewType + ?Sized> BinaryViewColumnGeneric<T> {
314314
.get_or_init(|| self.views.iter().map(|v| v.length as usize).sum::<usize>())
315315
}
316316

317-
pub fn memory_size(&self) -> usize {
318-
self.total_buffer_len() + self.len() * 16
317+
pub fn memory_size(&self, gc: bool) -> usize {
318+
if gc {
319+
self.total_bytes_len() + self.len() * 16
320+
} else {
321+
self.total_buffer_len() + self.len() * 16
322+
}
319323
}
320324

321325
fn total_unshared_buffer_len(&self) -> usize {

src/common/column/tests/it/binview/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,15 @@ fn test_slice() {
151151
];
152152

153153
let array: Utf8ViewColumn = data.into_iter().collect();
154-
assert_eq!(array.memory_size(), 156);
154+
assert_eq!(array.memory_size(false), 156);
155155

156156
let a3 = array.sliced(2, 3);
157157
assert_eq!(a3.into_iter().collect::<Vec<_>>(), vec![
158158
"databend",
159159
"yyyyyyyyyyyyyyyyyyyyy",
160160
"zzzzzzzzzzzzzzzzzzzzz",
161161
]);
162-
assert_eq!(a3.memory_size(), 108);
162+
assert_eq!(a3.memory_size(false), 108);
163163
}
164164

165165
#[test]

src/query/expression/src/block.rs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,14 @@ impl BlockEntry {
9898
pub fn memory_size(&self) -> usize {
9999
match self {
100100
BlockEntry::Const(scalar, _, _) => scalar.as_ref().memory_size(),
101-
BlockEntry::Column(column) => column.memory_size(),
101+
BlockEntry::Column(column) => column.memory_size(false),
102+
}
103+
}
104+
105+
pub fn memory_size_with_options(&self, gc: bool) -> usize {
106+
match self {
107+
BlockEntry::Const(scalar, _, _) => scalar.as_ref().memory_size(),
108+
BlockEntry::Column(column) => column.memory_size(gc),
102109
}
103110
}
104111

src/query/expression/src/types.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -607,7 +607,8 @@ pub trait AccessType: Debug + Clone + PartialEq + Sized + 'static {
607607
std::mem::size_of::<Self::Scalar>()
608608
}
609609

610-
fn column_memory_size(col: &Self::Column) -> usize {
610+
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
611+
let _ = gc;
611612
Self::column_len(col) * std::mem::size_of::<Self::Scalar>()
612613
}
613614

src/query/expression/src/types/any.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,8 @@ impl AccessType for AnyType {
8383
scalar.memory_size()
8484
}
8585

86-
fn column_memory_size(col: &Self::Column) -> usize {
87-
col.memory_size()
86+
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
87+
col.memory_size(gc)
8888
}
8989

9090
#[inline(always)]

src/query/expression/src/types/array.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,11 @@ impl<T: AccessType> AccessType for ArrayType<T> {
101101
}
102102

103103
fn scalar_memory_size(scalar: &Self::ScalarRef<'_>) -> usize {
104-
<T as AccessType>::column_memory_size(scalar)
104+
<T as AccessType>::column_memory_size(scalar, false)
105105
}
106106

107-
fn column_memory_size(col: &Self::Column) -> usize {
108-
col.memory_size()
107+
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
108+
col.memory_size(gc)
109109
}
110110

111111
fn compare(a: T::Column, b: T::Column) -> Ordering {
@@ -269,8 +269,8 @@ impl<T: AccessType> ArrayColumn<T> {
269269
}
270270
}
271271

272-
pub fn memory_size(&self) -> usize {
273-
T::column_memory_size(&self.underlying_column()) + self.offsets.len() * 8
272+
pub fn memory_size(&self, gc: bool) -> usize {
273+
T::column_memory_size(&self.underlying_column(), gc) + self.offsets.len() * 8
274274
}
275275

276276
// Note: if the array column has been sliced, the number of values may not match the offsets.

src/query/expression/src/types/binary.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ impl AccessType for BinaryType {
100100
scalar.len()
101101
}
102102

103-
fn column_memory_size(col: &Self::Column) -> usize {
103+
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
104104
col.data().len() + col.offsets().len() * 8
105105
}
106106

src/query/expression/src/types/bitmap.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ impl AccessType for BitmapType {
100100
scalar.len()
101101
}
102102

103-
fn column_memory_size(col: &Self::Column) -> usize {
103+
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
104104
col.memory_size()
105105
}
106106

src/query/expression/src/types/compute_view.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ where
111111
std::mem::size_of::<F>()
112112
}
113113

114-
fn column_memory_size(col: &Self::Column) -> usize {
114+
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
115115
F::column_len(col) * std::mem::size_of::<F>()
116116
}
117117

src/query/expression/src/types/generic.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ impl<const INDEX: usize> AccessType for GenericType<INDEX> {
8686
scalar.memory_size()
8787
}
8888

89-
fn column_memory_size(col: &Self::Column) -> usize {
90-
col.memory_size()
89+
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
90+
col.memory_size(gc)
9191
}
9292

9393
#[inline(always)]

0 commit comments

Comments
 (0)