Skip to content

Commit 260eae5

Browse files
rtadepallirtadepalli
committed
Merge remote-tracking branch 'upstream/main' into GH-79 
2 parents 86887d5 + 1795832 commit 260eae5

21 files changed

+111
-85
lines changed

‎vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ public int getBufferSizeFor(final int count) {
357357
if (count == 0) {
358358
return 0;
359359
}
360-
return (count * typeWidth) + getValidityBufferSizeFromCount(count);
360+
return (count * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(count);
361361
}
362362

363363
/**
@@ -370,7 +370,7 @@ public int getBufferSize() {
370370
if (valueCount == 0) {
371371
return 0;
372372
}
373-
return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount);
373+
return (valueCount * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
374374
}
375375

376376
/**
@@ -534,10 +534,10 @@ private void setReaderAndWriterIndex() {
534534
validityBuffer.writerIndex(0);
535535
valueBuffer.writerIndex(0);
536536
} else {
537-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
537+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
538538
if (typeWidth == 0) {
539539
/* specialized handling for BitVector */
540-
valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
540+
valueBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
541541
} else {
542542
valueBuffer.writerIndex((long) valueCount * typeWidth);
543543
}
@@ -658,7 +658,7 @@ private void splitAndTransferValueBuffer(
658658
protected void sliceAndTransferValidityBuffer(
659659
int startIndex, int length, BaseValueVector target) {
660660
final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
661-
final int byteSizeTarget = getValidityBufferSizeFromCount(length);
661+
final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
662662

663663
if (target.validityBuffer != null) {
664664
target.validityBuffer.getReferenceManager().release();

‎vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ private void setReaderAndWriterIndex() {
377377
valueBuffer.writerIndex(0);
378378
} else {
379379
final long lastDataOffset = getStartOffset(valueCount);
380-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
380+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
381381
offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
382382
valueBuffer.writerIndex(lastDataOffset);
383383
}
@@ -630,7 +630,7 @@ public int getBufferSizeFor(final int valueCount) {
630630
return 0;
631631
}
632632

633-
final long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
633+
final long validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
634634
final long offsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH;
635635
/* get the end offset for this valueCount */
636636
final long dataBufferSize = getStartOffset(valueCount);
@@ -810,7 +810,7 @@ private void splitAndTransferOffsetBuffer(
810810
protected void sliceAndTransferValidityBuffer(
811811
int startIndex, int length, BaseValueVector target) {
812812
final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
813-
final int byteSizeTarget = getValidityBufferSizeFromCount(length);
813+
final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
814814

815815
if (target.validityBuffer != null) {
816816
target.validityBuffer.getReferenceManager().release();

‎vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,14 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) {
114114
return buffer;
115115
}
116116

117-
/* number of bytes for the validity buffer for the given valueCount */
117+
/**
118+
* Compute the size of validity buffer required to manage a given number of elements in a vector.
119+
*
120+
* @param valueCount number of elements in the vector
121+
* @return buffer size
122+
* @deprecated -- use {@link BitVectorHelper#getValidityBufferSizeFromCount} instead.
123+
*/
124+
@Deprecated(forRemoval = true, since = "18.4.0")
118125
protected static int getValidityBufferSizeFromCount(final int valueCount) {
119126
return DataSizeRoundingUtil.divideBy8Ceil(valueCount);
120127
}

‎vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ private void setReaderAndWriterIndex() {
393393
valueBuffer.writerIndex(0);
394394
} else {
395395
final int lastDataOffset = getStartOffset(valueCount);
396-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
396+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
397397
offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
398398
valueBuffer.writerIndex(lastDataOffset);
399399
}
@@ -669,7 +669,7 @@ public int getBufferSizeFor(final int valueCount) {
669669
return 0;
670670
}
671671

672-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
672+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
673673
final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
674674
/* get the end offset for this valueCount */
675675
final int dataBufferSize = offsetBuffer.getInt((long) valueCount * OFFSET_WIDTH);
@@ -856,7 +856,7 @@ private void splitAndTransferOffsetBuffer(
856856
protected void sliceAndTransferValidityBuffer(
857857
int startIndex, int length, BaseValueVector target) {
858858
final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
859-
final int byteSizeTarget = getValidityBufferSizeFromCount(length);
859+
final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
860860

861861
if (target.validityBuffer != null) {
862862
target.validityBuffer.getReferenceManager().release();

‎vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,7 @@ private void setReaderAndWriterIndex() {
392392
validityBuffer.writerIndex(0);
393393
viewBuffer.writerIndex(0);
394394
} else {
395-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
395+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
396396
viewBuffer.writerIndex(valueCount * ELEMENT_SIZE);
397397
}
398398
}
@@ -675,7 +675,7 @@ public int getBufferSizeFor(final int valueCount) {
675675
return 0;
676676
}
677677

678-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
678+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
679679
final int viewBufferSize = valueCount * ELEMENT_SIZE;
680680
final int dataBufferSize = getDataBufferSize();
681681
return validityBufferSize + viewBufferSize + dataBufferSize;
@@ -855,7 +855,7 @@ protected void allocateValidityBuffer(final long size) {
855855
protected void sliceAndTransferValidityBuffer(
856856
int startIndex, int length, BaseValueVector target) {
857857
final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
858-
final int byteSizeTarget = getValidityBufferSizeFromCount(length);
858+
final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
859859

860860
if (target.validityBuffer != null) {
861861
target.validityBuffer.getReferenceManager().release();

‎vector/src/main/java/org/apache/arrow/vector/BitVector.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public MinorType getMinorType() {
9898
*/
9999
@Override
100100
public void setInitialCapacity(int valueCount) {
101-
final int size = getValidityBufferSizeFromCount(valueCount);
101+
final int size = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
102102
if (size * 2L > MAX_ALLOCATION_SIZE) {
103103
throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
104104
}
@@ -121,7 +121,7 @@ public int getBufferSizeFor(final int count) {
121121
if (count == 0) {
122122
return 0;
123123
}
124-
return 2 * getValidityBufferSizeFromCount(count);
124+
return 2 * BitVectorHelper.getValidityBufferSizeFromCount(count);
125125
}
126126

127127
/**
@@ -165,7 +165,7 @@ private ArrowBuf splitAndTransferBuffer(
165165
int startIndex, int length, ArrowBuf sourceBuffer, ArrowBuf destBuffer) {
166166
int firstByteSource = BitVectorHelper.byteIndex(startIndex);
167167
int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
168-
int byteSizeTarget = getValidityBufferSizeFromCount(length);
168+
int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length);
169169
int offset = startIndex % 8;
170170

171171
if (length > 0) {

‎vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,11 @@ public static void setValidityBit(ArrowBuf validityBuffer, int index, int value)
135135
public static ArrowBuf setValidityBit(
136136
ArrowBuf validityBuffer, BufferAllocator allocator, int valueCount, int index, int value) {
137137
if (validityBuffer == null) {
138-
validityBuffer = allocator.buffer(getValidityBufferSize(valueCount));
138+
validityBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount));
139139
}
140140
setValidityBit(validityBuffer, index, value);
141141
if (index == (valueCount - 1)) {
142-
validityBuffer.writerIndex(getValidityBufferSize(valueCount));
142+
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
143143
}
144144

145145
return validityBuffer;
@@ -165,7 +165,7 @@ public static int get(final ArrowBuf buffer, int index) {
165165
* @param valueCount number of elements in the vector
166166
* @return buffer size
167167
*/
168-
public static int getValidityBufferSize(int valueCount) {
168+
public static int getValidityBufferSizeFromCount(int valueCount) {
169169
return DataSizeRoundingUtil.divideBy8Ceil(valueCount);
170170
}
171171

@@ -182,7 +182,7 @@ public static int getNullCount(final ArrowBuf validityBuffer, final int valueCou
182182
return 0;
183183
}
184184
int count = 0;
185-
final int sizeInBytes = getValidityBufferSize(valueCount);
185+
final int sizeInBytes = getValidityBufferSizeFromCount(valueCount);
186186
// If value count is not a multiple of 8, then calculate number of used bits in the last byte
187187
final int remainder = valueCount % 8;
188188
final int fullBytesCount = remainder == 0 ? sizeInBytes : sizeInBytes - 1;
@@ -233,7 +233,7 @@ public static boolean checkAllBitsEqualTo(
233233
if (valueCount == 0) {
234234
return true;
235235
}
236-
final int sizeInBytes = getValidityBufferSize(valueCount);
236+
final int sizeInBytes = getValidityBufferSizeFromCount(valueCount);
237237

238238
// boundary check
239239
validityBuffer.checkBytes(0, sizeInBytes);
@@ -325,7 +325,7 @@ public static ArrowBuf loadValidityBuffer(
325325
sourceValidityBuffer == null || sourceValidityBuffer.capacity() == 0;
326326
if (isValidityBufferNull
327327
&& (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount)) {
328-
newBuffer = allocator.buffer(getValidityBufferSize(valueCount));
328+
newBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount));
329329
newBuffer.setZero(0, newBuffer.capacity());
330330
if (fieldNode.getNullCount() != 0) {
331331
/* all NULLs */

‎vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ public FixedSizeListVector(
108108
this.listSize = ((ArrowType.FixedSizeList) field.getFieldType().getType()).getListSize();
109109
Preconditions.checkArgument(listSize >= 0, "list size must be non-negative");
110110
this.valueCount = 0;
111-
this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
111+
this.validityAllocationSizeInBytes =
112+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
112113
}
113114

114115
@Override
@@ -187,7 +188,7 @@ public List<ArrowBuf> getFieldBuffers() {
187188

188189
private void setReaderAndWriterIndex() {
189190
validityBuffer.readerIndex(0);
190-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
191+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
191192
}
192193

193194
/**
@@ -264,7 +265,8 @@ private void reallocValidityBuffer() {
264265
if (validityAllocationSizeInBytes > 0) {
265266
newAllocationSize = validityAllocationSizeInBytes;
266267
} else {
267-
newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
268+
newAllocationSize =
269+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
268270
}
269271
}
270272

@@ -307,7 +309,7 @@ public UnionFixedSizeListWriter getWriter() {
307309

308310
@Override
309311
public void setInitialCapacity(int numRecords) {
310-
validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
312+
validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords);
311313
vector.setInitialCapacity(numRecords * listSize);
312314
}
313315

@@ -324,15 +326,15 @@ public int getBufferSize() {
324326
if (getValueCount() == 0) {
325327
return 0;
326328
}
327-
return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize();
329+
return BitVectorHelper.getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize();
328330
}
329331

330332
@Override
331333
public int getBufferSizeFor(int valueCount) {
332334
if (valueCount == 0) {
333335
return 0;
334336
}
335-
return getValidityBufferSizeFromCount(valueCount)
337+
return BitVectorHelper.getValidityBufferSizeFromCount(valueCount)
336338
+ vector.getBufferSizeFor(valueCount * listSize);
337339
}
338340

‎vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,8 @@ public LargeListVector(Field field, BufferAllocator allocator, CallBack callBack
129129
this.field = field;
130130
this.validityBuffer = allocator.getEmpty();
131131
this.callBack = callBack;
132-
this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
132+
this.validityAllocationSizeInBytes =
133+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
133134
this.lastSet = -1;
134135
this.offsetBuffer = allocator.getEmpty();
135136
this.vector = vector == null ? DEFAULT_DATA_VECTOR : vector;
@@ -154,7 +155,7 @@ public void initializeChildrenFromFields(List<Field> children) {
154155

155156
@Override
156157
public void setInitialCapacity(int numRecords) {
157-
validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
158+
validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords);
158159
offsetAllocationSizeInBytes = (long) (numRecords + 1) * OFFSET_WIDTH;
159160
if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) {
160161
vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
@@ -182,7 +183,7 @@ public void setInitialCapacity(int numRecords) {
182183
*/
183184
@Override
184185
public void setInitialCapacity(int numRecords, double density) {
185-
validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
186+
validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords);
186187
if ((numRecords * density) >= Integer.MAX_VALUE) {
187188
throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
188189
}
@@ -309,7 +310,7 @@ private void setReaderAndWriterIndex() {
309310
validityBuffer.writerIndex(0);
310311
offsetBuffer.writerIndex(0);
311312
} else {
312-
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
313+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
313314
offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
314315
}
315316
}
@@ -438,7 +439,8 @@ private void reallocValidityBuffer() {
438439
if (validityAllocationSizeInBytes > 0) {
439440
newAllocationSize = validityAllocationSizeInBytes;
440441
} else {
441-
newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
442+
newAllocationSize =
443+
BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
442444
}
443445
}
444446
newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize);
@@ -752,7 +754,7 @@ public int getBufferSize() {
752754
return 0;
753755
}
754756
final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
755-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
757+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
756758
return offsetBufferSize + validityBufferSize + vector.getBufferSize();
757759
}
758760

@@ -761,7 +763,7 @@ public int getBufferSizeFor(int valueCount) {
761763
if (valueCount == 0) {
762764
return 0;
763765
}
764-
final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
766+
final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount);
765767
long innerVectorValueCount = offsetBuffer.getLong((long) valueCount * OFFSET_WIDTH);
766768

767769
return ((valueCount + 1) * OFFSET_WIDTH)

0 commit comments

Comments
 (0)