Skip to content

Commit a70c580

Browse files
GiorgiGiorgi
committed
Merge branch 'Appender-Data-Chunk' into develop
2 parents 2bcbd41 + 1fa8ed5 commit a70c580

14 files changed

+324
-21
lines changed

‎DuckDB.NET.Bindings/DuckDBNativeObjects.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,10 +159,10 @@ public void Dispose()
159159
}
160160

161161
[StructLayout(LayoutKind.Sequential)]
162-
public readonly struct DuckDBListEntry
162+
public struct DuckDBListEntry(ulong offset, ulong length)
163163
{
164-
public ulong Offset { get; }
165-
public ulong Length { get; }
164+
public ulong Offset { get; private set; } = offset;
165+
public ulong Length { get; private set; } = length;
166166
}
167167

168168
public struct DuckDBString

‎DuckDB.NET.Bindings/NativeMethods/NativeMethods.Vectors.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ public static class Vectors
3333
[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_list_vector_get_size")]
3434
public static extern long DuckDBListVectorGetSize(IntPtr vector);
3535

36+
[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_list_vector_reserve")]
37+
public static extern DuckDBState DuckDBListVectorReserve(IntPtr vector, ulong requiredCapacity);
38+
3639
[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_struct_vector_get_child")]
3740
public static extern IntPtr DuckDBStructVectorGetChild(IntPtr vector, long index);
3841

‎DuckDB.NET.Data/DuckDBAppender.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using DuckDB.NET.Data.Internal.Writer;
1+
using DuckDB.NET.Data.Internal;
2+
using DuckDB.NET.Data.Internal.Writer;
23
using DuckDB.NET.Native;
34
using System;
45
using System.Diagnostics;
@@ -8,7 +9,7 @@ namespace DuckDB.NET.Data;
89

910
public class DuckDBAppender : IDisposable
1011
{
11-
private static readonly ulong DuckDBVectorSize = NativeMethods.Helpers.DuckDBVectorSize();
12+
private static readonly ulong DuckDBVectorSize = DuckDBGlobalData.VectorSize;
1213

1314
private bool closed;
1415
private readonly Native.DuckDBAppender nativeAppender;

‎DuckDB.NET.Data/DuckDBAppenderRow.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using DuckDB.NET.Data.Internal.Writer;
22
using DuckDB.NET.Native;
33
using System;
4+
using System.Collections.Generic;
45
using System.Numerics;
56

67
namespace DuckDB.NET.Data;
@@ -101,6 +102,12 @@ public DuckDBAppenderRow AppendValue(TimeSpan? value)
101102

102103
#endregion
103104

105+
#region Composite Types
106+
107+
public DuckDBAppenderRow AppendValue<T>(IEnumerable<T>? value) => AppendValueInternal(value);
108+
109+
#endregion
110+
104111
private DuckDBAppenderRow AppendValueInternal<T>(T? value)
105112
{
106113
CheckColumnAccess();

‎DuckDB.NET.Data/DuckDBDataReader.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,15 +76,18 @@ private bool InitChunkData()
7676

7777
currentChunkRowCount = (ulong)NativeMethods.DataChunks.DuckDBDataChunkGetSize(currentChunk);
7878

79-
vectorReaders = new VectorDataReaderBase[fieldCount];
80-
79+
if (vectorReaders.Length != fieldCount)
80+
{
81+
vectorReaders = new VectorDataReaderBase[fieldCount];
82+
}
83+
8184
for (int index = 0; index < fieldCount; index++)
8285
{
8386
var vector = NativeMethods.DataChunks.DuckDBDataChunkGetVector(currentChunk, index);
8487

8588
using var logicalType = NativeMethods.Query.DuckDBColumnLogicalType(ref currentResult, index);
8689

87-
vectorReaders[index] = VectorDataReaderFactory.CreateReader(vector, logicalType,
90+
vectorReaders[index] = VectorDataReaderFactory.CreateReader(vector, logicalType, vectorReaders[index]?.ColumnName ??
8891
NativeMethods.Query.DuckDBColumnName(ref currentResult, index).ToManagedString(false));
8992
}
9093

‎DuckDB.NET.Data/Internal/DuckDBGlobalData.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
using DuckDB.NET.Native;
2+
3+
namespace DuckDB.NET.Data.Internal;
4+
5+
public static class DuckDBGlobalData
6+
{
7+
public static ulong VectorSize { get; } = NativeMethods.Helpers.DuckDBVectorSize();
8+
}

‎DuckDB.NET.Data/Internal/IsExternalInit.cs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System.Collections.Generic;
1+
using System.Collections;
2+
using System.Collections.Generic;
23

34
namespace System.Runtime.CompilerServices
45
{
@@ -15,6 +16,21 @@ public static void Deconstruct<TKey, TValue>(this KeyValuePair<TKey, TValue> key
1516
}
1617
}
1718

19+
static class IEnumerableExtensions
20+
{
21+
public static bool TryGetNonEnumeratedCount<T>(this IEnumerable target, out int count)
22+
{
23+
if (target is ICollection collection)
24+
{
25+
count = collection.Count;
26+
return true;
27+
}
28+
29+
count = 0;
30+
return false;
31+
}
32+
}
33+
1834
namespace System.Diagnostics.CodeAnalysis
1935
{
2036
[AttributeUsage(AttributeTargets.Method)]

‎DuckDB.NET.Data/Internal/Writer/ListVectorDataWriter.cs‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
using DuckDB.NET.Native;
2+
using System;
3+
using System.Collections;
4+
using System.Collections.Generic;
5+
using System.Numerics;
6+
7+
namespace DuckDB.NET.Data.Internal.Writer;
8+
9+
internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase
10+
{
11+
private ulong offset = 0;
12+
private readonly ulong arraySize;
13+
private readonly VectorDataWriterBase listItemWriter;
14+
15+
public bool IsList => ColumnType == DuckDBType.List;
16+
private ulong vectorReservedSize = DuckDBGlobalData.VectorSize;
17+
18+
public ListVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType, DuckDBLogicalType logicalType) : base(vector, vectorData, columnType)
19+
{
20+
using var childType = IsList ? NativeMethods.LogicalType.DuckDBListTypeChildType(logicalType) : NativeMethods.LogicalType.DuckDBArrayTypeChildType(logicalType);
21+
var childVector = IsList ? NativeMethods.Vectors.DuckDBListVectorGetChild(vector) : NativeMethods.Vectors.DuckDBArrayVectorGetChild(vector);
22+
23+
arraySize = IsList ? 0 : (ulong)NativeMethods.LogicalType.DuckDBArrayVectorGetSize(logicalType);
24+
listItemWriter = VectorDataWriterFactory.CreateWriter(childVector, childType);
25+
}
26+
27+
internal override bool AppendCollection(ICollection value, int rowIndex)
28+
{
29+
var count = (ulong)value.Count;
30+
31+
ResizeVector(rowIndex % (int)DuckDBGlobalData.VectorSize, count);
32+
33+
_ = value switch
34+
{
35+
IEnumerable<bool> items => WriteItems(items),
36+
37+
IEnumerable<sbyte> items => WriteItems(items),
38+
IEnumerable<short> items => WriteItems(items),
39+
IEnumerable<int> items => WriteItems(items),
40+
IEnumerable<long> items => WriteItems(items),
41+
IEnumerable<byte> items => WriteItems(items),
42+
IEnumerable<ushort> items => WriteItems(items),
43+
IEnumerable<uint> items => WriteItems(items),
44+
IEnumerable<ulong> items => WriteItems(items),
45+
46+
IEnumerable<float> items => WriteItems(items),
47+
IEnumerable<double> items => WriteItems(items),
48+
49+
IEnumerable<decimal> items => WriteItems(items),
50+
IEnumerable<BigInteger> items => WriteItems(items),
51+
52+
IEnumerable<string> items => WriteItems(items),
53+
IEnumerable<Guid> items => WriteItems(items),
54+
IEnumerable<DateTime> items => WriteItems(items),
55+
IEnumerable<TimeSpan> items => WriteItems(items),
56+
IEnumerable<DuckDBDateOnly> items => WriteItems(items),
57+
IEnumerable<DuckDBTimeOnly> items => WriteItems(items),
58+
#if NET6_0_OR_GREATER
59+
IEnumerable<DateOnly> items => WriteItems(items),
60+
IEnumerable<TimeOnly> items => WriteItems(items),
61+
#endif
62+
IEnumerable<DateTimeOffset> items => WriteItems(items),
63+
64+
_ => WriteItems<object>((IEnumerable<object>)value)
65+
};
66+
67+
var duckDBListEntry = new DuckDBListEntry(offset, count);
68+
var result = !IsList || AppendValueInternal(duckDBListEntry, rowIndex);
69+
70+
offset += count;
71+
72+
return result;
73+
74+
int WriteItems<T>(IEnumerable<T> items)
75+
{
76+
if (IsList == false && count != arraySize)
77+
{
78+
throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}");
79+
}
80+
81+
var index = 0;
82+
83+
foreach (var item in items)
84+
{
85+
listItemWriter.AppendValue(item, (int)offset + (index++));
86+
}
87+
88+
return 0;
89+
}
90+
}
91+
92+
private void ResizeVector(int rowIndex, ulong count)
93+
{
94+
//If writing to a list column we need to make sure that enough space is allocated. Not needed for Arrays as DuckDB does it for us.
95+
if (!IsList || offset + count <= vectorReservedSize) return;
96+
97+
var factor = 2d;
98+
99+
if (rowIndex > DuckDBGlobalData.VectorSize * 0.25 && rowIndex < DuckDBGlobalData.VectorSize * 0.5)
100+
{
101+
factor = 1.75;
102+
}
103+
104+
if (rowIndex > DuckDBGlobalData.VectorSize * 0.5 && rowIndex < DuckDBGlobalData.VectorSize * 0.75)
105+
{
106+
factor = 1.5;
107+
}
108+
109+
if (rowIndex > DuckDBGlobalData.VectorSize * 0.75)
110+
{
111+
factor = 1.25;
112+
}
113+
114+
vectorReservedSize = (ulong)Math.Max(vectorReservedSize * factor, offset + count);
115+
var state = NativeMethods.Vectors.DuckDBListVectorReserve(Vector, vectorReservedSize);
116+
117+
if (!state.IsSuccess())
118+
{
119+
throw new DuckDBException($"Failed to reserve {vectorReservedSize} for the list vector");
120+
}
121+
122+
listItemWriter.FetchDataPointer();
123+
}
124+
}

‎DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using System;
2+
using System.Collections;
3+
using System.Collections.Generic;
24
using System.Numerics;
35
using DuckDB.NET.Native;
46

@@ -59,6 +61,7 @@ public unsafe void AppendValue<T>(T value, int rowIndex)
5961
TimeOnly val => AppendTimeOnly(val, rowIndex),
6062
#endif
6163
DateTimeOffset val => AppendDateTimeOffset(val, rowIndex),
64+
ICollection val => AppendCollection(val, rowIndex),
6265
_ => ThrowException<T>()
6366
};
6467
}
@@ -93,6 +96,8 @@ public unsafe void AppendValue<T>(T value, int rowIndex)
9396

9497
internal virtual bool AppendBigInteger(BigInteger value, int rowIndex) => ThrowException<BigInteger>();
9598

99+
internal virtual bool AppendCollection(ICollection value, int rowIndex) => ThrowException<bool>();
100+
96101
private bool ThrowException<T>()
97102
{
98103
throw new InvalidOperationException($"Cannot write {typeof(T).Name} to {columnType} column");
@@ -103,4 +108,9 @@ internal unsafe bool AppendValueInternal<T>(T value, int rowIndex) where T : unm
103108
((T*)vectorData)[rowIndex] = value;
104109
return true;
105110
}
111+
112+
internal void FetchDataPointer()
113+
{
114+
vectorData = NativeMethods.Vectors.DuckDBVectorGetData(Vector);
115+
}
106116
}

‎DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ public static unsafe VectorDataWriterBase CreateWriter(IntPtr vector, DuckDBLogi
2222
DuckDBType.Boolean => new BooleanVectorDataWriter(vector, dataPointer, columnType),
2323

2424
DuckDBType.Map => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"),
25-
DuckDBType.List => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"),
26-
DuckDBType.Array => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"),
25+
DuckDBType.List => new ListVectorDataWriter(vector, dataPointer, columnType, logicalType),
26+
DuckDBType.Array => new ListVectorDataWriter(vector, dataPointer, columnType, logicalType),
2727
DuckDBType.Blob => new StringVectorDataWriter(vector, dataPointer, columnType),
2828
DuckDBType.Varchar => new StringVectorDataWriter(vector, dataPointer, columnType),
2929
DuckDBType.Bit => throw new NotImplementedException($"Writing {columnType} to data chunk is not yet supported"),

0 commit comments

Comments
 (0)