Skip to content

Commit 1940376

Browse files
developernotesdevelopernotes
committed
Allow for improved cursor window loading heuristic
The underlying SQLiteCursor implementation will request the CursorWindow to fill based on a computed heuristic which attempts to load 1/3 of the rows preceeding to the requested index. This shows a marked improvment in data loading. The new loading heuristic will be the default behavior. This can be disabled by invoking the following on the underlying SQLiteCursor reference: SQLiteCursor cursor = ... cursor.setFillWindowForwardOnly(true); The backing memory used to store the content of the CursorWindow can be modified from the default behavior setting the initial, growth padding, and maximum memory to be allocated during the mapping process into the CursorWindow. This can be accomplished via the following: long initial = 1024 * 1024; long growthPadding = 1024 * 1024 * 2; long max = 1024 * 1024 * 4; CursorWindowAllocation allocation = new CustomCursorWindowAllocation(initial, growthPadding, max); CursorWindow.setCursorWindowAllocation(allocation);
1 parent baa00a9 commit 1940376

11 files changed

+206
-90
lines changed

‎android-database-sqlcipher/src/main/cpp/CursorWindow.cpp‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,24 +26,22 @@
2626

2727
namespace sqlcipher {
2828

29-
CursorWindow::CursorWindow(size_t initialSize, size_t fixedAllocationSize)
29+
CursorWindow::CursorWindow(size_t initialSize, size_t growthPaddingSize, size_t maxSize)
3030
{
3131
mInitialSize = initialSize;
32-
mFixedAllocationSize = fixedAllocationSize;
33-
LOG_WINDOW("CursorWindow::CursorWindow initialSize:%d fixedAllocationSize:%d",
34-
initialSize, fixedAllocationSize);
32+
mGrowthPaddingSize = growthPaddingSize;
33+
mMaxSize = maxSize;
34+
LOG_WINDOW("CursorWindow::CursorWindow initialSize:%d growBySize:%d maxSize:%d",
35+
initialSize, growBySize, maxSize);
3536
}
3637

3738
bool CursorWindow::initBuffer(bool localOnly)
3839
{
39-
size_t size = mFixedAllocationSize == WINDOW_ALLOCATION_UNBOUNDED
40-
? mInitialSize
41-
: mFixedAllocationSize;
42-
void* data = malloc(size);
40+
void* data = malloc(mInitialSize);
4341
if(data){
4442
mData = (uint8_t *) data;
4543
mHeader = (window_header_t *) mData;
46-
mSize = size;
44+
mSize = mInitialSize;
4745
clear();
4846
LOG_WINDOW("Created CursorWindow with new MemoryDealer: mFreeOffset = %d, mSize = %d, mInitialSize = %d, mFixedAllocationSize = %d, mData = %p",
4947
mFreeOffset, mSize, mInitialSize, mFixedAllocationSize, mData);
@@ -126,8 +124,8 @@ uint32_t CursorWindow::alloc(size_t requestedSize, bool aligned)
126124
if (size > freeSpace()) {
127125
LOGE("need to grow: mSize = %d, size = %d, freeSpace() = %d, numRows = %d",
128126
mSize, size, freeSpace(), mHeader->numRows);
129-
if(mFixedAllocationSize == WINDOW_ALLOCATION_UNBOUNDED) {
130-
new_allocation_sz = mSize + size - freeSpace() + GROW_WINDOW_SIZE_EXTRA;
127+
new_allocation_sz = mSize + size - freeSpace() + mGrowthPaddingSize;
128+
if(mMaxSize == 0 || mMaxSize < new_allocation_sz) {
131129
tempData = realloc((void *)mData, new_allocation_sz);
132130
if(tempData == NULL) return 0;
133131
mData = (uint8_t *)tempData;

‎android-database-sqlcipher/src/main/cpp/CursorWindow.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ typedef struct
100100
class CursorWindow
101101
{
102102
public:
103-
CursorWindow(size_t initialSize, size_t fixedAllocationSize);
103+
CursorWindow(size_t initialSize, size_t growthPaddingSize, size_t maxSize);
104104
CursorWindow(){}
105105
~CursorWindow();
106106

@@ -186,7 +186,8 @@ class CursorWindow
186186
uint8_t * mData;
187187
size_t mSize;
188188
size_t mInitialSize;
189-
size_t mFixedAllocationSize;
189+
size_t mGrowthPaddingSize;
190+
size_t mMaxSize;
190191
window_header_t * mHeader;
191192
/**
192193
* Offset of the lowest unused data byte in the array.

‎android-database-sqlcipher/src/main/cpp/net_sqlcipher_CursorWindow.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,14 @@ namespace sqlcipher {
5151
}
5252

5353
static void native_init_empty(JNIEnv * env, jobject object,
54-
jboolean localOnly, jlong fixedAllocationSize)
54+
jboolean localOnly, jlong initialSize,
55+
jlong growthPaddingSize, jlong maxSize)
5556
{
5657
uint8_t * data;
5758
size_t size;
5859
CursorWindow * window;
5960

60-
window = new CursorWindow(INITIAL_WINDOW_SIZE, fixedAllocationSize);
61+
window = new CursorWindow(initialSize, growthPaddingSize, maxSize);
6162
if (!window) {
6263
jniThrowException(env, "java/lang/RuntimeException", "No memory for native window object");
6364
return;
@@ -617,7 +618,7 @@ namespace sqlcipher {
617618
static JNINativeMethod sMethods[] =
618619
{
619620
/* name, signature, funcPtr */
620-
{"native_init", "(ZJ)V", (void *)native_init_empty},
621+
{"native_init", "(ZJJJ)V", (void *)native_init_empty},
621622
// {"native_init", "(Landroid/os/IBinder;)V", (void *)native_init_memory},
622623
// {"native_getBinder", "()Landroid/os/IBinder;", (void *)native_getBinder},
623624
{"native_clear", "()V", (void *)native_clear},

‎android-database-sqlcipher/src/main/cpp/net_sqlcipher_database_SQLiteQuery.cpp‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ static int finish_program_and_get_row_count(sqlite3_stmt *statement) {
104104
}
105105

106106
static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
107-
jint startPos, jint offsetParam, jint maxRead, jint lastPos)
107+
jint startPos, jint requiredPos,
108+
jint offsetParam, jint maxRead, jint lastPos)
108109
{
109110
int err;
110111
sqlite3_stmt * statement = GET_STATEMENT(env, object);
@@ -178,6 +179,13 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
178179
// the field data is being allocated.
179180
{
180181
field_slot_t * fieldDir = window->allocRow();
182+
if(!fieldDir && (startPos + numRows) < requiredPos) {
183+
LOG_WINDOW("Failed to allocate row, resetting window", startPos + numRows);
184+
window->clear();
185+
window->setNumColumns(numColumns);
186+
fieldDir = window->allocRow();
187+
LOG_WINDOW("Window reset, row allocated at %p", fieldDir);
188+
}
181189
if (!fieldDir) {
182190
LOGE("Failed allocating fieldDir at startPos %d row %d", startPos, numRows);
183191
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
@@ -186,7 +194,35 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
186194

187195
// Pack the row into the window
188196
int i;
197+
bool failed = false;
198+
bool reset = false;
189199
for (i = 0; i < numColumns; i++) {
200+
201+
if(reset) {
202+
LOG_WINDOW("Reset requested for row %d, likely cursor window not large enough for current row\n",
203+
startPos + numRows);
204+
if(!failed && (startPos + numRows) < requiredPos) {
205+
LOG_WINDOW("Reseting window, previously unable to map required row %d into window\n",
206+
requiredPos);
207+
i = 0;
208+
window->clear();
209+
window->setNumColumns(numColumns);
210+
field_slot_t * fieldDir = window->allocRow();
211+
if(!fieldDir) {
212+
LOG_WINDOW("Failed to allocate row in reset, bailing\n");
213+
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
214+
} else {
215+
LOG_WINDOW("Allocated row in reset set\n");
216+
}
217+
} else {
218+
LOG_WINDOW("Bailing from reset, requested row %d already mapped in cursor window\n",
219+
startPos + numRows);
220+
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
221+
}
222+
failed = true;
223+
reset = false;
224+
}
225+
190226
int type = sqlite3_column_type(statement, i);
191227
if (type == SQLITE_TEXT) {
192228
// TEXT data
@@ -197,7 +233,8 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
197233
window->freeLastRow();
198234
LOGE("Failed allocating %u bytes for text/blob at %d,%d", size,
199235
startPos + numRows, i);
200-
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
236+
reset = true;
237+
continue;
201238
}
202239

203240
window->copyIn(offset, text, size);
@@ -216,7 +253,8 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
216253
if (!window->putLong(numRows, i, value)) {
217254
window->freeLastRow();
218255
LOGE("Failed allocating space for a long in column %d", i);
219-
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
256+
reset = true;
257+
continue;
220258
}
221259
LOG_WINDOW("%d,%d is INTEGER 0x%016llx", startPos + numRows, i, value);
222260
} else if (type == SQLITE_FLOAT) {
@@ -225,7 +263,8 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
225263
if (!window->putDouble(numRows, i, value)) {
226264
window->freeLastRow();
227265
LOGE("Failed allocating space for a double in column %d", i);
228-
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
266+
reset = true;
267+
continue;
229268
}
230269
LOG_WINDOW("%d,%d is FLOAT %lf", startPos + numRows, i, value);
231270
} else if (type == SQLITE_BLOB) {
@@ -237,11 +276,10 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
237276
window->freeLastRow();
238277
LOGE("Failed allocating %u bytes for blob at %d,%d", size,
239278
startPos + numRows, i);
240-
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
279+
reset = true;
280+
continue;
241281
}
242-
243282
window->copyIn(offset, blob, size);
244-
245283
// This must be updated after the call to alloc(), since that
246284
// may move the field around in the window
247285
field_slot_t * fieldSlot = window->getFieldSlot(numRows, i);
@@ -264,9 +302,9 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
264302
}
265303

266304
if (i < numColumns) {
267-
// Not all the fields fit in the window
268-
// Unknown data error happened
269-
break;
305+
// Not all the fields fit in the window
306+
// Unknown data error happened
307+
break;
270308
}
271309

272310
// Mark the row as complete in the window
@@ -326,7 +364,7 @@ static jstring native_column_name(JNIEnv* env, jobject object, jint columnIndex)
326364
static JNINativeMethod sMethods[] =
327365
{
328366
/* name, signature, funcPtr */
329-
{"native_fill_window", "(Lnet/sqlcipher/CursorWindow;IIII)I", (void *)native_fill_window},
367+
{"native_fill_window", "(Lnet/sqlcipher/CursorWindow;IIIII)I", (void *)native_fill_window},
330368
{"native_column_count", "()I", (void*)native_column_count},
331369
{"native_column_name", "(I)Ljava/lang/String;", (void *)native_column_name},
332370
};

‎android-database-sqlcipher/src/main/java/net/sqlcipher/CursorWindow.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public class CursorWindow extends android.database.CursorWindow implements Parce
4646
*/
4747
private long nWindow;
4848
private int mStartPos;
49+
private int mRequiredPos;
4950

5051
private static CursorWindowAllocation allocation = new DefaultCursorWindowAllocation();
5152

@@ -68,7 +69,10 @@ public CursorWindow(boolean localWindow) {
6869
if(allocation == null){
6970
allocation = new DefaultCursorWindowAllocation();
7071
}
71-
native_init(localWindow, allocation.getAllocationSize());
72+
native_init(localWindow,
73+
allocation.getInitialAllocationSize(),
74+
allocation.getGrowthPaddingSize(),
75+
allocation.getMaxAllocationSize());
7276
}
7377

7478
/**
@@ -88,8 +92,16 @@ public int getStartPosition() {
8892
*/
8993
public void setStartPosition(int pos) {
9094
mStartPos = pos;
91-
}
92-
95+
}
96+
97+
public int getRequiredPosition(){
98+
return mRequiredPos;
99+
}
100+
101+
public void setRequiredPosition(int pos) {
102+
mRequiredPos = pos;
103+
}
104+
93105
/**
94106
* Returns the number of rows in this window.
95107
*
@@ -635,7 +647,8 @@ public CursorWindow(Parcel source,int foo) {
635647
private native IBinder native_getBinder();
636648

637649
/** Does the native side initialization for an empty window */
638-
private native void native_init(boolean localOnly, long fixedAllocationSize);
650+
private native void native_init(boolean localOnly, long initialSize,
651+
long growthPaddingSize, long maxSize);
639652

640653
/** Does the native side initialization with an existing binder from another process */
641654
private native void native_init(IBinder nativeBinder);

‎android-database-sqlcipher/src/main/java/net/sqlcipher/CursorWindowAllocation.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package net.sqlcipher;
22

33
public interface CursorWindowAllocation {
4-
long getAllocationSize();
4+
long getInitialAllocationSize();
5+
long getGrowthPaddingSize();
6+
long getMaxAllocationSize();
57
}

‎android-database-sqlcipher/src/main/java/net/sqlcipher/CustomCursorWindowAllocation.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package net.sqlcipher;
2+
3+
import net.sqlcipher.CursorWindowAllocation;
4+
5+
public class CustomCursorWindowAllocation implements CursorWindowAllocation {
6+
7+
private long initialAllocationSize = 0L;
8+
private long growthPaddingSize = 0L;
9+
private long maxAllocationSize = 0L;
10+
11+
public CustomCursorWindowAllocation(long initialSize,
12+
long growthPaddingSize,
13+
long maxAllocationSize){
14+
this.initialAllocationSize = initialSize;
15+
this.growthPaddingSize = growthPaddingSize;
16+
this.maxAllocationSize = maxAllocationSize;
17+
}
18+
19+
public long getInitialAllocationSize() {
20+
return initialAllocationSize;
21+
}
22+
23+
public long getGrowthPaddingSize() {
24+
return initialAllocationSize;
25+
}
26+
27+
public long getMaxAllocationSize() {
28+
return maxAllocationSize;
29+
}
30+
}

‎android-database-sqlcipher/src/main/java/net/sqlcipher/DefaultCursorWindowAllocation.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,19 @@
33
import net.sqlcipher.CursorWindowAllocation;
44

55
public class DefaultCursorWindowAllocation implements CursorWindowAllocation {
6-
public long getAllocationSize(){
7-
return 0;
6+
7+
private long initialAllocationSize = 1024 * 1024;
8+
private long WindowAllocationUnbounded = 0;
9+
10+
public long getInitialAllocationSize() {
11+
return initialAllocationSize;
12+
}
13+
14+
public long getGrowthPaddingSize() {
15+
return initialAllocationSize;
16+
}
17+
18+
public long getMaxAllocationSize() {
19+
return WindowAllocationUnbounded;
820
}
921
}

‎android-database-sqlcipher/src/main/java/net/sqlcipher/FixedCursorWindowAllocation.java‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

0 commit comments

Comments
 (0)