Fix crash related to invalid RowSlot or RowSlotChunk pointers

Details:
Fix two issues where a RowSlot* or RowSlotChunk* could point
to invalid addresses following a CursorWindow resize which moves
the entire allocation block, invalidating the previous address.
These scenarios could occur when a query required expanding the default
CursorWindow size and the allocation occured on an edge boundary.
This adjustment also allows us to remove the CURSOR_SIZE_EXTRA which
caused the test `testManyRowsLong` to identify the RowSlotChunk issue.
The test `shouldNotCauseRowSlotAllocationCrash` verified the fix
for the RowSlot error.

Author: developernotes

sqlcipher/src/androidTest/java/net/zetetic/database/sqlcipher_cts/AndroidSQLCipherTestCase.java

@@ -1,6 +1,7 @@
 package net.zetetic.database.sqlcipher_cts;
 
 import android.content.Context;
+import android.icu.text.NumberFormat;
 import android.util.Log;
 
 import androidx.test.ext.junit.runners.AndroidJUnit4;
@@ -19,6 +20,9 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
 
 @RunWith(AndroidJUnit4.class)
@@ -105,4 +109,205 @@ protected void log(String message, Object... args) {
   protected void loge(Exception ex, String message, Object... args) {
     Log.e(TAG, String.format(Locale.getDefault(), message, args), ex);
   }
+
+  public interface RowColumnValueBuilder {
+    Object buildRowColumnValue(String[] columns, int row, int column);
+  }
+
+  protected void buildDatabase(
+    SQLiteDatabase database,
+    int rows,
+    int columns,
+    RowColumnValueBuilder builder) {
+    var columnNames = new ArrayList<String>();
+    Log.i(TAG, String.format("Building database with %s rows, %d columns",
+      NumberFormat.getInstance().format(rows), columns));
+    var createTemplate = "CREATE TABLE t1(%s);";
+    var insertTemplate = "INSERT INTO t1 VALUES(%s);";
+    var createBuilder = new StringBuilder();
+    var insertBuilder = new StringBuilder();
+    for (int column = 0; column < columns; column++) {
+      var columnName = generateColumnName(columnNames, column);
+      createBuilder.append(String.format("%s BLOB%s",
+        columnName,
+        column != columns - 1 ? "," : ""));
+      insertBuilder.append(String.format("?%s", column != columns - 1 ? "," : ""));
+    }
+    var create = String.format(createTemplate, createBuilder.toString());
+    var insert = String.format(insertTemplate, insertBuilder.toString());
+    database.execSQL("DROP TABLE IF EXISTS t1;");
+    database.execSQL(create);
+    database.execSQL("BEGIN;");
+    var names = columnNames.toArray(new String[0]);
+    for (int row = 0; row < rows; row++) {
+      var insertArgs = new Object[columns];
+      for (var column = 0; column < columns; column++) {
+        insertArgs[column] = builder.buildRowColumnValue(names, row, column);
+      }
+      database.execSQL(insert, insertArgs);
+    }
+    database.execSQL("COMMIT;");
+    Log.i(TAG, String.format("Database built with %d columns, %d rows", columns, rows));
+  }
+
+  protected Integer[] generateRandomNumbers(
+    int max,
+    int times){
+    var random = new SecureRandom();
+    var numbers = new ArrayList<>();
+    for(var index = 0; index < times; index++){
+      boolean alreadyExists;
+      do {
+        var value = random.nextInt(max);
+        alreadyExists = numbers.contains(value);
+        if(!alreadyExists){
+          numbers.add(value);
+        }
+      } while(alreadyExists);
+    }
+    return numbers.toArray(new Integer[0]);
+  }
+
+  protected List<String> ReservedWords = Arrays.asList(
+    "ABORT",
+    "ACTION",
+    "ADD",
+    "AFTER",
+    "ALL",
+    "ALTER",
+    "ANALYZE",
+    "AND",
+    "AS",
+    "ASC",
+    "ATTACH",
+    "AUTOINCREMENT",
+    "BEFORE",
+    "BEGIN",
+    "BETWEEN",
+    "BY",
+    "CASCADE",
+    "CASE",
+    "CAST",
+    "CHECK",
+    "COLLATE",
+    "COLUMN",
+    "COMMIT",
+    "CONFLICT",
+    "CONSTRAINT",
+    "CREATE",
+    "CROSS",
+    "CURRENT_DATE",
+    "CURRENT_TIME",
+    "CURRENT_TIMESTAMP",
+    "DATABASE",
+    "DEFAULT",
+    "DEFERRABLE",
+    "DEFERRED",
+    "DELETE",
+    "DESC",
+    "DETACH",
+    "DISTINCT",
+    "DROP",
+    "EACH",
+    "ELSE",
+    "END",
+    "ESCAPE",
+    "EXCEPT",
+    "EXCLUSIVE",
+    "EXISTS",
+    "EXPLAIN",
+    "FAIL",
+    "FOR",
+    "FOREIGN",
+    "FROM",
+    "FULL",
+    "GLOB",
+    "GROUP",
+    "HAVING",
+    "IF",
+    "IGNORE",
+    "IMMEDIATE",
+    "IN",
+    "INDEX",
+    "INDEXED",
+    "INITIALLY",
+    "INNER",
+    "INSERT",
+    "INSTEAD",
+    "INTERSECT",
+    "INTO",
+    "IS",
+    "ISNULL",
+    "JOIN",
+    "KEY",
+    "LEFT",
+    "LIKE",
+    "LIMIT",
+    "MATCH",
+    "NATURAL",
+    "NO",
+    "NOT",
+    "NOTNULL",
+    "NULL",
+    "OF",
+    "OFFSET",
+    "ON",
+    "OR",
+    "ORDER",
+    "OUTER",
+    "PLAN",
+    "PRAGMA",
+    "PRIMARY",
+    "QUERY",
+    "RAISE",
+    "RECURSIVE",
+    "REFERENCES",
+    "REGEXP",
+    "REINDEX",
+    "RELEASE",
+    "RENAME",
+    "REPLACE",
+    "RESTRICT",
+    "RIGHT",
+    "ROLLBACK",
+    "ROW",
+    "SAVEPOINT",
+    "SELECT",
+    "SET",
+    "TABLE",
+    "TEMP",
+    "TEMPORARY",
+    "THEN",
+    "TO",
+    "TRANSACTION",
+    "TRIGGER",
+    "UNION",
+    "UNIQUE",
+    "UPDATE",
+    "USING",
+    "VACUUM",
+    "VALUES",
+    "VIEW",
+    "VIRTUAL",
+    "WHEN",
+    "WHERE",
+    "WITH",
+    "WITHOUT");
+
+  private String generateColumnName(
+    List<String> columnNames,
+    int columnIndex){
+    var random = new SecureRandom();
+    var labels = "abcdefghijklmnopqrstuvwxyz";
+    var element = columnIndex < labels.length()
+      ? String.valueOf(labels.charAt(columnIndex))
+      : String.valueOf(labels.charAt(random.nextInt(labels.length() - 1)));
+    while(columnNames.contains(element) || ReservedWords.contains(element.toUpperCase())){
+      element += labels.charAt(random.nextInt(labels.length() - 1));
+    }
+    columnNames.add(element);
+    Log.i(TAG, String.format("Generated column name:%s for index:%d", element, columnIndex));
+    return element;
+  }
+
 }

sqlcipher/src/androidTest/java/net/zetetic/database/sqlcipher_cts/SQLCipherDatabaseTest.java

@@ -18,9 +18,14 @@
 import org.junit.Test;
 
 import java.io.File;
+import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Random;
 import java.util.UUID;
 
 public class SQLCipherDatabaseTest extends AndroidSQLCipherTestCase {
@@ -571,4 +576,118 @@ public void shouldSupportDeleteWithNullWhereArgs(){
     }
     assertThat(rowsFound, is(0L));
   }
+
+
+  // This test recreated a scenario where the CursorWindow::allocRow
+  // would alloc a RowSlot*, then the alloc call to allocate the
+  // fieldDirOffset (based on fieldDirSize) would cause mData
+  // to move (when there was just enough space in mData for a RowSlot*,
+  // but not enough for the corresponding FieldSlot*), invalidating the
+  // previous rowSlot address. This has been addressed by reassigning the
+  // rowSlot pointer after alloc, prior to binding the fieldDirOffset
+  // and should not fail now.
+  /** @noinspection StatementWithEmptyBody*/
+  @Test
+  public void shouldNotCauseRowSlotAllocationCrash(){
+    SQLiteCursor.resetCursorWindowSize();
+    database.execSQL("create table t1(a INTEGER, b INTEGER, c TEXT, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, "+
+      "aa, bb, cc, dd, ee, ff, gg, hh, ii, jj, kk, ll, mm, nn, oo, pp, qq, rr, ss, tt, uu, vv);");
+    database.beginTransaction();
+    Random random = new Random();
+    for(int i = 0; i < 20; i++) {
+      int size = 1024*3 + 450;
+      database.execSQL("insert into t1(a, b, c) values(?, ?, randomblob(?));", new Object[]{i, size, size});
+    }
+    database.setTransactionSuccessful();
+    var value = false;
+    var cursor = database.rawQuery("select * from t1;");
+    if(cursor != null){
+      while(cursor.moveToNext()){}
+      value = true;
+    }
+    assertThat(value, is(true));
+  }
+
+  @Test
+  public void shouldBuildLargeDatabaseWithCustomCursorSizeAndNavigateValuesWithDigest() throws UnsupportedEncodingException, NoSuchAlgorithmException {
+    int rowCount = 1000;
+    int windowAllocationSize = 1024 * 1024 / 20;
+    buildDatabase(database, rowCount, 30, (columns, row, column) -> {
+			try {
+				var digest = MessageDigest.getInstance("SHA-1");
+				var columnName = columns[column];
+				var value = String.format("%s%d", columnName, row);
+				return digest.digest(value.getBytes(StandardCharsets.UTF_8));
+			} catch (Exception e) {
+				Log.e(TAG, e.toString());
+				return null;
+			}
+		});
+
+		var randomRows = generateRandomNumbers(rowCount, rowCount);
+    SQLiteCursor.setCursorWindowSize(windowAllocationSize);
+    var cursor = database.rawQuery("SELECT * FROM t1;", null);
+		var digest = MessageDigest.getInstance("SHA-1");
+    int row = 0;
+    Log.i(TAG, "Walking cursor forward");
+    while(cursor.moveToNext()){
+      var compare = compareDigestForAllColumns(cursor, digest, row);
+      assertThat(compare, is(true));
+      row++;
+    }
+    Log.i(TAG, "Walking cursor backward");
+    while(cursor.moveToPrevious()){
+      row--;
+      var compare = compareDigestForAllColumns(cursor, digest, row);
+      assertThat(compare, is(true));
+    }
+    Log.i(TAG, "Walking cursor randomly");
+    for(int randomRow : randomRows){
+      cursor.moveToPosition(randomRow);
+      var compare = compareDigestForAllColumns(cursor, digest, randomRow);
+      assertThat(compare, is(true));
+    }
+  }
+
+  @Test
+  public void shouldCheckAllTypesFromCursor(){
+    database.execSQL("drop table if exists t1;");
+    database.execSQL("create table t1(a text, b integer, c text, d real, e blob)");
+    byte[] data = new byte[10];
+    new SecureRandom().nextBytes(data);
+    database.execSQL("insert into t1(a, b, c, d, e) values(?, ?, ?, ?, ?)", new Object[]{"test1", 100, null, 3.25, data});
+    Cursor results = database.rawQuery("select * from t1", new String[]{});
+    results.moveToFirst();
+    int type_a = results.getType(0);
+    int type_b = results.getType(1);
+    int type_c = results.getType(2);
+    int type_d = results.getType(3);
+    int type_e = results.getType(4);
+    results.close();
+    assertThat(type_a, is(Cursor.FIELD_TYPE_STRING));
+    assertThat(type_b, is(Cursor.FIELD_TYPE_INTEGER));
+    assertThat(type_c, is(Cursor.FIELD_TYPE_NULL));
+    assertThat(type_d, is(Cursor.FIELD_TYPE_FLOAT));
+    assertThat(type_e, is(Cursor.FIELD_TYPE_BLOB));
+  }
+
+  private boolean compareDigestForAllColumns(
+    Cursor cursor,
+    MessageDigest digest,
+    int row) throws UnsupportedEncodingException {
+    var columnCount = cursor.getColumnCount();
+    for(var column = 0; column < columnCount; column++){
+      Log.i(TAG, String.format("Comparing SHA-1 digest for row:%d", row));
+      var columnName = cursor.getColumnName(column);
+      var actual = cursor.getBlob(column);
+      var value = String.format("%s%d", columnName, row);
+      var expected = digest.digest(value.getBytes(StandardCharsets.UTF_8));
+      if(!Arrays.equals(actual, expected)){
+        Log.e(TAG, String.format("SHA-1 digest mismatch for row:%d column:%d", row, column));
+        return false;
+      }
+    }
+    return true;
+  }
+
 }