From c51e08323163d7687d2878937a2ca7caf6c71794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 29 Jan 2026 14:24:00 +0000 Subject: [PATCH 1/4] Improvements in SQLiteProvider --- lib/storage/providers/SQLiteProvider.ts | 62 ++++++++++++++++++++----- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 1d2927d8c..a300c9f4b 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -107,13 +107,53 @@ const provider: StorageProvider = { throw new Error('Store is not initialized!'); } - const placeholders = keys.map(() => '?').join(','); - const command = `SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key IN (${placeholders});`; - return provider.store.executeAsync(command, keys).then(({rows}) => { - // eslint-disable-next-line no-underscore-dangle - const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); - return (result ?? []) as StorageKeyValuePair[]; - }); + if (keys.length <= 500) { + const placeholders = keys.map(() => '?').join(','); + const command = `SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key IN (${placeholders});`; + return provider.store.executeAsync(command, keys).then(({rows}) => { + // eslint-disable-next-line no-underscore-dangle + const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); + return result ?? []; + }); + } + + // FIXME: Not working because of "-" apparently. + // const tableName = `temp_multiMerge_${Str.guid()}`; + const tableName = `temp_multiMerge_${Date.now()}_${Math.random().toString(36).slice(2, 8)}}`; + + return provider.store + .executeAsync(`CREATE TEMP TABLE ${tableName} (record_key TEXT PRIMARY KEY);`) + .then(() => { + if (!provider.store) { + throw new Error('Store is not initialized!'); + } + + const insertQuery = `INSERT INTO ${tableName} (record_key) VALUES (?);`; + const insertParams = keys.map((key) => [key]); + return provider.store.executeBatchAsync([{query: insertQuery, params: insertParams}]); + }) + .then(() => { + if (!provider.store) { + throw new Error('Store is not initialized!'); + } + + return provider.store.executeAsync( + `SELECT k.record_key, k.valueJSON + FROM keyvaluepairs AS k + INNER JOIN ${tableName} AS t ON k.record_key = t.record_key;`, + ); + }) + .then(({rows}) => { + // eslint-disable-next-line no-underscore-dangle + const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); + + return result ?? []; + }) + .catch((error) => { + console.error('[SQLiteProvider] Error in multiGet:', error); + provider.store?.executeAsync(`DROP TABLE IF EXISTS ${tableName};`); + throw error; + }); }, setItem(key, value) { if (!provider.store) { @@ -127,7 +167,7 @@ const provider: StorageProvider = { throw new Error('Store is not initialized!'); } - const query = 'REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));'; + const query = 'REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, ?);'; const params = pairs.map((pair) => [pair[0], JSON.stringify(pair[1] === undefined ? null : pair[1])]); if (utils.isEmptyObject(params)) { return Promise.resolve(); @@ -143,15 +183,15 @@ const provider: StorageProvider = { // Query to merge the change into the DB value. const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON) - VALUES (:key, JSON(:value)) + VALUES (:key, :value) ON CONFLICT DO UPDATE - SET valueJSON = JSON_PATCH(valueJSON, JSON(:value)); + SET valueJSON = JSON_PATCH(valueJSON, :value); `; const patchQueryArguments: string[][] = []; // Query to fully replace the nested objects of the DB value. const replaceQuery = `UPDATE keyvaluepairs - SET valueJSON = JSON_REPLACE(valueJSON, ?, JSON(?)) + SET valueJSON = JSON_REPLACE(valueJSON, ?, ?) WHERE record_key = ?; `; const replaceQueryArguments: string[][] = []; From 85bf84d8ebfd3c7522a3dc0ea791032921f3cf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Fri, 6 Feb 2026 15:38:32 +0000 Subject: [PATCH 2/4] Minor fix in table name string --- lib/storage/providers/SQLiteProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index a300c9f4b..c3c8fe199 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -118,8 +118,8 @@ const provider: StorageProvider = { } // FIXME: Not working because of "-" apparently. - // const tableName = `temp_multiMerge_${Str.guid()}`; - const tableName = `temp_multiMerge_${Date.now()}_${Math.random().toString(36).slice(2, 8)}}`; + // const tableName = `temp_multiGet_${Str.guid()}`; + const tableName = `temp_multiGet_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; return provider.store .executeAsync(`CREATE TEMP TABLE ${tableName} (record_key TEXT PRIMARY KEY);`) From 774496650082f7257d5a0d920497cd2c784a089b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Fri, 6 Feb 2026 15:42:31 +0000 Subject: [PATCH 3/4] Add comments --- lib/storage/providers/SQLiteProvider.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index c3c8fe199..9ce7c5a85 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -107,6 +107,7 @@ const provider: StorageProvider = { throw new Error('Store is not initialized!'); } + // For a small number of keys, it's more efficient to just use a single query with multiple placeholders. if (keys.length <= 500) { const placeholders = keys.map(() => '?').join(','); const command = `SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key IN (${placeholders});`; @@ -117,10 +118,10 @@ const provider: StorageProvider = { }); } + // For a large number of keys, it becomes more performant if we insert the keys into a temporary table and perform a join. // FIXME: Not working because of "-" apparently. // const tableName = `temp_multiGet_${Str.guid()}`; const tableName = `temp_multiGet_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - return provider.store .executeAsync(`CREATE TEMP TABLE ${tableName} (record_key TEXT PRIMARY KEY);`) .then(() => { @@ -139,8 +140,8 @@ const provider: StorageProvider = { return provider.store.executeAsync( `SELECT k.record_key, k.valueJSON - FROM keyvaluepairs AS k - INNER JOIN ${tableName} AS t ON k.record_key = t.record_key;`, + FROM keyvaluepairs AS k + INNER JOIN ${tableName} AS t ON k.record_key = t.record_key;`, ); }) .then(({rows}) => { @@ -150,7 +151,6 @@ const provider: StorageProvider = { return result ?? []; }) .catch((error) => { - console.error('[SQLiteProvider] Error in multiGet:', error); provider.store?.executeAsync(`DROP TABLE IF EXISTS ${tableName};`); throw error; }); From 69296836d0a7695bdf58dbbc749ddc97e8298b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Fri, 6 Feb 2026 15:55:15 +0000 Subject: [PATCH 4/4] Always drop the temporary table --- lib/storage/providers/SQLiteProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 9ce7c5a85..198de19c4 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -150,9 +150,9 @@ const provider: StorageProvider = { return result ?? []; }) - .catch((error) => { + .finally(() => { + // Drops the temporary table to free up resources. provider.store?.executeAsync(`DROP TABLE IF EXISTS ${tableName};`); - throw error; }); }, setItem(key, value) {