diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 1d2927d8..198de19c 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -107,13 +107,53 @@ const provider: StorageProvider = { throw new Error('Store is not initialized!'); } - const placeholders = keys.map(() => '?').join(','); - const command = `SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key IN (${placeholders});`; - return provider.store.executeAsync(command, keys).then(({rows}) => { - // eslint-disable-next-line no-underscore-dangle - const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); - return (result ?? []) as StorageKeyValuePair[]; - }); + // For a small number of keys, it's more efficient to just use a single query with multiple placeholders. + if (keys.length <= 500) { + const placeholders = keys.map(() => '?').join(','); + const command = `SELECT record_key, valueJSON FROM keyvaluepairs WHERE record_key IN (${placeholders});`; + return provider.store.executeAsync(command, keys).then(({rows}) => { + // eslint-disable-next-line no-underscore-dangle + const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); + return result ?? []; + }); + } + + // For a large number of keys, it becomes more performant if we insert the keys into a temporary table and perform a join. + // FIXME: Not working because of "-" apparently. + // const tableName = `temp_multiGet_${Str.guid()}`; + const tableName = `temp_multiGet_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + return provider.store + .executeAsync(`CREATE TEMP TABLE ${tableName} (record_key TEXT PRIMARY KEY);`) + .then(() => { + if (!provider.store) { + throw new Error('Store is not initialized!'); + } + + const insertQuery = `INSERT INTO ${tableName} (record_key) VALUES (?);`; + const insertParams = keys.map((key) => [key]); + return provider.store.executeBatchAsync([{query: insertQuery, params: insertParams}]); + }) + .then(() => { + if (!provider.store) { + throw new Error('Store is not initialized!'); + } + + return provider.store.executeAsync( + `SELECT k.record_key, k.valueJSON + FROM keyvaluepairs AS k + INNER JOIN ${tableName} AS t ON k.record_key = t.record_key;`, + ); + }) + .then(({rows}) => { + // eslint-disable-next-line no-underscore-dangle + const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); + + return result ?? []; + }) + .finally(() => { + // Drops the temporary table to free up resources. + provider.store?.executeAsync(`DROP TABLE IF EXISTS ${tableName};`); + }); }, setItem(key, value) { if (!provider.store) { @@ -127,7 +167,7 @@ const provider: StorageProvider = { throw new Error('Store is not initialized!'); } - const query = 'REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));'; + const query = 'REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, ?);'; const params = pairs.map((pair) => [pair[0], JSON.stringify(pair[1] === undefined ? null : pair[1])]); if (utils.isEmptyObject(params)) { return Promise.resolve(); @@ -143,15 +183,15 @@ const provider: StorageProvider = { // Query to merge the change into the DB value. const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON) - VALUES (:key, JSON(:value)) + VALUES (:key, :value) ON CONFLICT DO UPDATE - SET valueJSON = JSON_PATCH(valueJSON, JSON(:value)); + SET valueJSON = JSON_PATCH(valueJSON, :value); `; const patchQueryArguments: string[][] = []; // Query to fully replace the nested objects of the DB value. const replaceQuery = `UPDATE keyvaluepairs - SET valueJSON = JSON_REPLACE(valueJSON, ?, JSON(?)) + SET valueJSON = JSON_REPLACE(valueJSON, ?, ?) WHERE record_key = ?; `; const replaceQueryArguments: string[][] = [];