Merge pull request #29 from AnswerDotAI/add-returning-to-inserts

Add returning to inserts

Author: jph00

README.md

@@ -22,6 +22,12 @@
 
 ## Use
 
+``` python
+from fastcore.utils import *
+from fastcore.test import *
+from typing import Any
+```
+
 First, import the sqlite-miniutils library. Through the use of the
 **all** attribute in our Python modules by using `import *` we only
 bring in the `Database`, `Queryable`, `Table`, `View` classes. There’s
@@ -107,7 +113,7 @@ users.insert(dict(name='Pigeon', age=3, pwd='keptsecret'))
 users.insert(dict(name='Eagle', age=7, pwd='s3cr3t'))
 ```
 
-    <Table Users (id, name, age, pwd)>
+    {'id': 5, 'name': 'Eagle', 'age': 7, 'pwd': 's3cr3t'}
 
 A simple unfiltered select can be executed using `rows` property on the
 table object.
@@ -116,7 +122,7 @@ table object.
 users.rows
 ```
 
-    <generator object Queryable.rows_where at 0x10849f6f0>
+    <generator object Queryable.rows_where>
 
 Let’s iterate over that generator to see the results:
 
@@ -169,3 +175,76 @@ except ValueError as e:
 ```
 
     Cannot use offset without limit
+
+## Transactions
+
+If you have any SQL calls outside an explicit transaction, they are
+committed instantly.
+
+To group 2 or more queries together into 1 transaction, wrap them in a
+BEGIN and COMMIT, executing ROLLBACK if an exception is caught:
+
+``` python
+users.get(1)
+```
+
+    {'id': 1, 'name': 'Raven', 'age': 8, 'pwd': 's3cret'}
+
+``` python
+db.begin()
+try:
+    users.delete([1])
+    db.execute('FNOOORD')
+    db.commit()
+except Exception as e:
+    print(e)
+    db.rollback()
+```
+
+    near "FNOOORD": syntax error
+
+Because the transaction was rolled back, the user was not deleted:
+
+``` python
+users.get(1)
+```
+
+    {'id': 1, 'name': 'Raven', 'age': 8, 'pwd': 's3cret'}
+
+Let’s do it again, but without the DB error, to check the transaction is
+successful:
+
+``` python
+db.begin()
+try:
+    users.delete([1])
+    db.commit()
+except Exception as e: db.rollback()
+```
+
+``` python
+try:
+    users.get(1)
+    print("Delete failed!")
+except: print("Delete succeeded!")
+```
+
+    Delete succeeded!
+
+## Returning
+
+sqlite-minutils is different from sqlite-utils in that write actions
+(`INSERT`, `UPDATE`, `UPSERT`) return back the record(s) they have
+affected without relying on `last_rowid`. It does this through the
+`RETURNING` SQL keyword.
+
+``` python
+user = users.insert(dict(name='Turkey', age=2, pwd='gravy'))
+user
+```
+
+    {'id': 8, 'name': 'Turkey', 'age': 2, 'pwd': 'gravy'}
+
+``` python
+test(user['name'], 'Turkey', equals)
+```

nbs/index.ipynb

@@ -1,5 +1,25 @@
 {
  "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "#| hide\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -50,8 +70,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#| hide\n",
     "from fastcore.utils import *\n",
+    "from fastcore.test import *\n",
     "from typing import Any"
    ]
   },
@@ -233,7 +253,7 @@
     {
      "data": {
       "text/plain": [
-       "<Table Users (id, name, age, pwd)>"
+       "{'id': 5, 'name': 'Eagle', 'age': 7, 'pwd': 's3cr3t'}"
       ]
      },
      "execution_count": null,
@@ -547,12 +567,49 @@
     "except: print(\"Delete succeeded!\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Returning"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "sqlite-minutils is different from sqlite-utils in that write actions (`INSERT`, `UPDATE`, `UPSERT`) return back the record(s) they have affected without relying on `last_rowid`. It does this through the `RETURNING` SQL keyword."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'id': 8, 'name': 'Turkey', 'age': 2, 'pwd': 'gravy'}"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "user = users.insert(dict(name='Turkey', age=2, pwd='gravy'))\n",
+    "user"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "test(user['name'], 'Turkey', equals)"
+   ]
   }
  ],
  "metadata": {

sqlite_minutils/db.py

@@ -2877,7 +2877,7 @@ def insert_chunk(
         num_records_processed,
         replace,
         ignore,
-    ):
+    ) -> List[Dict]:
         queries_and_params = self.build_insert_queries_and_params(
             extracts,
             chunk,
@@ -2893,68 +2893,24 @@ def insert_chunk(
             ignore,
         )
 
-        result = None
+        records = []
         for query, params in queries_and_params:
-            try:
-                result = self.db.execute(query, tuple(params))
-            except OperationalError as e:
-                if alter and (" column" in e.args[0]):
-                    # Attempt to add any missing columns, then try again
-                    self.add_missing_columns(chunk)
-                    result = self.db.execute(query, params)
-                elif e.args[0] == "too many SQL variables":
-                    first_half = chunk[: len(chunk) // 2]
-                    second_half = chunk[len(chunk) // 2 :]
-
-                    self.insert_chunk(
-                        alter,
-                        extracts,
-                        first_half,
-                        all_columns,
-                        hash_id,
-                        hash_id_columns,
-                        upsert,
-                        pk,
-                        not_null,
-                        conversions,
-                        num_records_processed,
-                        replace,
-                        ignore,
-                    )
-
-                    self.insert_chunk(
-                        alter,
-                        extracts,
-                        second_half,
-                        all_columns,
-                        hash_id,
-                        hash_id_columns,
-                        upsert,
-                        pk,
-                        not_null,
-                        conversions,
-                        num_records_processed,
-                        replace,
-                        ignore,
-                    )
+            cursor = self.db.execute(query, tuple(params))
+            columns = [c[0] for c in cursor.description]
+            record = dict(zip(columns, cursor.fetchone()))
+            records.append(record)
 
+            # Preserve old self.last_pk functionality
+            if (hash_id or pk) and not upsert:
+                if hash_id:
+                    self.last_pk = row[hash_id]
+                elif isinstance(pk, str):
+                    self.last_pk = row[pk]
                 else:
-                    raise
-            if num_records_processed == 1:
-                rid = self.db.get_last_rowid()
-                if rid is not None:
-                    self.last_pk = self.last_rowid = rid
-                    # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened
-                    if (hash_id or pk) and not upsert:
-                        row = list(self.rows_where("rowid = ?", [rid]))[0]
-                        if hash_id:
-                            self.last_pk = row[hash_id]
-                        elif isinstance(pk, str):
-                            self.last_pk = row[pk]
-                        else:
-                            self.last_pk = tuple(row[p] for p in pk)
-
-        return
+                    self.last_pk = tuple(row[p] for p in pk)
+            self.last_rowid = self.last_pk            
+
+        return records
 
     def insert(
         self,
@@ -2973,7 +2929,7 @@ def insert(
         conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT,
         columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT,
         strict: Optional[Union[bool, Default]] = DEFAULT,
-    ) -> "Table":
+    ) -> Dict:
         """
         Insert a single record into the table. The table will be created with a schema that matches
         the inserted record if it does not already exist, see :ref:`python_api_creating_tables`.
@@ -3023,7 +2979,7 @@ def insert(
             conversions=conversions,
             columns=columns,
             strict=strict,
-        )
+        )[0]
 
     def insert_all(
         self,
@@ -3046,7 +3002,7 @@ def insert_all(
         upsert=False,
         analyze=False,
         strict=DEFAULT,
-    ) -> "Table":
+    ) -> List[Dict]:
         """
         Like ``.insert()`` but takes a list of records and ensures that the table
         that it creates (if table does not exist) has columns for ALL of that data.
@@ -3105,6 +3061,7 @@ def insert_all(
         self.last_pk = None
         if truncate and self.exists():
             self.db.execute("DELETE FROM [{}];".format(self.name))
+        records = []
         for chunk in chunks(itertools.chain([first_record], records), batch_size):
             chunk = list(chunk)
             num_records_processed += len(chunk)
@@ -3139,7 +3096,7 @@ def insert_all(
 
             first = False
 
-            self.insert_chunk(
+            records.extend(self.insert_chunk(
                 alter,
                 extracts,
                 chunk,
@@ -3153,12 +3110,12 @@ def insert_all(
                 num_records_processed,
                 replace,
                 ignore,
-            )
+            ))
 
         if analyze:
             self.analyze()
 
-        return self
+        return records
 
     def upsert(
         self,