From 5ecb0edec5c00e03d9193d5e56858b381229e8e1 Mon Sep 17 00:00:00 2001 From: businessdeveloper69 Date: Sat, 21 Nov 2026 00:08:25 +0530 Subject: [PATCH] Use RETURNING to report single-insert primary keys --- docs/python-api.rst | 2 + sqlite_utils/db.py | 87 ++++++++++++++++++++++++++++++++++++++++++-- tests/test_create.py | 33 +++++++++++++++++ tests/test_tracer.py | 8 +++- 4 files changed, 125 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index d515642a5..6fc540eb2 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -725,6 +725,8 @@ You can also specify a primary key by passing the ``pk=`` parameter to the ``.in After inserting a row like this, the ``dogs.last_rowid`` property will return the SQLite ``rowid`` assigned to the most recently inserted record. The ``dogs.last_pk`` property will return the last inserted primary key value, if you specified one. This can be very useful when writing code that creates foreign keys or many-to-many relationships. +On SQLite 3.35 or later a single insert into a ``WITHOUT ROWID`` table returns its primary key in ``last_pk`` and leaves ``last_rowid`` set to ``None``. +This key is the value inserted by the statement, before any ``AFTER`` trigger on the ``WITHOUT ROWID`` table runs. .. _python_api_custom_columns: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c011d9bb0..7868e6a36 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2250,6 +2250,13 @@ def schema(self) -> str: ).fetchone()[0] +@dataclass +class _InsertReturning: + columns: str + has_rowid: bool + row: tuple[Any, ...] | None = None + + class Table(Queryable): """ Tables should usually be initialized using the ``db.table(table_name)`` or @@ -4517,6 +4524,51 @@ def build_insert_queries_and_params( self.last_pk = pk_values return queries_and_params + def _single_insert_info( + self, pk: PrimaryKey | None + ) -> tuple[bool, _InsertReturning | None]: + # The table-existence check also supplies the information needed to + # choose RETURNING, without a separate series of schema lookups. + schema, has_triggers, version = self.db.execute( + "select (select sql from sqlite_master where type = 'table' and name = :name), " + "exists(select 1 from sqlite_master where type = 'trigger' and tbl_name = :name collate nocase " + "union all select 1 from sqlite_temp_master where type = 'trigger' and tbl_name = :name collate nocase), " + "sqlite_version()", + {"name": self.name}, + ).fetchone() + exists = schema is not None + if tuple(map(int, version.split("."))) < (3, 35): + return exists, None + + # index_info(table) gives the primary key columns for a WITHOUT ROWID + # table. Confirm with SQLite if its schema contains the keyword. + without_rowid_pk = [] + if schema and "without" in schema.lower(): + without_rowid_pk = [ + row[2] + for row in self.db.execute( + f"PRAGMA index_info({quote_identifier(self.name)})" + ) + ] + if not without_rowid_pk and ( + has_triggers or (schema and _virtual_table_using_re.match(schema)) + ): + # RETURNING precedes AFTER triggers and can report a placeholder + # rowid on virtual tables. Keep the post-insert lookup for these. + return exists, None + if pk: + columns = [pk] if isinstance(pk, str) else list(pk) + elif without_rowid_pk: + columns = without_rowid_pk + elif schema and "rowid" not in schema.lower(): + columns = ["rowid"] + else: + # The schema could shadow rowid, or is still to be created. + return exists, None + return exists, _InsertReturning( + ", ".join(map(quote_identifier, columns)), not without_rowid_pk + ) + def insert_chunk( self, alter, @@ -4533,6 +4585,7 @@ def insert_chunk( replace, ignore, list_mode=False, + returning: _InsertReturning | None = None, ) -> sqlite3.Cursor | None: queries_and_params = self.build_insert_queries_and_params( extracts, @@ -4552,6 +4605,8 @@ def insert_chunk( result = None with self.db.atomic(): for query, params in queries_and_params: + if returning is not None: + query += " RETURNING " + returning.columns try: result = self.db.execute(query, params) except OperationalError as e: @@ -4599,6 +4654,11 @@ def insert_chunk( else: raise + if returning is not None and result is not None: + # SQLite cannot commit with a RETURNING statement in progress. + row = result.fetchone() + if row is not None: + returning.row = tuple(row) return result def insert( @@ -4824,11 +4884,16 @@ def insert_all( with self.db.atomic(): self.db.execute(f"DELETE FROM {quote_identifier(self.name)};") result = None + returning = None for chunk in chunks(itertools.chain([first_record], records_iter), batch_size): chunk = list(chunk) num_records_processed += len(chunk) if first: - if not self.exists(): + if num_records_processed == 1 and not upsert: + exists, returning = self._single_insert_info(pk) + else: + exists = self.exists() + if not exists: # Use the first batch to derive the table names if list_mode: # Convert list records to dicts for type detection @@ -4886,7 +4951,6 @@ def insert_all( ] first = False - result = self.insert_chunk( alter, extracts, @@ -4902,11 +4966,11 @@ def insert_all( replace, ignore, list_mode, + returning=returning if num_records_processed == 1 else None, ) # If we only handled a single row populate self.last_pk if num_records_processed == 1: - # For an insert we need to use result.lastrowid if not upsert and result is not None: ignored_insert = ignore and result.rowcount == 0 if ignored_insert: @@ -4954,8 +5018,13 @@ def insert_all( where = " and ".join( f"{quote_identifier(c)} = ?" for c in key_cols ) + rowid = ( + "NULL" + if returning and not returning.has_rowid + else "rowid" + ) existing = self.db.execute( - f"select rowid from {quote_identifier(self.name)} where {where} limit 1", + f"select {rowid} from {quote_identifier(self.name)} where {where} limit 1", key_values, ).fetchone() if existing is not None: @@ -4968,6 +5037,16 @@ def insert_all( if len(key_cols) == 1 else tuple(key_values) ) + elif returning is not None: + if returning.row is not None: + self.last_pk = ( + returning.row[0] + if len(returning.row) == 1 + else returning.row + ) + self.last_rowid = ( + result.lastrowid if returning.has_rowid else None + ) else: self.last_rowid = result.lastrowid # A rowid-alias pk resolves directly to the rowid, so there diff --git a/tests/test_create.py b/tests/test_create.py index b738df952..997cc2542 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1010,6 +1010,39 @@ def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): assert fresh_db.table("t").count == 0 +@pytest.mark.skipif(sqlite3.sqlite_version_info < (3, 35), reason="requires RETURNING") +@pytest.mark.parametrize("prior_insert", [False, True]) +@pytest.mark.parametrize("compound", [False, True]) +@pytest.mark.parametrize("specify_pk", [False, True]) +def test_insert_returns_pk_without_rowid(fresh_db, prior_insert, compound, specify_pk): + if prior_insert: + fresh_db.table("other").insert({"value": 1}) + pk = ('item"id', "locale") if compound else 'item"id' + key_sql = '"item""id", locale' if compound else '"item""id"' + fresh_db.execute( + f'create table items ("item""id" text, locale text, primary key ({key_sql})) without rowid' + ) + values = {'item"id': "item", "locale": "en"} + table = fresh_db.table("items") + for ignore in (False, True): + table.insert(values, pk=pk if specify_pk else None, ignore=ignore) + assert table.last_pk == (("item", "en") if compound else "item") + assert table.last_rowid is None + assert list(table.rows) == [values] + + +@pytest.mark.parametrize("temporary", ["", "temp "]) +def test_insert_reports_pk_after_trigger(fresh_db, temporary): + fresh_db.execute("create table items (key text primary key)") + fresh_db.execute( + f"create {temporary}trigger fix_key after insert on ITEMS begin " + "update items set key = upper(new.key) where rowid = new.rowid; end" + ) + table = fresh_db.table("items").insert({"key": "example"}, pk="key") + assert table.last_pk == "EXAMPLE" + assert table.last_rowid == 1 + + def test_insert_ignore(fresh_db): fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again diff --git a/tests/test_tracer.py b/tests/test_tracer.py index ec81f2f05..3ca21e83c 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -13,7 +13,13 @@ def test_tracer(): assert collected == [ ("PRAGMA recursive_triggers=on;", None), ("select name from sqlite_master where type = 'view'", None), - ("select name from sqlite_master where type = 'table'", None), + ( + "select (select sql from sqlite_master where type = 'table' and name = :name), " + "exists(select 1 from sqlite_master where type = 'trigger' and tbl_name = :name collate nocase " + "union all select 1 from sqlite_temp_master where type = 'trigger' and tbl_name = :name collate nocase), " + "sqlite_version()", + {"name": "dogs"}, + ), ("select name from sqlite_master where type = 'view'", None), ("select name from sqlite_master where type = 'view'", None), ("select name from sqlite_master where type = 'table'", None),