Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/python-api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,8 @@ You can also specify a primary key by passing the ``pk=`` parameter to the ``.in
After inserting a row like this, the ``dogs.last_rowid`` property will return the SQLite ``rowid`` assigned to the most recently inserted record.

The ``dogs.last_pk`` property will return the last inserted primary key value, if you specified one. This can be very useful when writing code that creates foreign keys or many-to-many relationships.
On SQLite 3.35 or later a single insert into a ``WITHOUT ROWID`` table returns its primary key in ``last_pk`` and leaves ``last_rowid`` set to ``None``.
This key is the value inserted by the statement, before any ``AFTER`` trigger on the ``WITHOUT ROWID`` table runs.

.. _python_api_custom_columns:

Expand Down
87 changes: 83 additions & 4 deletions sqlite_utils/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -2250,6 +2250,13 @@ def schema(self) -> str:
).fetchone()[0]


@dataclass
class _InsertReturning:
columns: str
has_rowid: bool
row: tuple[Any, ...] | None = None


class Table(Queryable):
"""
Tables should usually be initialized using the ``db.table(table_name)`` or
Expand Down Expand Up @@ -4517,6 +4524,51 @@ def build_insert_queries_and_params(
self.last_pk = pk_values
return queries_and_params

def _single_insert_info(
self, pk: PrimaryKey | None
) -> tuple[bool, _InsertReturning | None]:
# The table-existence check also supplies the information needed to
# choose RETURNING, without a separate series of schema lookups.
schema, has_triggers, version = self.db.execute(
"select (select sql from sqlite_master where type = 'table' and name = :name), "
"exists(select 1 from sqlite_master where type = 'trigger' and tbl_name = :name collate nocase "
"union all select 1 from sqlite_temp_master where type = 'trigger' and tbl_name = :name collate nocase), "
"sqlite_version()",
{"name": self.name},
).fetchone()
exists = schema is not None
if tuple(map(int, version.split("."))) < (3, 35):
return exists, None

# index_info(table) gives the primary key columns for a WITHOUT ROWID
# table. Confirm with SQLite if its schema contains the keyword.
without_rowid_pk = []
if schema and "without" in schema.lower():
without_rowid_pk = [
row[2]
for row in self.db.execute(
f"PRAGMA index_info({quote_identifier(self.name)})"
)
]
if not without_rowid_pk and (
has_triggers or (schema and _virtual_table_using_re.match(schema))
):
# RETURNING precedes AFTER triggers and can report a placeholder
# rowid on virtual tables. Keep the post-insert lookup for these.
return exists, None
if pk:
columns = [pk] if isinstance(pk, str) else list(pk)
elif without_rowid_pk:
columns = without_rowid_pk
elif schema and "rowid" not in schema.lower():
columns = ["rowid"]
else:
# The schema could shadow rowid, or is still to be created.
return exists, None
return exists, _InsertReturning(
", ".join(map(quote_identifier, columns)), not without_rowid_pk
)

def insert_chunk(
self,
alter,
Expand All @@ -4533,6 +4585,7 @@ def insert_chunk(
replace,
ignore,
list_mode=False,
returning: _InsertReturning | None = None,
) -> sqlite3.Cursor | None:
queries_and_params = self.build_insert_queries_and_params(
extracts,
Expand All @@ -4552,6 +4605,8 @@ def insert_chunk(
result = None
with self.db.atomic():
for query, params in queries_and_params:
if returning is not None:
query += " RETURNING " + returning.columns
try:
result = self.db.execute(query, params)
except OperationalError as e:
Expand Down Expand Up @@ -4599,6 +4654,11 @@ def insert_chunk(

else:
raise
if returning is not None and result is not None:
# SQLite cannot commit with a RETURNING statement in progress.
row = result.fetchone()
if row is not None:
returning.row = tuple(row)
return result

def insert(
Expand Down Expand Up @@ -4824,11 +4884,16 @@ def insert_all(
with self.db.atomic():
self.db.execute(f"DELETE FROM {quote_identifier(self.name)};")
result = None
returning = None
for chunk in chunks(itertools.chain([first_record], records_iter), batch_size):
chunk = list(chunk)
num_records_processed += len(chunk)
if first:
if not self.exists():
if num_records_processed == 1 and not upsert:
exists, returning = self._single_insert_info(pk)
else:
exists = self.exists()
if not exists:
# Use the first batch to derive the table names
if list_mode:
# Convert list records to dicts for type detection
Expand Down Expand Up @@ -4886,7 +4951,6 @@ def insert_all(
]

first = False

result = self.insert_chunk(
alter,
extracts,
Expand All @@ -4902,11 +4966,11 @@ def insert_all(
replace,
ignore,
list_mode,
returning=returning if num_records_processed == 1 else None,
)

# If we only handled a single row populate self.last_pk
if num_records_processed == 1:
# For an insert we need to use result.lastrowid
if not upsert and result is not None:
ignored_insert = ignore and result.rowcount == 0
if ignored_insert:
Expand Down Expand Up @@ -4954,8 +5018,13 @@ def insert_all(
where = " and ".join(
f"{quote_identifier(c)} = ?" for c in key_cols
)
rowid = (
"NULL"
if returning and not returning.has_rowid
else "rowid"
)
existing = self.db.execute(
f"select rowid from {quote_identifier(self.name)} where {where} limit 1",
f"select {rowid} from {quote_identifier(self.name)} where {where} limit 1",
key_values,
).fetchone()
if existing is not None:
Expand All @@ -4968,6 +5037,16 @@ def insert_all(
if len(key_cols) == 1
else tuple(key_values)
)
elif returning is not None:
if returning.row is not None:
self.last_pk = (
returning.row[0]
if len(returning.row) == 1
else returning.row
)
self.last_rowid = (
result.lastrowid if returning.has_rowid else None
)
else:
self.last_rowid = result.lastrowid
# A rowid-alias pk resolves directly to the rowid, so there
Expand Down
33 changes: 33 additions & 0 deletions tests/test_create.py
Original file line number Diff line number Diff line change
Expand Up @@ -1010,6 +1010,39 @@ def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db):
assert fresh_db.table("t").count == 0


@pytest.mark.skipif(sqlite3.sqlite_version_info < (3, 35), reason="requires RETURNING")
@pytest.mark.parametrize("prior_insert", [False, True])
@pytest.mark.parametrize("compound", [False, True])
@pytest.mark.parametrize("specify_pk", [False, True])
def test_insert_returns_pk_without_rowid(fresh_db, prior_insert, compound, specify_pk):
if prior_insert:
fresh_db.table("other").insert({"value": 1})
pk = ('item"id', "locale") if compound else 'item"id'
key_sql = '"item""id", locale' if compound else '"item""id"'
fresh_db.execute(
f'create table items ("item""id" text, locale text, primary key ({key_sql})) without rowid'
)
values = {'item"id': "item", "locale": "en"}
table = fresh_db.table("items")
for ignore in (False, True):
table.insert(values, pk=pk if specify_pk else None, ignore=ignore)
assert table.last_pk == (("item", "en") if compound else "item")
assert table.last_rowid is None
assert list(table.rows) == [values]


@pytest.mark.parametrize("temporary", ["", "temp "])
def test_insert_reports_pk_after_trigger(fresh_db, temporary):
fresh_db.execute("create table items (key text primary key)")
fresh_db.execute(
f"create {temporary}trigger fix_key after insert on ITEMS begin "
"update items set key = upper(new.key) where rowid = new.rowid; end"
)
table = fresh_db.table("items").insert({"key": "example"}, pk="key")
assert table.last_pk == "EXAMPLE"
assert table.last_rowid == 1


def test_insert_ignore(fresh_db):
fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id")
# Should raise an error if we try this again
Expand Down
8 changes: 7 additions & 1 deletion tests/test_tracer.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,13 @@ def test_tracer():
assert collected == [
("PRAGMA recursive_triggers=on;", None),
("select name from sqlite_master where type = 'view'", None),
("select name from sqlite_master where type = 'table'", None),
(
"select (select sql from sqlite_master where type = 'table' and name = :name), "
"exists(select 1 from sqlite_master where type = 'trigger' and tbl_name = :name collate nocase "
"union all select 1 from sqlite_temp_master where type = 'trigger' and tbl_name = :name collate nocase), "
"sqlite_version()",
{"name": "dogs"},
),
("select name from sqlite_master where type = 'view'", None),
("select name from sqlite_master where type = 'view'", None),
("select name from sqlite_master where type = 'table'", None),
Expand Down
Loading