From 0d6a1625888cda067d9883d499c83faf9524bce6 Mon Sep 17 00:00:00 2001 From: Mo0nl19ht <81469045+Mo0nl19ht@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:24:19 +0900 Subject: [PATCH] Copy the original schema in table.duplicate() table.duplicate() and "sqlite-utils duplicate" used CREATE TABLE ... AS SELECT, so the copy lost the primary key, NOT NULL, UNIQUE, CHECK and DEFAULT constraints, foreign keys, STRICT, WITHOUT ROWID, indexes and rowid values, and INTEGER columns became INT. The copy is now created from the original CREATE TABLE statement with only the table name changed, and rows are copied along with their rowids. Indexes are recreated under new names, since index names must be unique. Self-referencing foreign keys and table.column references point at the copy, and the AUTOINCREMENT counter is carried over. Triggers are not copied. Virtual tables are still copied with CREATE TABLE ... AS SELECT. This is a behavior change: the copy now enforces the original table's constraints. Refs #956 --- docs/cli.rst | 2 + docs/python-api.rst | 11 ++ sqlite_utils/create_table_parser.py | 84 +++++++++++ sqlite_utils/db.py | 71 +++++++++- tests/test_cli.py | 16 +++ tests/test_duplicate.py | 210 +++++++++++++++++++++++++++- 6 files changed, 389 insertions(+), 5 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 78c33b8ae..16a3e5fea 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2201,6 +2201,8 @@ The ``duplicate`` command duplicates a table - creating a new table with the sam sqlite-utils duplicate books.db authors authors_copy +The copy keeps the original table's column types, constraints, foreign keys, ``STRICT`` and ``WITHOUT ROWID`` options, ``rowid`` values and indexes (recreated under new names). Triggers are not copied. See :ref:`table.duplicate() ` for details. + .. note:: In Python: :ref:`table.duplicate() ` CLI reference: :ref:`sqlite-utils duplicate ` diff --git a/docs/python-api.rst b/docs/python-api.rst index d515642a5..b83855478 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1060,6 +1060,17 @@ The ``table.duplicate()`` method creates a copy of the table, copying both the t The new ``authors_copy`` table will now contain a duplicate copy of the data from ``authors``. +The new table is created from the original table's ``CREATE TABLE`` statement with only the table name changed, so it keeps the same column types, primary key, ``NOT NULL``, ``UNIQUE``, ``CHECK`` and ``DEFAULT`` constraints, foreign keys, ``STRICT`` and ``WITHOUT ROWID`` options. Those constraints are enforced on the copy. Row ``rowid`` values and the ``AUTOINCREMENT`` counter are preserved too. + +A few parts of the schema are adjusted for the new table name: + +- References to the table itself, such as a self-referencing foreign key or an ``authors.name`` column reference in a ``CHECK`` constraint, are updated to point at the copy. +- Indexes are recreated under new names, because index names must be unique within a database. If the index name contains the original table name that part is replaced, so ``idx_authors_name`` becomes ``idx_authors_copy_name``. Other index names are prefixed with the new table name, so ``by_name`` becomes ``authors_copy_by_name``. A numeric suffix is added if that name is already in use. + +Triggers are not copied. + +Duplicating a virtual table, such as a full-text search table, creates a regular table containing a copy of its rows. + This method raises ``sqlite_utils.db.NoTable`` if the table does not exist. .. note:: diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 7377f4fd6..f6de0fcdc 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -785,6 +785,90 @@ def parse_uniques(create_sql: str) -> list[Unique]: return uniques +def parse_without_rowid(create_sql: str) -> bool: + """Return True if a CREATE TABLE statement ends with WITHOUT ROWID.""" + if _table_body(create_sql) is None: + return False + tokens = _meaningful(_lex(create_sql)) + close = max(i for i, token in enumerate(tokens) if token.text == ")") + return any(token.is_keyword("WITHOUT") for token in tokens[close + 1 :]) + + +def rename_create_statement( + create_sql: str, new_name: str, table: str | None = None +) -> str: + """ + Rename the table or index created by a CREATE TABLE or CREATE INDEX statement. + + For CREATE INDEX, ``table`` replaces the name of the indexed table. Everything + else in the statement, including comments and whitespace, is left unchanged. + """ + tokens = _meaningful(_lex(create_sql)) + if not tokens or not tokens[0].is_keyword("CREATE"): + raise ParseError("Expected CREATE TABLE or CREATE INDEX") + index = 1 + while index < len(tokens) and tokens[index].text.upper() in ( + "TEMP", + "TEMPORARY", + "UNIQUE", + ): + index += 1 + if index >= len(tokens) or not ( + tokens[index].is_keyword("TABLE") or tokens[index].is_keyword("INDEX") + ): + raise ParseError("Expected CREATE TABLE or CREATE INDEX") + index += 1 + if ( + index + 2 < len(tokens) + and tokens[index].is_keyword("IF") + and tokens[index + 1].is_keyword("NOT") + and tokens[index + 2].is_keyword("EXISTS") + ): + index += 3 + if index + 1 < len(tokens) and tokens[index + 1].text == ".": + index += 2 + if index >= len(tokens): + raise ParseError("CREATE statement is missing its name") + replacements = [(tokens[index], new_name)] + if table is not None: + if index + 2 >= len(tokens) or not tokens[index + 1].is_keyword("ON"): + raise ParseError("CREATE INDEX is missing ON") + replacements.append((tokens[index + 2], table)) + for token, replacement in reversed(replacements): + create_sql = ( + create_sql[: token.start] + + _quote_replacement(token.text, replacement) + + create_sql[token.end :] + ) + return create_sql + + +def rewrite_table_references(sql: str, old_table: str, new_table: str) -> str: + """ + Rename ``old_table`` where it qualifies a column (``table.column``) or is the + target of ``REFERENCES``. Strings and comments are left unchanged. + """ + tokens = _meaningful(_lex(sql)) + folded = _ascii_fold(old_table) + replacements = [ + token + for i, token in enumerate(tokens) + if token.kind in ("word", "identifier") + and _ascii_fold(_unquote(token.text)) == folded + and ( + (i + 1 < len(tokens) and tokens[i + 1].text == ".") + or (i > 0 and tokens[i - 1].is_keyword("REFERENCES")) + ) + ] + for token in reversed(replacements): + sql = ( + sql[: token.start] + + _quote_replacement(token.text, new_table) + + sql[token.end :] + ) + return sql + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c011d9bb0..744ca3265 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -37,7 +37,10 @@ parse_checks, parse_column_comments, parse_uniques, + parse_without_rowid, + rename_create_statement, rewrite_check_expression, + rewrite_table_references, sql_ends_in_line_comment, ) from .utils import ( @@ -2629,15 +2632,77 @@ def create( def duplicate(self, new_name: str) -> "Table": """ - Create a duplicate of this table, copying across the schema and all row data. + Create a duplicate of this table, copying across the schema, indexes, rowids + and all row data. Triggers are not copied. :param new_name: Name of the new table """ if not self.exists(): raise NoTable(f"Table {self.name} does not exist") with self.db.atomic(): - sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" - self.db.execute(sql) + if self.virtual_table_using is not None: + sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" + self.db.execute(sql) + return self.db.table(new_name) + self.db.execute( + rewrite_table_references( + rename_create_statement(self.schema, new_name), self.name, new_name + ) + ) + columns = [column.name for column in self.columns] + if not parse_without_rowid(self.schema) and "rowid" not in { + c.lower() for c in columns + }: + columns.insert(0, "rowid") + columns_sql = ", ".join(quote_identifier(c) for c in columns) + self.db.execute( + f"INSERT INTO {quote_identifier(new_name)} ({columns_sql}) " + f"SELECT {columns_sql} FROM {quote_identifier(self.name)};" + ) + if parse_autoincrement(self.schema): + sequence_row = self.db.execute( + "SELECT seq FROM sqlite_sequence WHERE name = ?", [self.name] + ).fetchone() + if sequence_row is not None: + self.db.execute( + "UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = ?", + [sequence_row[0], new_name], + ) + self.db.execute( + "INSERT INTO sqlite_sequence (name, seq) SELECT ?, ? " + "WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)", + [new_name, sequence_row[0], new_name], + ) + # Index names must be unique across the database + existing_names = { + row[0].lower() + for row in self.db.execute("SELECT name FROM sqlite_master") + } + for index in self.indexes: + row = self.db.execute( + "SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", + [index.name], + ).fetchone() + if row is None or row[0] is None: + # Automatic index for a PRIMARY KEY or UNIQUE constraint + continue + if self.name in index.name: + base_name = index.name.replace(self.name, new_name, 1) + else: + base_name = f"{new_name}_{index.name}" + index_name = base_name + suffix = 2 + while index_name.lower() in existing_names: + index_name = f"{base_name}_{suffix}" + suffix += 1 + existing_names.add(index_name.lower()) + self.db.execute( + rewrite_table_references( + rename_create_statement(row[0], index_name, table=new_name), + self.name, + new_name, + ) + ) return self.db.table(new_name) def transform( diff --git a/tests/test_cli.py b/tests/test_cli.py index 064026a0b..d4cc0e8f2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2847,6 +2847,22 @@ def test_duplicate_table(tmpdir): assert list(db.table("one").rows) == list(db.table("two").rows) +def test_duplicate_table_copies_schema_and_indexes(db_path): + db = Database(db_path) + db.execute("CREATE TABLE one (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.execute("CREATE INDEX idx_one_name ON one (name)") + db.table("one").insert({"id": 1, "name": "Cleo"}) + + result = CliRunner().invoke(cli.cli, ["duplicate", db_path, "one", "two"]) + + assert result.exit_code == 0, result.output + assert db.table("two").schema == ( + "CREATE TABLE two (id INTEGER PRIMARY KEY, name TEXT NOT NULL)" + ) + assert [index.name for index in db.table("two").indexes] == ["idx_two_name"] + assert list(db.table("two").rows) == [{"id": 1, "name": "Cleo"}] + + @pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") @pytest.mark.parametrize( "entrypoint,should_pass,should_fail", diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index c7a5612a2..e98cbb65c 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,4 +1,5 @@ import datetime +import sqlite3 import pytest @@ -32,8 +33,8 @@ def test_duplicate(fresh_db): assert [ {"name": "text_col", "type": "TEXT"}, {"name": "real_col", "type": "REAL"}, - {"name": "int_col", "type": "INT"}, - {"name": "bool_col", "type": "INT"}, + {"name": "int_col", "type": "INTEGER"}, + {"name": "bool_col", "type": "INTEGER"}, {"name": "datetime_col", "type": "TEXT"}, ] == [{"name": col.name, "type": col.type} for col in table2.columns] @@ -41,3 +42,208 @@ def test_duplicate(fresh_db): def test_duplicate_fails_if_table_does_not_exist(fresh_db): with pytest.raises(NoTable): fresh_db.table("not_a_table").duplicate("duplicated") + + +def test_duplicate_copies_create_table_statement(fresh_db): + fresh_db.execute("CREATE TABLE owners (id INTEGER PRIMARY KEY)") + fresh_db.execute(""" + CREATE TABLE pets ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + age INTEGER CHECK (age >= 0) DEFAULT 1, + owner_id INTEGER REFERENCES owners(id) + ) + """) + fresh_db.table("owners").insert({"id": 1}) + fresh_db.table("pets").insert({"id": 5, "name": "Cleo", "age": 3, "owner_id": 1}) + + copy = fresh_db.table("pets").duplicate("pets_copy") + + assert copy.schema == fresh_db.table("pets").schema.replace( + "TABLE pets", "TABLE pets_copy" + ) + assert copy.pks == ["id"] + assert [(fk.column, fk.other_table) for fk in copy.foreign_keys] == [ + ("owner_id", "owners") + ] + assert list(copy.rows) == [{"id": 5, "name": "Cleo", "age": 3, "owner_id": 1}] + copy.insert({"id": 6, "name": "Pancakes"}) + assert copy.get(6)["age"] == 1 + for bad_row in ( + {"id": 7, "name": "Cleo"}, + {"id": 8, "name": None}, + {"id": 9, "name": "Rex", "age": -1}, + ): + with pytest.raises(sqlite3.IntegrityError): + copy.insert(bad_row) + + +def test_duplicate_preserves_rowids(fresh_db): + table = fresh_db.table("plain") + table.insert_all([{"v": "a"}, {"v": "b"}, {"v": "c"}]) + fresh_db.execute("DELETE FROM plain WHERE v = 'a'") + + table.duplicate("plain_copy") + + assert fresh_db.execute("SELECT rowid, v FROM plain_copy").fetchall() == [ + (2, "b"), + (3, "c"), + ] + + +def test_duplicate_without_rowid(fresh_db): + fresh_db.execute("CREATE TABLE tags (k TEXT PRIMARY KEY, v TEXT) WITHOUT ROWID") + fresh_db.table("tags").insert({"k": "a", "v": "one"}) + + copy = fresh_db.table("tags").duplicate("tags_copy") + + assert copy.schema.endswith("WITHOUT ROWID") + assert list(copy.rows) == [{"k": "a", "v": "one"}] + + +def test_duplicate_strict(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.table("t").create({"id": int}, strict=True) + + assert fresh_db.table("t").duplicate("t_copy").strict + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 31, 0), + reason="Generated columns need SQLite 3.31", +) +def test_duplicate_generated_columns(fresh_db): + fresh_db.execute("CREATE TABLE t (a INTEGER, b INTEGER AS (a + 1))") + fresh_db.table("t").insert({"a": 1}) + + fresh_db.table("t").duplicate("t_copy") + + assert fresh_db.execute("SELECT a, b FROM t_copy").fetchall() == [(1, 2)] + + +def test_duplicate_recreates_indexes_with_new_names(fresh_db): + fresh_db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT, age INT)") + fresh_db.execute("CREATE UNIQUE INDEX idx_pets_name ON pets (name)") + fresh_db.execute("CREATE INDEX young ON pets (age) WHERE age < 2") + fresh_db.table("pets").insert({"id": 1, "name": "Cleo", "age": 1}) + + copy = fresh_db.table("pets").duplicate("pets_copy") + + assert {index.name for index in fresh_db.table("pets").indexes} == { + "idx_pets_name", + "young", + } + assert {(index.name, index.unique) for index in copy.indexes} == { + ("idx_pets_copy_name", 1), + ("pets_copy_young", 0), + } + assert fresh_db.execute( + "SELECT sql FROM sqlite_master WHERE name = 'pets_copy_young'" + ).fetchone() == ("CREATE INDEX pets_copy_young ON pets_copy (age) WHERE age < 2",) + with pytest.raises(sqlite3.IntegrityError): + copy.insert({"id": 2, "name": "Cleo"}) + + +def test_duplicate_index_name_avoids_existing_names(fresh_db): + fresh_db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT)") + fresh_db.execute("CREATE INDEX idx_pets_name ON pets (name)") + fresh_db.execute("CREATE TABLE other (name TEXT)") + fresh_db.execute("CREATE INDEX idx_copy_name ON other (name)") + + copy = fresh_db.table("pets").duplicate("copy") + + assert [index.name for index in copy.indexes] == ["idx_copy_name_2"] + + +def test_duplicate_does_not_copy_triggers(fresh_db): + fresh_db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT)") + fresh_db.execute("CREATE TABLE log (name TEXT)") + fresh_db.execute(""" + CREATE TRIGGER pets_insert AFTER INSERT ON pets + BEGIN INSERT INTO log VALUES (new.name); END + """) + + copy = fresh_db.table("pets").duplicate("pets_copy") + copy.insert({"id": 1, "name": "Cleo"}) + + assert copy.triggers == [] + assert fresh_db.table("log").count == 0 + + +def test_duplicate_fts_table_copies_rows(fresh_db): + fresh_db.table("docs").insert({"title": "hello"}) + fresh_db.table("docs").enable_fts(["title"]) + + copy = fresh_db.table("docs_fts").duplicate("docs_fts_copy") + + assert copy.virtual_table_using is None + assert list(copy.rows) == [{"title": "hello"}] + + +def test_duplicate_self_referencing_foreign_key_points_at_copy(fresh_db): + fresh_db.execute(""" + CREATE TABLE nodes ( + id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES nodes(id) + ) + """) + fresh_db.table("nodes").insert_all([{"id": 1}, {"id": 2, "parent_id": 1}]) + + copy = fresh_db.table("nodes").duplicate("nodes_copy") + + assert [(fk.column, fk.other_table) for fk in copy.foreign_keys] == [ + ("parent_id", "nodes_copy") + ] + assert list(copy.rows) == [ + {"id": 1, "parent_id": None}, + {"id": 2, "parent_id": 1}, + ] + + +def test_duplicate_preserves_autoincrement_sequence(fresh_db): + fresh_db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)") + table = fresh_db.table("t") + table.insert_all([{"v": "a"}, {"v": "b"}, {"v": "c"}]) + fresh_db.execute("DELETE FROM t WHERE id = 3") + + copy = table.duplicate("t_copy") + + assert "AUTOINCREMENT" in copy.schema + assert copy.insert({"v": "d"}).last_rowid == 4 + + +def test_duplicate_rewrites_table_qualified_columns(fresh_db): + fresh_db.execute(""" + CREATE TABLE pets ( + age INTEGER CHECK (pets.age >= 0), + note TEXT DEFAULT 'pets.age' + ) + """) + fresh_db.execute('CREATE INDEX young ON pets (age) WHERE "pets".age < 2') + fresh_db.table("pets").insert({"age": 1}) + + copy = fresh_db.table("pets").duplicate("pets_copy") + + assert "CHECK (pets_copy.age >= 0)" in copy.schema + assert "DEFAULT 'pets.age'" in copy.schema + assert fresh_db.execute( + "SELECT sql FROM sqlite_master WHERE name = 'pets_copy_young'" + ).fetchone() == ( + 'CREATE INDEX pets_copy_young ON pets_copy (age) WHERE "pets_copy".age < 2', + ) + with pytest.raises(sqlite3.IntegrityError): + copy.insert({"age": -1}) + + +def test_duplicate_names_that_need_quoting(fresh_db): + table = fresh_db.table("my pets") + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.create_index(["name"]) + + copy = table.duplicate('copy "2"') + + assert copy.schema.startswith('CREATE TABLE "copy ""2"""') + assert copy.pks == ["id"] + assert [index.name for index in copy.indexes] == ['copy "2"_idx_my_pets_name'] + assert list(copy.rows) == [{"id": 1, "name": "Cleo"}]