diff --git a/docs/cli.rst b/docs/cli.rst index 78c33b8ae..16a3e5fea 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2201,6 +2201,8 @@ The ``duplicate`` command duplicates a table - creating a new table with the sam sqlite-utils duplicate books.db authors authors_copy +The copy keeps the original table's column types, constraints, foreign keys, ``STRICT`` and ``WITHOUT ROWID`` options, ``rowid`` values and indexes (recreated under new names). Triggers are not copied. See :ref:`table.duplicate() ` for details. + .. note:: In Python: :ref:`table.duplicate() ` CLI reference: :ref:`sqlite-utils duplicate ` diff --git a/docs/python-api.rst b/docs/python-api.rst index d515642a5..b83855478 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1060,6 +1060,17 @@ The ``table.duplicate()`` method creates a copy of the table, copying both the t The new ``authors_copy`` table will now contain a duplicate copy of the data from ``authors``. +The new table is created from the original table's ``CREATE TABLE`` statement with only the table name changed, so it keeps the same column types, primary key, ``NOT NULL``, ``UNIQUE``, ``CHECK`` and ``DEFAULT`` constraints, foreign keys, ``STRICT`` and ``WITHOUT ROWID`` options. Those constraints are enforced on the copy. Row ``rowid`` values and the ``AUTOINCREMENT`` counter are preserved too. + +A few parts of the schema are adjusted for the new table name: + +- References to the table itself, such as a self-referencing foreign key or an ``authors.name`` column reference in a ``CHECK`` constraint, are updated to point at the copy. +- Indexes are recreated under new names, because index names must be unique within a database. If the index name contains the original table name that part is replaced, so ``idx_authors_name`` becomes ``idx_authors_copy_name``. Other index names are prefixed with the new table name, so ``by_name`` becomes ``authors_copy_by_name``. A numeric suffix is added if that name is already in use. + +Triggers are not copied. + +Duplicating a virtual table, such as a full-text search table, creates a regular table containing a copy of its rows. + This method raises ``sqlite_utils.db.NoTable`` if the table does not exist. .. note:: diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 7377f4fd6..f6de0fcdc 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -785,6 +785,90 @@ def parse_uniques(create_sql: str) -> list[Unique]: return uniques +def parse_without_rowid(create_sql: str) -> bool: + """Return True if a CREATE TABLE statement ends with WITHOUT ROWID.""" + if _table_body(create_sql) is None: + return False + tokens = _meaningful(_lex(create_sql)) + close = max(i for i, token in enumerate(tokens) if token.text == ")") + return any(token.is_keyword("WITHOUT") for token in tokens[close + 1 :]) + + +def rename_create_statement( + create_sql: str, new_name: str, table: str | None = None +) -> str: + """ + Rename the table or index created by a CREATE TABLE or CREATE INDEX statement. + + For CREATE INDEX, ``table`` replaces the name of the indexed table. Everything + else in the statement, including comments and whitespace, is left unchanged. + """ + tokens = _meaningful(_lex(create_sql)) + if not tokens or not tokens[0].is_keyword("CREATE"): + raise ParseError("Expected CREATE TABLE or CREATE INDEX") + index = 1 + while index < len(tokens) and tokens[index].text.upper() in ( + "TEMP", + "TEMPORARY", + "UNIQUE", + ): + index += 1 + if index >= len(tokens) or not ( + tokens[index].is_keyword("TABLE") or tokens[index].is_keyword("INDEX") + ): + raise ParseError("Expected CREATE TABLE or CREATE INDEX") + index += 1 + if ( + index + 2 < len(tokens) + and tokens[index].is_keyword("IF") + and tokens[index + 1].is_keyword("NOT") + and tokens[index + 2].is_keyword("EXISTS") + ): + index += 3 + if index + 1 < len(tokens) and tokens[index + 1].text == ".": + index += 2 + if index >= len(tokens): + raise ParseError("CREATE statement is missing its name") + replacements = [(tokens[index], new_name)] + if table is not None: + if index + 2 >= len(tokens) or not tokens[index + 1].is_keyword("ON"): + raise ParseError("CREATE INDEX is missing ON") + replacements.append((tokens[index + 2], table)) + for token, replacement in reversed(replacements): + create_sql = ( + create_sql[: token.start] + + _quote_replacement(token.text, replacement) + + create_sql[token.end :] + ) + return create_sql + + +def rewrite_table_references(sql: str, old_table: str, new_table: str) -> str: + """ + Rename ``old_table`` where it qualifies a column (``table.column``) or is the + target of ``REFERENCES``. Strings and comments are left unchanged. + """ + tokens = _meaningful(_lex(sql)) + folded = _ascii_fold(old_table) + replacements = [ + token + for i, token in enumerate(tokens) + if token.kind in ("word", "identifier") + and _ascii_fold(_unquote(token.text)) == folded + and ( + (i + 1 < len(tokens) and tokens[i + 1].text == ".") + or (i > 0 and tokens[i - 1].is_keyword("REFERENCES")) + ) + ] + for token in reversed(replacements): + sql = ( + sql[: token.start] + + _quote_replacement(token.text, new_table) + + sql[token.end :] + ) + return sql + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c011d9bb0..744ca3265 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -37,7 +37,10 @@ parse_checks, parse_column_comments, parse_uniques, + parse_without_rowid, + rename_create_statement, rewrite_check_expression, + rewrite_table_references, sql_ends_in_line_comment, ) from .utils import ( @@ -2629,15 +2632,77 @@ def create( def duplicate(self, new_name: str) -> "Table": """ - Create a duplicate of this table, copying across the schema and all row data. + Create a duplicate of this table, copying across the schema, indexes, rowids + and all row data. Triggers are not copied. :param new_name: Name of the new table """ if not self.exists(): raise NoTable(f"Table {self.name} does not exist") with self.db.atomic(): - sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" - self.db.execute(sql) + if self.virtual_table_using is not None: + sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" + self.db.execute(sql) + return self.db.table(new_name) + self.db.execute( + rewrite_table_references( + rename_create_statement(self.schema, new_name), self.name, new_name + ) + ) + columns = [column.name for column in self.columns] + if not parse_without_rowid(self.schema) and "rowid" not in { + c.lower() for c in columns + }: + columns.insert(0, "rowid") + columns_sql = ", ".join(quote_identifier(c) for c in columns) + self.db.execute( + f"INSERT INTO {quote_identifier(new_name)} ({columns_sql}) " + f"SELECT {columns_sql} FROM {quote_identifier(self.name)};" + ) + if parse_autoincrement(self.schema): + sequence_row = self.db.execute( + "SELECT seq FROM sqlite_sequence WHERE name = ?", [self.name] + ).fetchone() + if sequence_row is not None: + self.db.execute( + "UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = ?", + [sequence_row[0], new_name], + ) + self.db.execute( + "INSERT INTO sqlite_sequence (name, seq) SELECT ?, ? " + "WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)", + [new_name, sequence_row[0], new_name], + ) + # Index names must be unique across the database + existing_names = { + row[0].lower() + for row in self.db.execute("SELECT name FROM sqlite_master") + } + for index in self.indexes: + row = self.db.execute( + "SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", + [index.name], + ).fetchone() + if row is None or row[0] is None: + # Automatic index for a PRIMARY KEY or UNIQUE constraint + continue + if self.name in index.name: + base_name = index.name.replace(self.name, new_name, 1) + else: + base_name = f"{new_name}_{index.name}" + index_name = base_name + suffix = 2 + while index_name.lower() in existing_names: + index_name = f"{base_name}_{suffix}" + suffix += 1 + existing_names.add(index_name.lower()) + self.db.execute( + rewrite_table_references( + rename_create_statement(row[0], index_name, table=new_name), + self.name, + new_name, + ) + ) return self.db.table(new_name) def transform( diff --git a/tests/test_cli.py b/tests/test_cli.py index 064026a0b..d4cc0e8f2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2847,6 +2847,22 @@ def test_duplicate_table(tmpdir): assert list(db.table("one").rows) == list(db.table("two").rows) +def test_duplicate_table_copies_schema_and_indexes(db_path): + db = Database(db_path) + db.execute("CREATE TABLE one (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.execute("CREATE INDEX idx_one_name ON one (name)") + db.table("one").insert({"id": 1, "name": "Cleo"}) + + result = CliRunner().invoke(cli.cli, ["duplicate", db_path, "one", "two"]) + + assert result.exit_code == 0, result.output + assert db.table("two").schema == ( + "CREATE TABLE two (id INTEGER PRIMARY KEY, name TEXT NOT NULL)" + ) + assert [index.name for index in db.table("two").indexes] == ["idx_two_name"] + assert list(db.table("two").rows) == [{"id": 1, "name": "Cleo"}] + + @pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") @pytest.mark.parametrize( "entrypoint,should_pass,should_fail", diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index c7a5612a2..e98cbb65c 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,4 +1,5 @@ import datetime +import sqlite3 import pytest @@ -32,8 +33,8 @@ def test_duplicate(fresh_db): assert [ {"name": "text_col", "type": "TEXT"}, {"name": "real_col", "type": "REAL"}, - {"name": "int_col", "type": "INT"}, - {"name": "bool_col", "type": "INT"}, + {"name": "int_col", "type": "INTEGER"}, + {"name": "bool_col", "type": "INTEGER"}, {"name": "datetime_col", "type": "TEXT"}, ] == [{"name": col.name, "type": col.type} for col in table2.columns] @@ -41,3 +42,208 @@ def test_duplicate(fresh_db): def test_duplicate_fails_if_table_does_not_exist(fresh_db): with pytest.raises(NoTable): fresh_db.table("not_a_table").duplicate("duplicated") + + +def test_duplicate_copies_create_table_statement(fresh_db): + fresh_db.execute("CREATE TABLE owners (id INTEGER PRIMARY KEY)") + fresh_db.execute(""" + CREATE TABLE pets ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + age INTEGER CHECK (age >= 0) DEFAULT 1, + owner_id INTEGER REFERENCES owners(id) + ) + """) + fresh_db.table("owners").insert({"id": 1}) + fresh_db.table("pets").insert({"id": 5, "name": "Cleo", "age": 3, "owner_id": 1}) + + copy = fresh_db.table("pets").duplicate("pets_copy") + + assert copy.schema == fresh_db.table("pets").schema.replace( + "TABLE pets", "TABLE pets_copy" + ) + assert copy.pks == ["id"] + assert [(fk.column, fk.other_table) for fk in copy.foreign_keys] == [ + ("owner_id", "owners") + ] + assert list(copy.rows) == [{"id": 5, "name": "Cleo", "age": 3, "owner_id": 1}] + copy.insert({"id": 6, "name": "Pancakes"}) + assert copy.get(6)["age"] == 1 + for bad_row in ( + {"id": 7, "name": "Cleo"}, + {"id": 8, "name": None}, + {"id": 9, "name": "Rex", "age": -1}, + ): + with pytest.raises(sqlite3.IntegrityError): + copy.insert(bad_row) + + +def test_duplicate_preserves_rowids(fresh_db): + table = fresh_db.table("plain") + table.insert_all([{"v": "a"}, {"v": "b"}, {"v": "c"}]) + fresh_db.execute("DELETE FROM plain WHERE v = 'a'") + + table.duplicate("plain_copy") + + assert fresh_db.execute("SELECT rowid, v FROM plain_copy").fetchall() == [ + (2, "b"), + (3, "c"), + ] + + +def test_duplicate_without_rowid(fresh_db): + fresh_db.execute("CREATE TABLE tags (k TEXT PRIMARY KEY, v TEXT) WITHOUT ROWID") + fresh_db.table("tags").insert({"k": "a", "v": "one"}) + + copy = fresh_db.table("tags").duplicate("tags_copy") + + assert copy.schema.endswith("WITHOUT ROWID") + assert list(copy.rows) == [{"k": "a", "v": "one"}] + + +def test_duplicate_strict(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.table("t").create({"id": int}, strict=True) + + assert fresh_db.table("t").duplicate("t_copy").strict + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 31, 0), + reason="Generated columns need SQLite 3.31", +) +def test_duplicate_generated_columns(fresh_db): + fresh_db.execute("CREATE TABLE t (a INTEGER, b INTEGER AS (a + 1))") + fresh_db.table("t").insert({"a": 1}) + + fresh_db.table("t").duplicate("t_copy") + + assert fresh_db.execute("SELECT a, b FROM t_copy").fetchall() == [(1, 2)] + + +def test_duplicate_recreates_indexes_with_new_names(fresh_db): + fresh_db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT, age INT)") + fresh_db.execute("CREATE UNIQUE INDEX idx_pets_name ON pets (name)") + fresh_db.execute("CREATE INDEX young ON pets (age) WHERE age < 2") + fresh_db.table("pets").insert({"id": 1, "name": "Cleo", "age": 1}) + + copy = fresh_db.table("pets").duplicate("pets_copy") + + assert {index.name for index in fresh_db.table("pets").indexes} == { + "idx_pets_name", + "young", + } + assert {(index.name, index.unique) for index in copy.indexes} == { + ("idx_pets_copy_name", 1), + ("pets_copy_young", 0), + } + assert fresh_db.execute( + "SELECT sql FROM sqlite_master WHERE name = 'pets_copy_young'" + ).fetchone() == ("CREATE INDEX pets_copy_young ON pets_copy (age) WHERE age < 2",) + with pytest.raises(sqlite3.IntegrityError): + copy.insert({"id": 2, "name": "Cleo"}) + + +def test_duplicate_index_name_avoids_existing_names(fresh_db): + fresh_db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT)") + fresh_db.execute("CREATE INDEX idx_pets_name ON pets (name)") + fresh_db.execute("CREATE TABLE other (name TEXT)") + fresh_db.execute("CREATE INDEX idx_copy_name ON other (name)") + + copy = fresh_db.table("pets").duplicate("copy") + + assert [index.name for index in copy.indexes] == ["idx_copy_name_2"] + + +def test_duplicate_does_not_copy_triggers(fresh_db): + fresh_db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT)") + fresh_db.execute("CREATE TABLE log (name TEXT)") + fresh_db.execute(""" + CREATE TRIGGER pets_insert AFTER INSERT ON pets + BEGIN INSERT INTO log VALUES (new.name); END + """) + + copy = fresh_db.table("pets").duplicate("pets_copy") + copy.insert({"id": 1, "name": "Cleo"}) + + assert copy.triggers == [] + assert fresh_db.table("log").count == 0 + + +def test_duplicate_fts_table_copies_rows(fresh_db): + fresh_db.table("docs").insert({"title": "hello"}) + fresh_db.table("docs").enable_fts(["title"]) + + copy = fresh_db.table("docs_fts").duplicate("docs_fts_copy") + + assert copy.virtual_table_using is None + assert list(copy.rows) == [{"title": "hello"}] + + +def test_duplicate_self_referencing_foreign_key_points_at_copy(fresh_db): + fresh_db.execute(""" + CREATE TABLE nodes ( + id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES nodes(id) + ) + """) + fresh_db.table("nodes").insert_all([{"id": 1}, {"id": 2, "parent_id": 1}]) + + copy = fresh_db.table("nodes").duplicate("nodes_copy") + + assert [(fk.column, fk.other_table) for fk in copy.foreign_keys] == [ + ("parent_id", "nodes_copy") + ] + assert list(copy.rows) == [ + {"id": 1, "parent_id": None}, + {"id": 2, "parent_id": 1}, + ] + + +def test_duplicate_preserves_autoincrement_sequence(fresh_db): + fresh_db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)") + table = fresh_db.table("t") + table.insert_all([{"v": "a"}, {"v": "b"}, {"v": "c"}]) + fresh_db.execute("DELETE FROM t WHERE id = 3") + + copy = table.duplicate("t_copy") + + assert "AUTOINCREMENT" in copy.schema + assert copy.insert({"v": "d"}).last_rowid == 4 + + +def test_duplicate_rewrites_table_qualified_columns(fresh_db): + fresh_db.execute(""" + CREATE TABLE pets ( + age INTEGER CHECK (pets.age >= 0), + note TEXT DEFAULT 'pets.age' + ) + """) + fresh_db.execute('CREATE INDEX young ON pets (age) WHERE "pets".age < 2') + fresh_db.table("pets").insert({"age": 1}) + + copy = fresh_db.table("pets").duplicate("pets_copy") + + assert "CHECK (pets_copy.age >= 0)" in copy.schema + assert "DEFAULT 'pets.age'" in copy.schema + assert fresh_db.execute( + "SELECT sql FROM sqlite_master WHERE name = 'pets_copy_young'" + ).fetchone() == ( + 'CREATE INDEX pets_copy_young ON pets_copy (age) WHERE "pets_copy".age < 2', + ) + with pytest.raises(sqlite3.IntegrityError): + copy.insert({"age": -1}) + + +def test_duplicate_names_that_need_quoting(fresh_db): + table = fresh_db.table("my pets") + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.create_index(["name"]) + + copy = table.duplicate('copy "2"') + + assert copy.schema.startswith('CREATE TABLE "copy ""2"""') + assert copy.pks == ["id"] + assert [index.name for index in copy.indexes] == ['copy "2"_idx_my_pets_name'] + assert list(copy.rows) == [{"id": 1, "name": "Cleo"}]