From 6dec072430b40190365931a590ab479bc92157a0 Mon Sep 17 00:00:00 2001 From: Mo0nl19ht <81469045+Mo0nl19ht@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:43:31 +0900 Subject: [PATCH] Preserve column COLLATE through transforms table.transform() rebuilt each column from PRAGMA table_info, which does not report collation, so a column declared "name TEXT COLLATE NOCASE" became plain "name TEXT" and comparisons on it turned case-sensitive. Parse column collations from the existing schema with a new parse_column_collations() helper and pass them to create_table_sql() via a private _column_collations argument. Collations follow renamed columns and are dropped with dropped columns. Refs #954 --- docs/python-api.rst | 2 ++ sqlite_utils/create_table_parser.py | 35 +++++++++++++++++++++++--- sqlite_utils/db.py | 18 ++++++++++++++ tests/test_cli.py | 12 +++++++++ tests/test_create_table_parser.py | 31 +++++++++++++++++++++++ tests/test_transform.py | 38 +++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index d515642a5..052c3b9d7 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2015,6 +2015,8 @@ A column-level check is removed if its owning column is dropped. Dropping a colu Comments immediately before or after a column definition are preserved too. They move with that column if it is renamed or reordered, and are removed if the column is dropped. A comment between two column definitions is treated as belonging to the following column. +Column collations such as ``name TEXT COLLATE NOCASE`` are preserved as well. They move with the column if it is renamed, and are removed if the column is dropped. + .. _python_api_transform_views: Tables referenced by views diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 7377f4fd6..3405fb26b 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -661,10 +661,7 @@ def _unique_columns( return tuple(columns), close + 1 -def _column_uniques( - item: str, tokens: list[_Token], column: str, base_offset: int -) -> list[Unique]: - uniques: list[Unique] = [] +def _column_collation(tokens: list[_Token]) -> str: collation = "" collation_index = 1 while collation_index < len(tokens): @@ -679,6 +676,14 @@ def _column_uniques( collation_index += 2 continue collation_index += 1 + return collation + + +def _column_uniques( + item: str, tokens: list[_Token], column: str, base_offset: int +) -> list[Unique]: + uniques: list[Unique] = [] + collation = _column_collation(tokens) pending_name = "" pending_start: int | None = None index = 1 @@ -785,6 +790,28 @@ def parse_uniques(create_sql: str) -> list[Unique]: return uniques +def parse_column_collations(create_sql: str) -> dict[str, str]: + """Return the COLLATE sequence declared on each column definition.""" + body_info = _table_body(create_sql) + if body_info is None: + return {} + body, _ = body_info + collations: dict[str, str] = {} + for item, _, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + head = item_tokens[0] + if ( + head.kind == "word" and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ) or head.is_keyword("CONSTRAINT"): + continue + collation = _column_collation(item_tokens) + if collation: + collations[_unquote(head.text)] = collation + return collations + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c011d9bb0..d518a3f9a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -35,6 +35,7 @@ check_references_identifier, parse_autoincrement, parse_checks, + parse_column_collations, parse_column_comments, parse_uniques, rewrite_check_expression, @@ -1444,6 +1445,7 @@ def create_table_sql( _column_comments: Mapping[str, ColumnComments] | None = None, _autoincrement: str | None = None, _uniques: Iterable[Unique] | None = None, + _column_collations: Mapping[str, str] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1493,6 +1495,10 @@ def create_table_sql( resolve_casing(name, columns): comments for name, comments in (_column_comments or {}).items() } + column_collations = { + resolve_casing(name, columns): collation + for name, collation in (_column_collations or {}).items() + } checks = list(_checks or ()) checks_by_column: dict[str, list[Check]] = {} table_checks: list[Check] = [] @@ -1609,6 +1615,10 @@ def sort_key(p): raise ValueError("AUTOINCREMENT requires a single-column primary key") for column_name, column_type in column_items: column_extras = [] + if column_name in column_collations: + column_extras.append( + f"COLLATE {quote_identifier(column_collations[column_name])}" + ) if column_name == single_pk: column_extras.append("PRIMARY KEY") if column_name == _autoincrement: @@ -2843,6 +2853,7 @@ def transform_sql( try: existing_checks = self.checks existing_column_comments = parse_column_comments(self.schema) + existing_column_collations = parse_column_collations(self.schema) existing_autoincrement = parse_autoincrement(self.schema) existing_uniques = parse_uniques(self.schema) except ParseError as ex: @@ -2908,6 +2919,12 @@ def transform_sql( if owner not in drop: create_table_column_comments[rename.get(owner) or owner] = comments + create_table_column_collations: dict[str, str] = {} + for column, collation in existing_column_collations.items(): + owner = resolve_casing(column, existing_columns) + if owner not in drop: + create_table_column_collations[rename.get(owner) or owner] = collation + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -3088,6 +3105,7 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: _column_comments=create_table_column_comments, _autoincrement=create_table_autoincrement, _uniques=create_table_uniques, + _column_collations=create_table_column_collations, ).strip() ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 064026a0b..0e44c16b9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2008,6 +2008,18 @@ def test_transform_strict_option_with_invalid_data(db_path): assert not any(name.startswith("dogs_new_") for name in db.table_names()) +def test_transform_preserves_column_collation(db_path): + db = Database(db_path) + db.execute("CREATE TABLE dogs (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)") + + result = CliRunner().invoke( + cli.cli, ["transform", db_path, "dogs", "--rename", "name", "dog_name"] + ) + + assert result.exit_code == 0, result.output + assert '"dog_name" TEXT COLLATE "NOCASE"' in db.table("dogs").schema + + def test_transform_column_to_any(db_path): db = Database(db_path) if not db.supports_strict: diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index 74a089cc0..2ce5a9912 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -12,6 +12,7 @@ UniqueColumn, parse_autoincrement, parse_checks, + parse_column_collations, parse_column_comments, parse_uniques, ) @@ -151,6 +152,36 @@ def test_parse_autoincrement(sql, expected): assert parse_autoincrement(sql) == expected +@pytest.mark.parametrize( + "sql,expected", + [ + ( + "CREATE TABLE t(id INTEGER, name TEXT COLLATE NOCASE)", + {"name": "NOCASE"}, + ), + ( + 'CREATE TABLE t("quoted name" TEXT COLLATE "RTRIM" NOT NULL, other TEXT)', + {"quoted name": "RTRIM"}, + ), + ( + "CREATE TABLE t(code VARCHAR(10) PRIMARY KEY COLLATE BINARY)", + {"code": "BINARY"}, + ), + ( + "CREATE TABLE t(name TEXT CHECK (name = 'x' COLLATE NOCASE))", + {}, + ), + ( + "CREATE TABLE t(name TEXT /* COLLATE NOCASE */, UNIQUE (name COLLATE NOCASE))", + {}, + ), + ], +) +def test_parse_column_collations(sql, expected): + sqlite3.connect(":memory:").execute(sql) + assert parse_column_collations(sql) == expected + + def test_parse_column_and_table_uniques(): sql = """ CREATE TABLE memberships ( diff --git a/tests/test_transform.py b/tests/test_transform.py index 873871308..cd1fa4e5a 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1090,6 +1090,44 @@ def test_transform_preserves_column_unique_collation(fresh_db): people.insert({"id": 2, "full_name": "cleo"}) +def test_transform_preserves_column_collation(fresh_db): + fresh_db.execute(""" + CREATE TABLE people ( + id INTEGER PRIMARY KEY, + name TEXT COLLATE NOCASE NOT NULL, + code TEXT COLLATE RTRIM, + age INTEGER + ) + """) + people = fresh_db.table("people") + people.insert({"id": 1, "name": "Cleo", "code": "a", "age": 5}) + + people.transform(types={"age": str}) + + assert '"name" TEXT COLLATE "NOCASE" NOT NULL' in people.schema + assert '"code" TEXT COLLATE "RTRIM"' in people.schema + assert people.columns_dict["age"] is str + assert fresh_db.execute( + "select count(*) from people where name = 'cleo'" + ).fetchone() == (1,) + + +def test_transform_collation_follows_renamed_column_and_drop(fresh_db): + fresh_db.execute(""" + CREATE TABLE people ( + id INTEGER PRIMARY KEY, + name TEXT COLLATE NOCASE, + nickname TEXT COLLATE NOCASE + ) + """) + people = fresh_db.table("people") + + people.transform(rename={"name": "full_name"}, drop={"nickname"}) + + assert '"full_name" TEXT COLLATE "NOCASE"' in people.schema + assert "nickname" not in people.schema + + def test_transform_drops_entire_composite_unique_constraint(fresh_db): fresh_db.execute(""" CREATE TABLE memberships (