From c15c0e5f092bd0485579541de3fe6c50a434e28a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 8 Sep 2026 12:12:43 +0200 Subject: [PATCH 01/19] Add direct table option --- crates/core/src/schema/common.rs | 19 ++++++++ crates/core/src/schema/inspection.rs | 12 +++-- crates/core/src/schema/management.rs | 47 +++++++++++-------- crates/core/src/schema/raw_table.rs | 43 ++++++++++++++---- crates/core/src/schema/table_info.rs | 61 ++++++++++++++++++++++++- crates/core/src/views.rs | 13 ++++++ dart/test/schema_test.dart | 68 ++++++++++++++++++++++++++++ 7 files changed, 231 insertions(+), 32 deletions(-) diff --git a/crates/core/src/schema/common.rs b/crates/core/src/schema/common.rs index f133d44..2e47d25 100644 --- a/crates/core/src/schema/common.rs +++ b/crates/core/src/schema/common.rs @@ -18,6 +18,25 @@ pub enum SchemaTable<'a> { } impl<'a> SchemaTable<'a> { + /// The type name used for the table when referenced in `ps_crud`, `ps_oplog` and other tables. + pub fn name(&self) -> &str { + match self { + SchemaTable::Json(table) => &table.name, + SchemaTable::Raw { + definition, + schema: _, + } => &definition.name, + } + } + + pub fn data_column(&self) -> Option<&'static str> { + if let SchemaTable::Json(table) = self { + Some(table.data_column_name()) + } else { + None + } + } + pub fn common_options(&self) -> &CommonTableOptions { match self { Self::Json(table) => &table.options, diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 09b72f7..1cc39fb 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -12,7 +12,9 @@ pub struct ExistingView { /// The name of the view itself. pub name: String, /// SQL contents of the `CREATE VIEW` statement. - pub sql: String, + /// + /// This is not set for as_raw_table tables, which don't have a view. + pub sql: Option, /// SQL contents of all triggers implementing deletes by forwarding to /// `ps_data` and `ps_crud`. pub delete_trigger_sql: String, @@ -52,7 +54,7 @@ SELECT results.push(ExistingView { name, - sql, + sql: Some(sql), delete_trigger_sql: delete, insert_trigger_sql: insert, update_trigger_sql: update, @@ -69,8 +71,10 @@ SELECT } pub fn create(&self, db: Database) -> Result<()> { - Self::drop_by_name(db, &self.name)?; - db.exec_safe_str(&self.sql)?; + if let Some(create_view) = &self.sql { + Self::drop_by_name(db, &self.name)?; + db.exec_safe_str(create_view)?; + } db.exec_safe_str(&self.delete_trigger_sql)?; db.exec_safe_str(&self.insert_trigger_sql)?; db.exec_safe_str(&self.update_trigger_sql)?; diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 90f5d8c..20e742a 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -54,28 +54,33 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { } // New table. - let quoted_internal_name = SqlBuffer::quote_identifier(&table.internal_name()); + let data_column = table.data_column_name(); + let mut create_table = SqlBuffer::default(); + + create_table.push_str("CREATE TABLE "); + table.write_name(&mut create_table); + _ = write!( + &mut create_table, + "(id TEXT PRIMARY KEY NOT NULL, {data_column} TEXT" + ); + + if table.direct { + for column in &table.columns { + create_table.push_char(','); + let _ = create_table.identifier().write_str(&column.name); + let _ = write!(&mut create_table, " {}", column.type_name); + } - db.exec_safe_str(&format!( - "CREATE TABLE {:}(id TEXT PRIMARY KEY NOT NULL, data TEXT)", - quoted_internal_name - ))?; + create_table.push_str(") STRICT /* ps-managed */;"); + } else { + create_table.push_str(");"); + } + + db.exec_safe_str(&create_table.sql)?; if !table.local_only() { // MOVE data if any - db.exec_text( - &format!( - "INSERT INTO {:}(id, data) - SELECT id, data - FROM ps_untyped - WHERE type = ?", - quoted_internal_name - ), - &table.name, - )?; - - // language=SQLite - db.exec_text("DELETE FROM ps_untyped WHERE type = ?", &table.name)?; + table.move_from_ps_untyped(db)?; } } @@ -217,7 +222,11 @@ fn update_views(db: Database, schema: &Schema) -> Result<()> { }; for table in &schema.tables { - let view_sql = powersync_view_sql(table); + let view_sql = if table.direct { + None + } else { + Some(powersync_view_sql(table)) + }; let delete_trigger_sql = powersync_trigger_delete_sql(table)?; let insert_trigger_sql = powersync_trigger_insert_sql(table)?; let update_trigger_sql = powersync_trigger_update_sql(table)?; diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index ad146aa..9d9cd3a 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -232,6 +232,22 @@ pub fn generate_raw_table_trigger( schema: &resolved_table, }; + generate_schema_table_trigger( + local_table_name, + as_schema_table, + synced_columns.as_ref(), + trigger_name, + write, + ) +} + +pub fn generate_schema_table_trigger( + local_table_name: &str, + table: SchemaTable, + synced_columns: Option<&ColumnFilter>, + trigger_name: &str, + write: WriteType, +) -> Result { let mut buffer = SqlBuffer::new(); buffer.create_trigger("", trigger_name); buffer.trigger_after(write, local_table_name); @@ -242,7 +258,7 @@ pub fn generate_raw_table_trigger( buffer.push_str(" AND\n("); // If we have a filter for synced columns (instead of syncing all of them), we want to add // additional WHEN clauses to enesure the trigger runs for updates on those columns only. - for (i, name) in as_schema_table.column_names().enumerate() { + for (i, name) in table.column_names().enumerate() { if i != 0 { buffer.push_str(" OR "); } @@ -258,14 +274,14 @@ pub fn generate_raw_table_trigger( buffer.push_str(" BEGIN\n"); - if table.schema.options.flags.insert_only() { + if table.common_options().flags.insert_only() { if write != WriteType::Insert { // Prevent illegal writes to a table marked as insert-only by raising errors here. buffer.push_str("SELECT RAISE(FAIL, 'Unexpected update on insert-only table');\n"); } else { // Insert-only tables use manual CRUD writes so they don't block incoming data. - let fragment = table_columns_to_json_object("NEW", &as_schema_table)?; - buffer.powersync_crud_manual_put(&table.name, &fragment); + let fragment = table_columns_to_json_object("NEW", &table)?; + buffer.powersync_crud_manual_put(table.name(), &fragment); } } else { if write == WriteType::Update { @@ -273,9 +289,9 @@ pub fn generate_raw_table_trigger( buffer.check_id_not_changed(); } - let json_fragment_new = table_columns_to_json_object("NEW", &as_schema_table)?; + let json_fragment_new = table_columns_to_json_object("NEW", &table)?; let json_fragment_old = if write == WriteType::Update { - Some(table_columns_to_json_object("OLD", &as_schema_table)?) + Some(table_columns_to_json_object("OLD", &table)?) } else { None }; @@ -294,15 +310,26 @@ pub fn generate_raw_table_trigger( write!(f, ", {json_fragment_new}))") }); + if write == WriteType::Update + && let Some(data_column) = table.data_column() + { + // If the table has a __data column storing the full JSON row, we also need to update + // that. + let _ = write!( + &mut buffer, + "UPDATE {local_table_name} SET {data_column} = {json_fragment_new} WHERE id = NEW.id;\n" + ); + } + buffer.insert_into_powersync_crud(InsertIntoCrud { op: write, - table: &as_schema_table, + table: &table, id_expr: if write == WriteType::Delete { "OLD.id" } else { "NEW.id" }, - type_name: &table.name, + type_name: table.name(), data: match write { // There is no data for deleted rows. WriteType::Delete => None, diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index fc46b9d..67bd96d 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -1,3 +1,5 @@ +use core::fmt::Write; + use alloc::rc::Rc; use alloc::string::ToString; use alloc::vec; @@ -5,7 +7,10 @@ use alloc::{collections::btree_set::BTreeSet, format, string::String, vec::Vec}; use serde::{Deserialize, de::Visitor}; use crate::error::PowerSyncError; -use crate::schema::ColumnFilter; +use crate::schema::raw_table::generate_schema_table_trigger; +use crate::schema::{ColumnFilter, SchemaTable}; +use crate::utils::database::Database; +use crate::utils::{SqlBuffer, WriteType}; #[derive(Deserialize)] pub struct Table { @@ -17,6 +22,7 @@ pub struct Table { pub indexes: Vec, #[serde(flatten)] pub options: CommonTableOptions, + pub direct: bool, } /// Options shared between regular and raw tables. @@ -78,6 +84,59 @@ impl Table { format!("ps_data__{:}", self.name) } } + + pub fn move_from_ps_untyped(&self, db: Database) -> Result<(), PowerSyncError> { + let mut stmt = SqlBuffer::default(); + let direct = self.direct; + + stmt.push_str("INSERT INTO "); + self.write_name(&mut stmt); + let _ = write!(&mut stmt, "(id, {}", self.data_column_name()); + + if direct { + for column in &self.columns { + stmt.push_char(','); + let _ = stmt.identifier().write_str(&column.name); + } + } + + stmt.push_str(") SELECT id, data"); + if direct { + for column in &self.columns { + stmt.push_char(','); + stmt.json_extract_and_cast("data", &column.name, &column.type_name); + } + } + + stmt.push_str(" FROM ps_untyped WHERE type = ?"); + + db.exec_text(&stmt.sql, &self.name)?; + db.exec_text("DELETE FROM ps_untyped WHERE type = ?", &self.name) + } + + pub fn write_name(&self, buffer: &mut SqlBuffer) { + if self.direct { + // Direct tables don't have views, so use the name of the table directly. + let _ = buffer.identifier().write_str(&self.name); + } else { + buffer.quote_internal_name(&self.name, self.local_only()); + } + } + + pub fn data_column_name(&self) -> &'static str { + if self.direct { "__data" } else { "data" } + } + + pub fn generate_direct_trigger(&self, write: WriteType) -> Result { + debug_assert!(self.direct); + generate_schema_table_trigger( + &self.name, + SchemaTable::Json(self), + None, + &format!("{}_trigger_{}", self.name, write), + write, + ) + } } impl RawTable { diff --git a/crates/core/src/views.rs b/crates/core/src/views.rs index a2c1b51..f5e7089 100644 --- a/crates/core/src/views.rs +++ b/crates/core/src/views.rs @@ -59,6 +59,10 @@ pub fn powersync_view_sql(table_info: &Table) -> String { } pub fn powersync_trigger_delete_sql(table_info: &Table) -> Result { + if table_info.direct { + return table_info.generate_direct_trigger(WriteType::Delete); + } + if table_info.options.flags.insert_only() { // Insert-only tables have no DELETE triggers return Ok(String::new()); @@ -117,6 +121,10 @@ pub fn powersync_trigger_delete_sql(table_info: &Table) -> Result { } pub fn powersync_trigger_insert_sql(table_info: &Table) -> Result { + if table_info.direct { + return table_info.generate_direct_trigger(WriteType::Insert); + } + let name = &table_info.name; let view_name = table_info.view_name(); let local_only = table_info.options.flags.local_only(); @@ -168,6 +176,10 @@ pub fn powersync_trigger_insert_sql(table_info: &Table) -> Result { } pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { + if table_info.direct { + return table_info.generate_direct_trigger(WriteType::Update); + } + if table_info.options.flags.insert_only() { // Insert-only tables have no UPDATE triggers return Ok(String::new()); @@ -359,6 +371,7 @@ mod test { ], indexes: vec![], options: Default::default(), + direct: false, }; } diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index dfbd4ff..d925a4a 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -322,6 +322,74 @@ END''', test('#$i', () => testCase.testWith(db)); } }); + + group('direct tables', () { + final table = { + 'name': 'users', + 'columns': [ + {'name': 'name', 'type': 'text'} + ], + 'direct': true, + }; + + test('create', () { + db.executeInTx('SELECT powersync_replace_schema(?)', [ + json.encode({'tables': []}) + ]); + db.execute('INSERT INTO ps_untyped (type, id, data) VALUES (?, ?, ?)', [ + 'users', + 'user-id', + json.encode({'name': 'Name', 'other': 3}) + ]); + db.executeInTx('SELECT powersync_replace_schema(?)', [ + json.encode({ + 'tables': [table] + }) + ]); + + expect(db.select('SELECT * FROM users'), [ + { + 'id': 'user-id', + 'name': 'Name', + '__data': '{"name":"Name","other":3}' + }, + ]); + + final createTable = db.select( + 'SELECT sql FROM sqlite_schema WHERE type = ? AND tbl_name = ?', + ['table', 'users'], + )[0].columnAt(0); + expect( + createTable, + 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL, __data TEXT,"name" text) STRICT /* ps-managed */', + ); + + final triggers = db + .select( + 'SELECT sql FROM sqlite_schema WHERE type = ? AND tbl_name = ? ORDER BY name', + ['trigger', 'users'], + ) + .map((r) => r['sql']) + .toList(); + + expect(triggers, [ + r''' +CREATE TRIGGER "users_trigger_DELETE" AFTER DELETE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN +INSERT INTO powersync_crud(op,id,type) VALUES ('DELETE', OLD.id, 'users'); +END''', + r''' +CREATE TRIGGER "users_trigger_INSERT" AFTER INSERT ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN +INSERT INTO powersync_crud(op,id,type,data) VALUES ('PUT', NEW.id, 'users', json(powersync_diff('{}', json_object('name', powersync_strip_subtype(NEW."name"))))); +END''', + r''' +CREATE TRIGGER "users_trigger_UPDATE" AFTER UPDATE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN +SELECT CASE WHEN (OLD.id != NEW.id) THEN RAISE (FAIL, 'Cannot update id') END; +UPDATE users SET __data = json_object('name', powersync_strip_subtype(NEW."name")) WHERE id = NEW.id; +INSERT INTO powersync_crud(op,id,type,data,options) VALUES ('PATCH', NEW.id, 'users', json(powersync_diff(json_object('name', powersync_strip_subtype(OLD."name")), json_object('name', powersync_strip_subtype(NEW."name")))), 0); +END''' + ]); + }); + }); }); } From d60d9800bd586dc8496331310ffbf05935d0302f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 8 Sep 2026 13:46:47 +0200 Subject: [PATCH 02/19] Prepare sync local support --- crates/core/src/schema/common.rs | 116 +++++++++++++++++++++++- crates/core/src/schema/raw_table.rs | 126 +++------------------------ crates/core/src/schema/table_info.rs | 2 + crates/core/src/sync/sync_local.rs | 22 ++++- 4 files changed, 145 insertions(+), 121 deletions(-) diff --git a/crates/core/src/schema/common.rs b/crates/core/src/schema/common.rs index 2e47d25..846ce6e 100644 --- a/crates/core/src/schema/common.rs +++ b/crates/core/src/schema/common.rs @@ -1,10 +1,18 @@ -use core::slice; +use core::{fmt::Write, slice}; -use alloc::{string::String, vec::Vec}; +use alloc::{ + string::{String, ToString}, + vec, + vec::Vec, +}; use serde::Deserialize; -use crate::schema::{ - Column, CommonTableOptions, RawTable, Table, raw_table::InferredTableStructure, +use crate::{ + schema::{ + Column, CommonTableOptions, PendingStatement, PendingStatementValue, RawTable, Table, + raw_table::InferredTableStructure, + }, + utils::SqlBuffer, }; /// Utility to wrap both PowerSync-managed JSON tables and raw tables (with their schema snapshot @@ -57,6 +65,57 @@ impl<'a> SchemaTable<'a> { } => SchemaTableColumnIterator::Raw(schema.columns.iter()), } } + + /// Generates a statement of the form `INSERT INTO $tbl ($cols) VALUES (?, ...) ON CONFLICT (id) + /// DO UPDATE SET ...` for the sync client. + pub fn infer_put_stmt(&self, table_name: &str) -> PendingStatement { + let mut buffer = SqlBuffer::new(); + let mut params = vec![]; + + buffer.push_str("INSERT INTO "); + let _ = buffer.identifier().write_str(table_name); + buffer.push_str(" (id"); + for column in self.column_names() { + buffer.comma(); + let _ = buffer.identifier().write_str(column); + } + buffer.push_str(") VALUES (?1"); + params.push(PendingStatementValue::Id); + for (i, column) in self.column_names().enumerate() { + buffer.comma(); + let _ = write!(&mut buffer, "?{}", i + 2); + params.push(PendingStatementValue::Column(column.to_string())); + } + buffer.push_str(") ON CONFLICT (id) DO UPDATE SET "); + let mut do_update = buffer.comma_separated(); + // Generated an "x" = ? for all synced columns to update them without affecting local-only + // columns. + for (i, column) in self.column_names().enumerate() { + let entry = do_update.element(); + let _ = entry.identifier().write_str(column); + let _ = write!(entry, " = ?{}", i + 2); + } + + PendingStatement { + sql: buffer.sql, + params, + named_parameters_index: None, + } + } + + /// Generates a statement of the form `DELETE FROM $tbl WHERE id = ?` for the sync client. + pub fn infer_delete_stmt(&self, table_name: &str) -> PendingStatement { + let mut buffer = SqlBuffer::new(); + buffer.push_str("DELETE FROM "); + let _ = buffer.identifier().write_str(table_name); + buffer.push_str(" WHERE id = ?"); + + PendingStatement { + sql: buffer.sql, + params: vec![PendingStatementValue::Id], + named_parameters_index: None, + } + } } impl<'a> From<&'a Table> for SchemaTable<'a> { @@ -118,3 +177,52 @@ impl<'de> Deserialize<'de> for ColumnFilter { Ok(Self::from(Vec::::deserialize(deserializer)?)) } } +#[cfg(test)] +mod test { + use alloc::{string::ToString, vec}; + use core::assert_matches; + + use crate::schema::{ + PendingStatementValue, RawTable, SchemaTable, raw_table::InferredTableStructure, + table_info::RawTableSchema, + }; + + #[test] + fn infer_sync_statements() { + let raw_table = RawTable { + name: "users".to_string(), + schema: RawTableSchema::default(), + put: None, + delete: None, + clear: None, + }; + let structure = InferredTableStructure { + columns: vec!["foo".to_string(), "bar".to_string()], + }; + let schema_table = SchemaTable::Raw { + definition: &raw_table, + schema: &structure, + }; + + let put = schema_table.infer_put_stmt("tbl"); + assert_eq!( + put.sql, + r#"INSERT INTO "tbl" (id, "foo", "bar") VALUES (?1, ?2, ?3) ON CONFLICT (id) DO UPDATE SET "foo" = ?2, "bar" = ?3"# + ); + assert_eq!(put.params.len(), 3); + assert_matches!(put.params[0], PendingStatementValue::Id); + assert_matches!( + put.params[1], + PendingStatementValue::Column(ref name) if name == "foo" + ); + assert_matches!( + put.params[2], + PendingStatementValue::Column(ref name) if name == "bar" + ); + + let delete = schema_table.infer_delete_stmt("tbl"); + assert_eq!(delete.sql, r#"DELETE FROM "tbl" WHERE id = ?"#); + assert_eq!(delete.params.len(), 1); + assert_matches!(delete.params[0], PendingStatementValue::Id); + } +} diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index 9d9cd3a..878b23f 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -15,13 +15,12 @@ use powersync_sqlite_nostd::Destructor; use crate::{ error::{PowerSyncError, Result}, - schema::{ColumnFilter, PendingStatement, PendingStatementValue, RawTable, SchemaTable}, + schema::{ColumnFilter, PendingStatement, RawTable, SchemaTable}, utils::{InsertIntoCrud, SqlBuffer, WriteType, database::Database}, views::table_columns_to_json_object, }; pub struct InferredTableStructure { - pub name: String, pub columns: Vec, } @@ -59,61 +58,7 @@ impl InferredTableStructure { "Table {table_name} has no id column." ))) } else { - Ok(Self { - name: table_name.to_string(), - columns, - }) - } - } - - /// Generates a statement of the form `INSERT INTO $tbl ($cols) VALUES (?, ...) ON CONFLICT (id) - /// DO UPDATE SET ...` for the sync client. - pub fn infer_put_stmt(&self) -> PendingStatement { - let mut buffer = SqlBuffer::new(); - let mut params = vec![]; - - buffer.push_str("INSERT INTO "); - let _ = buffer.identifier().write_str(&self.name); - buffer.push_str(" (id"); - for column in &self.columns { - buffer.comma(); - let _ = buffer.identifier().write_str(column); - } - buffer.push_str(") VALUES (?1"); - params.push(PendingStatementValue::Id); - for (i, column) in self.columns.iter().enumerate() { - buffer.comma(); - let _ = write!(&mut buffer, "?{}", i + 2); - params.push(PendingStatementValue::Column(column.clone())); - } - buffer.push_str(") ON CONFLICT (id) DO UPDATE SET "); - let mut do_update = buffer.comma_separated(); - // Generated an "x" = ? for all synced columns to update them without affecting local-only - // columns. - for (i, column) in self.columns.iter().enumerate() { - let entry = do_update.element(); - let _ = entry.identifier().write_str(column); - let _ = write!(entry, " = ?{}", i + 2); - } - - PendingStatement { - sql: buffer.sql, - params, - named_parameters_index: None, - } - } - - /// Generates a statement of the form `DELETE FROM $tbl WHERE id = ?` for the sync client. - pub fn infer_delete_stmt(&self) -> PendingStatement { - let mut buffer = SqlBuffer::new(); - buffer.push_str("DELETE FROM "); - let _ = buffer.identifier().write_str(&self.name); - buffer.push_str(" WHERE id = ?"); - - PendingStatement { - sql: buffer.sql, - params: vec![PendingStatementValue::Id], - named_parameters_index: None, + Ok(Self { columns }) } } } @@ -141,7 +86,7 @@ impl InferredSchemaCache { schema_version: usize, tbl: &RawTable, ) -> Result> { - self.with_entry(db, schema_version, tbl, SchemaCacheEntry::put) + self.with_entry(db, schema_version, tbl, |entry| entry.put_stmt.clone()) } pub fn infer_delete_statement( @@ -150,7 +95,7 @@ impl InferredSchemaCache { schema_version: usize, tbl: &RawTable, ) -> Result> { - self.with_entry(db, schema_version, tbl, SchemaCacheEntry::delete) + self.with_entry(db, schema_version, tbl, |entry| entry.delete_stmt.clone()) } fn with_entry( @@ -179,9 +124,8 @@ impl InferredSchemaCache { pub struct SchemaCacheEntry { schema_version: usize, - structure: InferredTableStructure, - put_stmt: Option>, - delete_stmt: Option>, + pub put_stmt: Rc, + pub delete_stmt: Rc, } impl SchemaCacheEntry { @@ -192,26 +136,17 @@ impl SchemaCacheEntry { db, &table.schema.synced_columns, )?; + let schema_table = SchemaTable::Raw { + definition: table, + schema: &structure, + }; Ok(Self { schema_version, - structure, - put_stmt: None, - delete_stmt: None, + put_stmt: Rc::new(schema_table.infer_put_stmt(local_table_name)), + delete_stmt: Rc::new(schema_table.infer_delete_stmt(local_table_name)), }) } - - fn put(&mut self) -> Rc { - self.put_stmt - .get_or_insert_with(|| Rc::new(self.structure.infer_put_stmt())) - .clone() - } - - fn delete(&mut self) -> Rc { - self.delete_stmt - .get_or_insert_with(|| Rc::new(self.structure.infer_delete_stmt())) - .clone() - } } /// Generates a `CREATE TRIGGER` statement to capture writes on raw tables and to forward them to @@ -342,40 +277,3 @@ pub fn generate_schema_table_trigger( buffer.trigger_end(); Ok(buffer.sql) } - -#[cfg(test)] -mod test { - use alloc::{string::ToString, vec}; - use core::assert_matches; - - use crate::schema::{PendingStatementValue, raw_table::InferredTableStructure}; - - #[test] - fn infer_sync_statements() { - let structure = InferredTableStructure { - name: "tbl".to_string(), - columns: vec!["foo".to_string(), "bar".to_string()], - }; - - let put = structure.infer_put_stmt(); - assert_eq!( - put.sql, - r#"INSERT INTO "tbl" (id, "foo", "bar") VALUES (?1, ?2, ?3) ON CONFLICT (id) DO UPDATE SET "foo" = ?2, "bar" = ?3"# - ); - assert_eq!(put.params.len(), 3); - assert_matches!(put.params[0], PendingStatementValue::Id); - assert_matches!( - put.params[1], - PendingStatementValue::Column(ref name) if name == "foo" - ); - assert_matches!( - put.params[2], - PendingStatementValue::Column(ref name) if name == "bar" - ); - - let delete = structure.infer_delete_stmt(); - assert_eq!(delete.sql, r#"DELETE FROM "tbl" WHERE id = ?"#); - assert_eq!(delete.params.len(), 1); - assert_matches!(delete.params[0], PendingStatementValue::Id); - } -} diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 67bd96d..858d330 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -78,6 +78,8 @@ impl Table { } pub fn internal_name(&self) -> String { + debug_assert!(!self.direct); + if self.local_only() { format!("ps_data_local__{:}", self.name) } else { diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index fb3c449..5ba78cd 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -11,7 +11,8 @@ use serde::ser::SerializeMap; use crate::error::{PowerSyncError, Result}; use crate::schema::inspection::ExistingTable; use crate::schema::{ - InferredSchemaCache, PendingStatement, PendingStatementValue, RawTable, Schema, + InferredSchemaCache, PendingStatement, PendingStatementValue, RawTable, Schema, SchemaTable, + Table, }; use crate::state::DatabaseState; use crate::sync::BucketPriority; @@ -124,7 +125,6 @@ WHERE target.key = '{TARGET_CHECKPOINT_REQUEST_ID_KEY}' "expected oplog data to be an object", ) })?; - let rest = stmt.render_rest_object(json_object)?; stmt.bind_for_put(id, data, Some(json_object), rest.as_ref())?; stmt.exec(type_name, id, Some(&data))?; @@ -340,6 +340,15 @@ impl<'a> ParsedDatabaseSchema<'a> { } fn add_from_schema(&mut self, schema: &'a Schema) { + for regular in &schema.tables { + if regular.direct { + self.tables.insert( + regular.name.clone(), + ParsedSchemaTable::new(TableDefinition::Direct(regular)), + ); + } + } + for raw in &schema.raw_tables { self.tables.insert( raw.name.clone(), @@ -420,6 +429,9 @@ impl<'a> ParsedSchemaTable<'a> { named_parameters_index: None, }) } + TableDefinition::Direct(table) => { + Rc::new(SchemaTable::Json(table).infer_put_stmt(&table.name)) + } }) }) } @@ -448,6 +460,9 @@ impl<'a> ParsedSchemaTable<'a> { named_parameters_index: None, }) } + TableDefinition::Direct(table) => { + Rc::new(SchemaTable::Json(table).infer_delete_stmt(&table.name)) + } }) }) } @@ -456,12 +471,13 @@ impl<'a> ParsedSchemaTable<'a> { enum TableDefinition<'a> { Raw(&'a RawTable), JsonView { local_table: String }, + Direct(&'a Table), } struct PreparedPendingStatement { stmt: Statement, - definition: Rc, needs_parsed_json: bool, + definition: Rc, } impl PreparedPendingStatement { From 7c6b236e47e7afdd7c087a7a33e8e348cfaf1663 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 8 Sep 2026 18:03:00 +0200 Subject: [PATCH 03/19] Simple sync test --- crates/core/src/schema/common.rs | 23 ++++++++++-- crates/core/src/schema/table_info.rs | 3 +- dart/test/sync_test.dart | 55 ++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/crates/core/src/schema/common.rs b/crates/core/src/schema/common.rs index 846ce6e..7165859 100644 --- a/crates/core/src/schema/common.rs +++ b/crates/core/src/schema/common.rs @@ -71,29 +71,46 @@ impl<'a> SchemaTable<'a> { pub fn infer_put_stmt(&self, table_name: &str) -> PendingStatement { let mut buffer = SqlBuffer::new(); let mut params = vec![]; + let data_column = self.data_column(); buffer.push_str("INSERT INTO "); let _ = buffer.identifier().write_str(table_name); buffer.push_str(" (id"); + if let Some(data_column) = data_column { + let _ = write!(&mut buffer, ", {data_column}"); + } + for column in self.column_names() { buffer.comma(); let _ = buffer.identifier().write_str(column); } buffer.push_str(") VALUES (?1"); params.push(PendingStatementValue::Id); + if data_column.is_some() { + params.push(PendingStatementValue::Row); + buffer.push_str(", ?2"); + } + + let data_start_index = if data_column.is_some() { 3 } else { 2 }; for (i, column) in self.column_names().enumerate() { buffer.comma(); - let _ = write!(&mut buffer, "?{}", i + 2); + let _ = write!(&mut buffer, "?{}", i + data_start_index); params.push(PendingStatementValue::Column(column.to_string())); } buffer.push_str(") ON CONFLICT (id) DO UPDATE SET "); let mut do_update = buffer.comma_separated(); - // Generated an "x" = ? for all synced columns to update them without affecting local-only + + if let Some(data_column) = data_column { + let entry = do_update.element(); + let _ = write!(entry, "{data_column} = ?2"); + } + + // Generate an "x" = ? for all synced columns to update them without affecting local-only // columns. for (i, column) in self.column_names().enumerate() { let entry = do_update.element(); let _ = entry.identifier().write_str(column); - let _ = write!(entry, " = ?{}", i + 2); + let _ = write!(entry, " = ?{}", i + data_start_index); } PendingStatement { diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 858d330..879dab3 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -22,6 +22,7 @@ pub struct Table { pub indexes: Vec, #[serde(flatten)] pub options: CommonTableOptions, + #[serde(default)] pub direct: bool, } @@ -78,8 +79,6 @@ impl Table { } pub fn internal_name(&self) -> String { - debug_assert!(!self.direct); - if self.local_only() { format!("ps_data_local__{:}", self.name) } else { diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index b7c969b..4aaf199 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -2180,6 +2180,61 @@ CREATE TRIGGER users_ref_delete }); }); + group('direct tables', () { + test('smoke test', () { + final schema = { + 'tables': [ + { + 'name': 'users', + 'columns': [ + {'name': 'name', 'type': 'text'} + ], + 'direct': true, + } + ] + }; + + db.executeInTx( + 'SELECT powersync_replace_schema(?)', [json.encode(schema)]); + invokeControl('start', json.encode({'schema': schema})); + + // Insert + pushCheckpoint(buckets: [bucketDescription('a')]); + pushSyncData( + 'a', + '1', + 'my_user', + 'PUT', + {'name': 'First user'}, + objectType: 'users', + ); + pushCheckpointComplete(); + + final users = db.select('SELECT * FROM users;'); + expect(users, [ + { + 'id': 'my_user', + 'name': 'First user', + '__data': '{"name":"First user"}' + } + ]); + + // Delete + pushCheckpoint(buckets: [bucketDescription('a')]); + pushSyncData( + 'a', + '1', + 'my_user', + 'REMOVE', + null, + objectType: 'users', + ); + pushCheckpointComplete(); + + expect(db.select('SELECT * FROM users'), isEmpty); + }); + }); + test('can close database while iteration is active', () { // The sync client caches prepared statements, we need to ensure those are // freed when we close the connection since SQLite would keep files open From 837402e91e2f159752567fc7a6332cf303a98245 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Sep 2026 13:18:56 +0200 Subject: [PATCH 04/19] Allow removing tables --- crates/core/src/schema/common.rs | 43 +++++++++++++--------------- crates/core/src/schema/inspection.rs | 34 +++++++++++++++------- crates/core/src/schema/management.rs | 13 ++++----- crates/core/src/schema/raw_table.rs | 21 +++++++------- crates/core/src/schema/table_info.rs | 6 +++- crates/core/src/sync/sync_local.rs | 2 +- dart/test/schema_test.dart | 19 +++++++++++- 7 files changed, 84 insertions(+), 54 deletions(-) diff --git a/crates/core/src/schema/common.rs b/crates/core/src/schema/common.rs index 7165859..3600b22 100644 --- a/crates/core/src/schema/common.rs +++ b/crates/core/src/schema/common.rs @@ -1,4 +1,4 @@ -use core::{fmt::Write, slice}; +use core::fmt::Write; use alloc::{ string::{String, ToString}, @@ -55,17 +55,21 @@ impl<'a> SchemaTable<'a> { } } - /// Iterates over defined column names in this table (not including the `id` column). - pub fn column_names(&self) -> impl Iterator { + pub fn columns(&self) -> &'a [Column] { match self { - Self::Json(table) => SchemaTableColumnIterator::Json(table.columns.iter()), + Self::Json(table) => &table.columns, Self::Raw { definition: _, schema, - } => SchemaTableColumnIterator::Raw(schema.columns.iter()), + } => &schema.columns, } } + /// Iterates over defined column names in this table (not including the `id` column). + pub fn column_names(&self) -> impl Iterator { + self.columns().iter().map(|c| &*c.name) + } + /// Generates a statement of the form `INSERT INTO $tbl ($cols) VALUES (?, ...) ON CONFLICT (id) /// DO UPDATE SET ...` for the sync client. pub fn infer_put_stmt(&self, table_name: &str) -> PendingStatement { @@ -141,22 +145,6 @@ impl<'a> From<&'a Table> for SchemaTable<'a> { } } -enum SchemaTableColumnIterator<'a> { - Json(slice::Iter<'a, Column>), - Raw(slice::Iter<'a, String>), -} - -impl<'a> Iterator for SchemaTableColumnIterator<'a> { - type Item = &'a str; - - fn next(&mut self) -> Option { - Some(match self { - Self::Json(iter) => &iter.next()?.name, - Self::Raw(iter) => iter.next()?.as_ref(), - }) - } -} - #[derive(Default)] pub struct ColumnFilter { sorted_names: Vec, @@ -200,7 +188,7 @@ mod test { use core::assert_matches; use crate::schema::{ - PendingStatementValue, RawTable, SchemaTable, raw_table::InferredTableStructure, + Column, PendingStatementValue, RawTable, SchemaTable, raw_table::InferredTableStructure, table_info::RawTableSchema, }; @@ -214,7 +202,16 @@ mod test { clear: None, }; let structure = InferredTableStructure { - columns: vec!["foo".to_string(), "bar".to_string()], + columns: vec![ + Column { + name: "foo".to_string(), + type_name: "TEXT".to_string(), + }, + Column { + name: "bar".to_string(), + type_name: "TEXT".to_string(), + }, + ], }; let schema_table = SchemaTable::Raw { definition: &raw_table, diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 1cc39fb..fc43d64 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -3,6 +3,7 @@ use alloc::{format, vec}; use alloc::{string::String, vec::Vec}; use crate::error::Result; +use crate::schema::raw_table::InferredTableStructure; use crate::utils::SqlBuffer; use crate::utils::database::Database; @@ -87,28 +88,39 @@ pub struct ExistingTable { pub name: String, pub internal_name: String, pub local_only: bool, + pub direct: Option, } impl ExistingTable { pub fn list(db: Database) -> Result> { let mut results = vec![]; - let stmt = db.prepare_v2( - " -SELECT name FROM sqlite_master WHERE type = 'table' AND name GLOB 'ps_data_*'; - ", - )?; + let stmt = db.prepare_v2("SELECT name, sql FROM sqlite_master WHERE type = 'table';")?; while stmt.step()? { let internal_name = stmt.column_text(0)?; - let Some((name, local_only)) = Self::external_name(internal_name) else { + let Ok(sql) = stmt.column_text(1) else { continue; }; - results.push(ExistingTable { - internal_name: internal_name.to_owned(), - name: name.to_owned(), - local_only: local_only, - }); + if let Some((name, local_only)) = Self::external_name(internal_name) { + results.push(ExistingTable { + internal_name: internal_name.to_owned(), + name: name.to_owned(), + local_only: local_only, + direct: None, + }); + } else if sql.contains("/* ps-managed */") { + results.push(ExistingTable { + internal_name: internal_name.to_owned(), + name: internal_name.to_owned(), + local_only: false, + direct: Some(InferredTableStructure::read_from_database( + internal_name, + db, + &None, + )?), + }); + } } Ok(results) diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 20e742a..3c71717 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -16,7 +16,7 @@ use sqlite::{Connection, ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; use crate::schema::inspection::{ExistingTable, ExistingView}; -use crate::schema::table_info::Index; +use crate::schema::table_info::{Index, data_column_name}; use crate::state::DatabaseState; use crate::utils::database::Database; use crate::utils::{SqlBuffer, verify_in_transaction}; @@ -65,17 +65,15 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { ); if table.direct { + create_table.push_str("/* ps-managed */"); + for column in &table.columns { create_table.push_char(','); let _ = create_table.identifier().write_str(&column.name); let _ = write!(&mut create_table, " {}", column.type_name); } - - create_table.push_str(") STRICT /* ps-managed */;"); - } else { - create_table.push_str(");"); } - + create_table.push_str(");"); db.exec_safe_str(&create_table.sql)?; if !table.local_only() { @@ -90,7 +88,8 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { if !remaining.local_only { db.exec_text( &format!( - "INSERT INTO ps_untyped(type, id, data) SELECT ?, id, data FROM {:}", + "INSERT INTO ps_untyped(type, id, data) SELECT ?, id, {} FROM {:}", + data_column_name(remaining.direct.is_some()), SqlBuffer::quote_identifier(&remaining.internal_name) ), &remaining.name, diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index 878b23f..4168dfd 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -4,24 +4,20 @@ use core::{ }; use alloc::{ - collections::btree_map::BTreeMap, - format, - rc::Rc, - string::{String, ToString}, - vec, + borrow::ToOwned, collections::btree_map::BTreeMap, format, rc::Rc, string::String, vec, vec::Vec, }; use powersync_sqlite_nostd::Destructor; use crate::{ error::{PowerSyncError, Result}, - schema::{ColumnFilter, PendingStatement, RawTable, SchemaTable}, + schema::{Column, ColumnFilter, PendingStatement, RawTable, SchemaTable}, utils::{InsertIntoCrud, SqlBuffer, WriteType, database::Database}, views::table_columns_to_json_object, }; pub struct InferredTableStructure { - pub columns: Vec, + pub columns: Vec, } impl InferredTableStructure { @@ -30,7 +26,7 @@ impl InferredTableStructure { db: Database, synced_columns: &Option, ) -> Result { - let stmt = db.prepare_v2("select name from pragma_table_info(?)")?; + let stmt = db.prepare_v2("select name, type from pragma_table_info(?)")?; stmt.bind_text(1, table_name, Destructor::STATIC)?; let mut has_id_column = false; @@ -38,6 +34,8 @@ impl InferredTableStructure { while stmt.step()? { let name = stmt.column_text(0)?; + let column_type = stmt.column_text(1)?; + if name == "id" { has_id_column = true; } else if let Some(filter) = synced_columns @@ -45,7 +43,10 @@ impl InferredTableStructure { { // This column isn't part of the synced columns, skip. } else { - columns.push(name.to_string()); + columns.push(Column { + name: name.to_owned(), + type_name: column_type.to_owned(), + }); } } @@ -245,7 +246,7 @@ pub fn generate_schema_table_trigger( write!(f, ", {json_fragment_new}))") }); - if write == WriteType::Update + if write != WriteType::Delete && let Some(data_column) = table.data_column() { // If the table has a __data column storing the full JSON row, we also need to update diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 879dab3..f4d3f0e 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -125,7 +125,7 @@ impl Table { } pub fn data_column_name(&self) -> &'static str { - if self.direct { "__data" } else { "data" } + data_column_name(self.direct) } pub fn generate_direct_trigger(&self, write: WriteType) -> Result { @@ -140,6 +140,10 @@ impl Table { } } +pub fn data_column_name(is_direct: bool) -> &'static str { + if is_direct { "__data" } else { "data" } +} + impl RawTable { pub fn require_table_name(&self) -> Result<&str, PowerSyncError> { let Some(local_table_name) = self.schema.table_name.as_ref() else { diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index 5ba78cd..fe68f72 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -360,7 +360,7 @@ impl<'a> ParsedDatabaseSchema<'a> { fn add_from_db(&mut self, db: Database) -> Result<()> { let tables = ExistingTable::list(db)?; for table in tables { - if !table.local_only { + if !table.local_only && !self.tables.contains_key(&table.name) { let visible_name = table.name; self.tables.insert( diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index d925a4a..a585acf 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -361,7 +361,7 @@ END''', )[0].columnAt(0); expect( createTable, - 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL, __data TEXT,"name" text) STRICT /* ps-managed */', + 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL, __data TEXT/* ps-managed */,"name" text)', ); final triggers = db @@ -389,6 +389,23 @@ INSERT INTO powersync_crud(op,id,type,data,options) VALUES ('PATCH', NEW.id, 'us END''' ]); }); + + test('remove from schema', () { + db.executeInTx('SELECT powersync_replace_schema(?)', [ + json.encode({ + 'tables': [table] + }) + ]); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + db.executeInTx('SELECT powersync_replace_schema(?)', [ + json.encode({'tables': []}) + ]); + + expect(db.select('SELECT * FROM ps_untyped'), [ + {'type': 'users', 'id': 'id', 'data': '{"name":"name"}'} + ]); + }); }); }); } From a1be5e623adc38b3c5bec181b24601699f683b18 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Sep 2026 13:33:20 +0200 Subject: [PATCH 05/19] Support local-only direct tables --- crates/core/src/schema/inspection.rs | 6 +++- crates/core/src/schema/management.rs | 3 +- crates/core/src/schema/raw_table.rs | 47 +++++++++++++++++----------- crates/core/src/sync/sync_local.rs | 6 ++-- dart/test/schema_test.dart | 43 +++++++++++++++---------- dart/test/sync_test.dart | 33 ++++++++++++++++--- 6 files changed, 96 insertions(+), 42 deletions(-) diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index fc43d64..902320d 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -93,6 +93,10 @@ pub struct ExistingTable { impl ExistingTable { pub fn list(db: Database) -> Result> { + Self::list_filtered(db, false) + } + + pub fn list_filtered(db: Database, ignore_direct: bool) -> Result> { let mut results = vec![]; let stmt = db.prepare_v2("SELECT name, sql FROM sqlite_master WHERE type = 'table';")?; @@ -109,7 +113,7 @@ impl ExistingTable { local_only: local_only, direct: None, }); - } else if sql.contains("/* ps-managed */") { + } else if sql.contains("/* ps-managed */") && !ignore_direct { results.push(ExistingTable { internal_name: internal_name.to_owned(), name: internal_name.to_owned(), diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 3c71717..24a841d 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -39,11 +39,12 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { for table in &schema.tables { if let Some(existing) = existing_tables.remove(&*table.name) { - if existing.local_only != table.local_only() { + if !table.direct && existing.local_only != table.local_only() { // Migrating between local-only and synced tables. This works by deleting // existing and re-creating the table from scratch. We can re-create first and // delete the old table afterwards because they have a different name // (local-only tables have a ps_data_local prefix). + // Direct tables are the same whether they're local or not. // To delete the old existing table in the end. existing_tables.insert(&existing.name, existing); diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index 4168dfd..54dbdec 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -209,12 +209,14 @@ pub fn generate_schema_table_trigger( } buffer.push_str(" BEGIN\n"); + let flags = table.common_options().flags; + let mut has_stmt = false; - if table.common_options().flags.insert_only() { + if flags.insert_only() { if write != WriteType::Insert { // Prevent illegal writes to a table marked as insert-only by raising errors here. buffer.push_str("SELECT RAISE(FAIL, 'Unexpected update on insert-only table');\n"); - } else { + } else if !flags.local_only() { // Insert-only tables use manual CRUD writes so they don't block incoming data. let fragment = table_columns_to_json_object("NEW", &table)?; buffer.powersync_crud_manual_put(table.name(), &fragment); @@ -255,24 +257,33 @@ pub fn generate_schema_table_trigger( &mut buffer, "UPDATE {local_table_name} SET {data_column} = {json_fragment_new} WHERE id = NEW.id;\n" ); + + has_stmt = true; } - buffer.insert_into_powersync_crud(InsertIntoCrud { - op: write, - table: &table, - id_expr: if write == WriteType::Delete { - "OLD.id" - } else { - "NEW.id" - }, - type_name: table.name(), - data: match write { - // There is no data for deleted rows. - WriteType::Delete => None, - _ => Some(&write_data), - }, - metadata: None::<&'static str>, - })?; + if !flags.local_only() { + has_stmt = true; + buffer.insert_into_powersync_crud(InsertIntoCrud { + op: write, + table: &table, + id_expr: if write == WriteType::Delete { + "OLD.id" + } else { + "NEW.id" + }, + type_name: table.name(), + data: match write { + // There is no data for deleted rows. + WriteType::Delete => None, + _ => Some(&write_data), + }, + metadata: None::<&'static str>, + })?; + } + } + + if !has_stmt { + return Ok(Default::default()); } buffer.trigger_end(); diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index fe68f72..24f6b19 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -341,7 +341,7 @@ impl<'a> ParsedDatabaseSchema<'a> { fn add_from_schema(&mut self, schema: &'a Schema) { for regular in &schema.tables { - if regular.direct { + if regular.direct && !regular.local_only() { self.tables.insert( regular.name.clone(), ParsedSchemaTable::new(TableDefinition::Direct(regular)), @@ -358,7 +358,9 @@ impl<'a> ParsedDatabaseSchema<'a> { } fn add_from_db(&mut self, db: Database) -> Result<()> { - let tables = ExistingTable::list(db)?; + // Ignore direct tables here, we can rely on them being added via add_from_schema. + // TODO: Remove this function, SDKs should always pass the used schema when they connect. + let tables = ExistingTable::list_filtered(db, true)?; for table in tables { if !table.local_only && !self.tables.contains_key(&table.name) { let visible_name = table.name; diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index a585acf..f51106f 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -324,13 +324,20 @@ END''', }); group('direct tables', () { - final table = { - 'name': 'users', - 'columns': [ - {'name': 'name', 'type': 'text'} - ], - 'direct': true, - }; + Object schema({Map additionalOptions = const {}}) { + return { + 'tables': [ + { + 'name': 'users', + 'columns': [ + {'name': 'name', 'type': 'text'} + ], + 'direct': true, + ...additionalOptions, + } + ] + }; + } test('create', () { db.executeInTx('SELECT powersync_replace_schema(?)', [ @@ -341,11 +348,8 @@ END''', 'user-id', json.encode({'name': 'Name', 'other': 3}) ]); - db.executeInTx('SELECT powersync_replace_schema(?)', [ - json.encode({ - 'tables': [table] - }) - ]); + db.executeInTx( + 'SELECT powersync_replace_schema(?)', [json.encode(schema())]); expect(db.select('SELECT * FROM users'), [ { @@ -390,12 +394,19 @@ END''' ]); }); - test('remove from schema', () { + test('local-only', () { db.executeInTx('SELECT powersync_replace_schema(?)', [ - json.encode({ - 'tables': [table] - }) + json.encode(schema(additionalOptions: {'local_only': true})) ]); + + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + expect(db.select('SELECT * FROM ps_crud'), isEmpty); + }); + + test('remove from schema', () { + db.executeInTx( + 'SELECT powersync_replace_schema(?)', [json.encode(schema())]); db.execute( 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); db.executeInTx('SELECT powersync_replace_schema(?)', [ diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 4aaf199..253ad66 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -2181,8 +2181,8 @@ CREATE TRIGGER users_ref_delete }); group('direct tables', () { - test('smoke test', () { - final schema = { + Object schema({Map additionalOptions = const {}}) { + return { 'tables': [ { 'name': 'users', @@ -2190,13 +2190,16 @@ CREATE TRIGGER users_ref_delete {'name': 'name', 'type': 'text'} ], 'direct': true, + ...additionalOptions, } ] }; + } + test('smoke test', () { db.executeInTx( - 'SELECT powersync_replace_schema(?)', [json.encode(schema)]); - invokeControl('start', json.encode({'schema': schema})); + 'SELECT powersync_replace_schema(?)', [json.encode(schema())]); + invokeControl('start', json.encode({'schema': schema()})); // Insert pushCheckpoint(buckets: [bucketDescription('a')]); @@ -2233,6 +2236,28 @@ CREATE TRIGGER users_ref_delete expect(db.select('SELECT * FROM users'), isEmpty); }); + + test('local only', () { + final localOnlySchema = schema(additionalOptions: {'local_only': true}); + + db.executeInTx( + 'SELECT powersync_replace_schema(?)', [json.encode(localOnlySchema)]); + invokeControl('start', json.encode({'schema': localOnlySchema})); + + // Insert + pushCheckpoint(buckets: [bucketDescription('a')]); + pushSyncData( + 'a', + '1', + 'my_user', + 'PUT', + {'name': 'First user'}, + objectType: 'users', + ); + pushCheckpointComplete(); + + expect(db.select('SELECT * FROM ps_untyped'), hasLength(1)); + }); }); test('can close database while iteration is active', () { From 6060ec5ed4c7a6bda66398f65865d7145c1282ae Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Sep 2026 14:55:12 +0200 Subject: [PATCH 06/19] Use _rest column pattern --- crates/core/src/json_util.rs | 6 ++- crates/core/src/schema/common.rs | 36 ++++++++-------- crates/core/src/schema/inspection.rs | 33 ++++++++++++++- crates/core/src/schema/management.rs | 27 +++++------- crates/core/src/schema/raw_table.rs | 26 ++++-------- crates/core/src/schema/table_info.rs | 61 ++++++++++++++++------------ crates/core/src/sync/mod.rs | 1 + crates/core/src/sync/sync_local.rs | 2 +- crates/core/src/utils/sql_buffer.rs | 4 +- crates/core/src/views.rs | 27 +++++------- dart/test/schema_test.dart | 10 +++-- dart/test/sync_test.dart | 2 +- 12 files changed, 130 insertions(+), 105 deletions(-) diff --git a/crates/core/src/json_util.rs b/crates/core/src/json_util.rs index 2d3fa04..6585181 100644 --- a/crates/core/src/json_util.rs +++ b/crates/core/src/json_util.rs @@ -6,8 +6,8 @@ use core::ffi::c_int; use crate::constants::SUBTYPE_JSON; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; -use powersync_sqlite_nostd as sqlite; use powersync_sqlite_nostd::bindings::{SQLITE_RESULT_SUBTYPE, SQLITE_SUBTYPE}; +use powersync_sqlite_nostd::{self as sqlite, ColumnType}; use powersync_sqlite_nostd::{Connection, Context, Value}; use sqlite::ResultCode; @@ -38,6 +38,10 @@ fn powersync_json_merge_impl( } let mut result = String::from("{"); for arg in args { + if arg.value_type() == ColumnType::Null { + continue; + } + let chunk = arg.text(); if chunk.is_empty() || !chunk.starts_with('{') || !chunk.ends_with('}') { return Err(PowerSyncError::argument_error("Expected json object")); diff --git a/crates/core/src/schema/common.rs b/crates/core/src/schema/common.rs index 3600b22..c0d93d5 100644 --- a/crates/core/src/schema/common.rs +++ b/crates/core/src/schema/common.rs @@ -10,7 +10,7 @@ use serde::Deserialize; use crate::{ schema::{ Column, CommonTableOptions, PendingStatement, PendingStatementValue, RawTable, Table, - raw_table::InferredTableStructure, + raw_table::InferredTableStructure, table_info::RestColumnIndex, }, utils::SqlBuffer, }; @@ -37,14 +37,6 @@ impl<'a> SchemaTable<'a> { } } - pub fn data_column(&self) -> Option<&'static str> { - if let SchemaTable::Json(table) = self { - Some(table.data_column_name()) - } else { - None - } - } - pub fn common_options(&self) -> &CommonTableOptions { match self { Self::Json(table) => &table.options, @@ -75,13 +67,16 @@ impl<'a> SchemaTable<'a> { pub fn infer_put_stmt(&self, table_name: &str) -> PendingStatement { let mut buffer = SqlBuffer::new(); let mut params = vec![]; - let data_column = self.data_column(); + let mut rest = match self { + SchemaTable::Json(_) => Some(("_rest", RestColumnIndex::default())), + SchemaTable::Raw { .. } => None, + }; buffer.push_str("INSERT INTO "); let _ = buffer.identifier().write_str(table_name); buffer.push_str(" (id"); - if let Some(data_column) = data_column { - let _ = write!(&mut buffer, ", {data_column}"); + if let Some((column, _)) = rest { + let _ = write!(&mut buffer, ", {column}"); } for column in self.column_names() { @@ -90,23 +85,28 @@ impl<'a> SchemaTable<'a> { } buffer.push_str(") VALUES (?1"); params.push(PendingStatementValue::Id); - if data_column.is_some() { - params.push(PendingStatementValue::Row); + if let Some((_, ref mut rest_index)) = rest { + params.push(PendingStatementValue::Rest); buffer.push_str(", ?2"); + rest_index.rest_parameter_positions.push(1); // this is zero-indexed } - let data_start_index = if data_column.is_some() { 3 } else { 2 }; + let data_start_index = if rest.is_some() { 3 } else { 2 }; for (i, column) in self.column_names().enumerate() { buffer.comma(); let _ = write!(&mut buffer, "?{}", i + data_start_index); params.push(PendingStatementValue::Column(column.to_string())); + + if let Some((_, ref mut index)) = rest { + index.named_parameters.insert(column.to_string()); + } } buffer.push_str(") ON CONFLICT (id) DO UPDATE SET "); let mut do_update = buffer.comma_separated(); - if let Some(data_column) = data_column { + if let Some((column, _)) = rest { let entry = do_update.element(); - let _ = write!(entry, "{data_column} = ?2"); + let _ = write!(entry, "{column} = ?2"); } // Generate an "x" = ? for all synced columns to update them without affecting local-only @@ -120,7 +120,7 @@ impl<'a> SchemaTable<'a> { PendingStatement { sql: buffer.sql, params, - named_parameters_index: None, + named_parameters_index: rest.map(|e| e.1), } } diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 902320d..181c60d 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -1,3 +1,5 @@ +use core::fmt::Write; + use alloc::borrow::ToOwned; use alloc::{format, vec}; use alloc::{string::String, vec::Vec}; @@ -6,6 +8,7 @@ use crate::error::Result; use crate::schema::raw_table::InferredTableStructure; use crate::utils::SqlBuffer; use crate::utils::database::Database; +use crate::views::table_columns_to_json_object; /// An existing PowerSync-managed view that was found in the schema. #[derive(PartialEq)] @@ -113,15 +116,16 @@ impl ExistingTable { local_only: local_only, direct: None, }); - } else if sql.contains("/* ps-managed */") && !ignore_direct { + } else if sql.contains("/* ps-managed") && !ignore_direct { results.push(ExistingTable { internal_name: internal_name.to_owned(), name: internal_name.to_owned(), - local_only: false, + local_only: sql.contains("local-only"), direct: Some(InferredTableStructure::read_from_database( internal_name, db, &None, + true, )?), }); } @@ -145,4 +149,29 @@ impl ExistingTable { None } } + + pub fn move_into_ps_untyped(&self, db: Database) -> Result<()> { + if self.local_only { + return Ok(()); + } + + let mut buffer = SqlBuffer::new(); + buffer.push_str("INSERT INTO ps_untyped(type, id, data) SELECT ?, id, "); + + if let Some(ref schema) = self.direct { + buffer.push_str("powersync_json_merge("); + buffer.push_str(&table_columns_to_json_object( + &self.internal_name, + &schema.columns, + )?); + buffer.push_str(", _rest)"); + } else { + buffer.push_str("data"); + } + + buffer.push_str(" FROM "); + let _ = buffer.identifier().write_str(&self.internal_name); + + db.exec_text(&buffer.sql, &self.name) + } } diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 24a841d..03e3eb5 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -16,7 +16,7 @@ use sqlite::{Connection, ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; use crate::schema::inspection::{ExistingTable, ExistingView}; -use crate::schema::table_info::{Index, data_column_name}; +use crate::schema::table_info::Index; use crate::state::DatabaseState; use crate::utils::database::Database; use crate::utils::{SqlBuffer, verify_in_transaction}; @@ -55,24 +55,26 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { } // New table. - let data_column = table.data_column_name(); let mut create_table = SqlBuffer::default(); create_table.push_str("CREATE TABLE "); table.write_name(&mut create_table); - _ = write!( - &mut create_table, - "(id TEXT PRIMARY KEY NOT NULL, {data_column} TEXT" - ); + _ = write!(&mut create_table, "(id TEXT PRIMARY KEY NOT NULL"); if table.direct { - create_table.push_str("/* ps-managed */"); + create_table.push_str(", _rest TEXT /* ps-managed "); + if table.local_only() { + create_table.push_str("local-only "); + } + create_table.push_str("*/"); for column in &table.columns { create_table.push_char(','); let _ = create_table.identifier().write_str(&column.name); let _ = write!(&mut create_table, " {}", column.type_name); } + } else { + create_table.push_str(", data TEXT"); } create_table.push_str(");"); db.exec_safe_str(&create_table.sql)?; @@ -86,16 +88,7 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { // Remaining tables need to be dropped. But first, we want to move their contents to // ps_untyped. for remaining in existing_tables.values() { - if !remaining.local_only { - db.exec_text( - &format!( - "INSERT INTO ps_untyped(type, id, data) SELECT ?, id, {} FROM {:}", - data_column_name(remaining.direct.is_some()), - SqlBuffer::quote_identifier(&remaining.internal_name) - ), - &remaining.name, - )?; - } + remaining.move_into_ps_untyped(db)?; } // We cannot have any open queries on sqlite_master at the point that we drop tables, otherwise diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index 54dbdec..57631b7 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -25,6 +25,7 @@ impl InferredTableStructure { table_name: &str, db: Database, synced_columns: &Option, + is_direct: bool, ) -> Result { let stmt = db.prepare_v2("select name, type from pragma_table_info(?)")?; stmt.bind_text(1, table_name, Destructor::STATIC)?; @@ -42,6 +43,8 @@ impl InferredTableStructure { && !filter.matches(name) { // This column isn't part of the synced columns, skip. + } else if is_direct && name == "_rest" { + // _rest column is an artifact of direct tables, skip. } else { columns.push(Column { name: name.to_owned(), @@ -136,6 +139,7 @@ impl SchemaCacheEntry { local_table_name, db, &table.schema.synced_columns, + false, )?; let schema_table = SchemaTable::Raw { definition: table, @@ -161,7 +165,7 @@ pub fn generate_raw_table_trigger( let local_table_name = table.require_table_name()?; let synced_columns = &table.schema.synced_columns; let resolved_table = - InferredTableStructure::read_from_database(local_table_name, db, synced_columns)?; + InferredTableStructure::read_from_database(local_table_name, db, synced_columns, false)?; let as_schema_table = SchemaTable::Raw { definition: table, @@ -218,18 +222,19 @@ pub fn generate_schema_table_trigger( buffer.push_str("SELECT RAISE(FAIL, 'Unexpected update on insert-only table');\n"); } else if !flags.local_only() { // Insert-only tables use manual CRUD writes so they don't block incoming data. - let fragment = table_columns_to_json_object("NEW", &table)?; + let fragment = table_columns_to_json_object("NEW", table.columns())?; buffer.powersync_crud_manual_put(table.name(), &fragment); } } else { if write == WriteType::Update { // Updates must not change the id. buffer.check_id_not_changed(); + has_stmt = true; } - let json_fragment_new = table_columns_to_json_object("NEW", &table)?; + let json_fragment_new = table_columns_to_json_object("NEW", table.columns())?; let json_fragment_old = if write == WriteType::Update { - Some(table_columns_to_json_object("OLD", &table)?) + Some(table_columns_to_json_object("OLD", table.columns())?) } else { None }; @@ -248,19 +253,6 @@ pub fn generate_schema_table_trigger( write!(f, ", {json_fragment_new}))") }); - if write != WriteType::Delete - && let Some(data_column) = table.data_column() - { - // If the table has a __data column storing the full JSON row, we also need to update - // that. - let _ = write!( - &mut buffer, - "UPDATE {local_table_name} SET {data_column} = {json_fragment_new} WHERE id = NEW.id;\n" - ); - - has_stmt = true; - } - if !flags.local_only() { has_stmt = true; buffer.insert_into_powersync_crud(InsertIntoCrud { diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index f4d3f0e..8de9984 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -4,11 +4,13 @@ use alloc::rc::Rc; use alloc::string::ToString; use alloc::vec; use alloc::{collections::btree_set::BTreeSet, format, string::String, vec::Vec}; +use powersync_sqlite_nostd::Destructor; use serde::{Deserialize, de::Visitor}; use crate::error::PowerSyncError; use crate::schema::raw_table::generate_schema_table_trigger; use crate::schema::{ColumnFilter, SchemaTable}; +use crate::sync::PreparedPendingStatement; use crate::utils::database::Database; use crate::utils::{SqlBuffer, WriteType}; @@ -87,32 +89,44 @@ impl Table { } pub fn move_from_ps_untyped(&self, db: Database) -> Result<(), PowerSyncError> { - let mut stmt = SqlBuffer::default(); let direct = self.direct; - stmt.push_str("INSERT INTO "); - self.write_name(&mut stmt); - let _ = write!(&mut stmt, "(id, {}", self.data_column_name()); + let mut delete_stmt = SqlBuffer::new(); + delete_stmt.push_str("DELETE FROM ps_untyped WHERE type = ?"); if direct { - for column in &self.columns { - stmt.push_char(','); - let _ = stmt.identifier().write_str(&column.name); - } - } - - stmt.push_str(") SELECT id, data"); - if direct { - for column in &self.columns { - stmt.push_char(','); - stmt.json_extract_and_cast("data", &column.name, &column.type_name); + // Copying into direct tables reqires extracting from JSON. This essentially replays a + // sync_local step for the table, using ps_untyped as source. + let stmt = Rc::new(SchemaTable::Json(self).infer_put_stmt(&self.name)); + let stmt = PreparedPendingStatement::prepare(db, stmt)?; + + let _ = delete_stmt.write_str(" RETURNING id, data"); + let source = db.prepare_v2(&delete_stmt.sql)?; + source.bind_text(1, &self.name, Destructor::STATIC)?; + + while source.step()? { + let id = source.column_text(0)?; + let data = source.column_text(1)?; + + let parsed: serde_json::Value = + serde_json::from_str(data).map_err(PowerSyncError::json_local_error)?; + let json_object = parsed.as_object().ok_or_else(|| { + PowerSyncError::argument_error("expected oplog data to be an object") + })?; + let rest = stmt.render_rest_object(json_object)?; + stmt.bind_for_put(id, data, Some(json_object), rest.as_ref())?; + stmt.exec(&self.name, id, Some(&data))?; } + } else { + let mut stmt = SqlBuffer::default(); + stmt.push_str("INSERT INTO "); + self.write_name(&mut stmt); + let _ = stmt.write_str(" (id, data) SELECT id, data FROM ps_untyped WHERE type = ?"); + let _ = db.exec_text(&stmt.sql, &self.name); + db.exec_text(&delete_stmt.sql, &self.name)?; } - stmt.push_str(" FROM ps_untyped WHERE type = ?"); - - db.exec_text(&stmt.sql, &self.name)?; - db.exec_text("DELETE FROM ps_untyped WHERE type = ?", &self.name) + Ok(()) } pub fn write_name(&self, buffer: &mut SqlBuffer) { @@ -124,10 +138,6 @@ impl Table { } } - pub fn data_column_name(&self) -> &'static str { - data_column_name(self.direct) - } - pub fn generate_direct_trigger(&self, write: WriteType) -> Result { debug_assert!(self.direct); generate_schema_table_trigger( @@ -140,10 +150,6 @@ impl Table { } } -pub fn data_column_name(is_direct: bool) -> &'static str { - if is_direct { "__data" } else { "data" } -} - impl RawTable { pub fn require_table_name(&self) -> Result<&str, PowerSyncError> { let Some(local_table_name) = self.schema.table_name.as_ref() else { @@ -367,6 +373,7 @@ pub struct PendingStatement { pub named_parameters_index: Option, } +#[derive(Default)] pub struct RestColumnIndex { /// All column names referenced by this statement. pub named_parameters: BTreeSet, diff --git a/crates/core/src/sync/mod.rs b/crates/core/src/sync/mod.rs index 874eb29..2c150de 100644 --- a/crates/core/src/sync/mod.rs +++ b/crates/core/src/sync/mod.rs @@ -19,6 +19,7 @@ pub use checksum::Checksum; use crate::state::DatabaseState; pub use streaming_sync::SyncClient; +pub use sync_local::PreparedPendingStatement; pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<(), ResultCode> { interface::register(db, state) diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index 24f6b19..01a3ac7 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -476,7 +476,7 @@ enum TableDefinition<'a> { Direct(&'a Table), } -struct PreparedPendingStatement { +pub struct PreparedPendingStatement { stmt: Statement, needs_parsed_json: bool, definition: Rc, diff --git a/crates/core/src/utils/sql_buffer.rs b/crates/core/src/utils/sql_buffer.rs index 6a9c97d..3c68e24 100644 --- a/crates/core/src/utils/sql_buffer.rs +++ b/crates/core/src/utils/sql_buffer.rs @@ -123,7 +123,7 @@ impl SqlBuffer { Some(include_old) => { let old_values = table_columns_to_json_object_with_filter( "OLD", - insert.table, + insert.table.columns(), include_old.column_filter(), )?; @@ -134,7 +134,7 @@ impl SqlBuffer { // only include the powersync_diff of columns matched by the filter. let filtered_new_fragment = table_columns_to_json_object_with_filter( "NEW", - insert.table, + insert.table.columns(), include_old.column_filter(), )?; diff --git a/crates/core/src/views.rs b/crates/core/src/views.rs index f5e7089..ffe2e3e 100644 --- a/crates/core/src/views.rs +++ b/crates/core/src/views.rs @@ -6,7 +6,7 @@ use core::fmt::{Write, from_fn}; use core::mem; use crate::error::{PowerSyncError, Result}; -use crate::schema::{ColumnFilter, SchemaTable, Table}; +use crate::schema::{Column, ColumnFilter, SchemaTable, Table}; use crate::utils::{InsertIntoCrud, SqlBuffer, WriteType}; pub fn powersync_view_sql(table_info: &Table) -> String { @@ -140,7 +140,7 @@ pub fn powersync_trigger_insert_sql(table_info: &Table) -> Result { sql.check_id_valid(); } - let json_fragment = table_columns_to_json_object("NEW", &as_schema_table)?; + let json_fragment = table_columns_to_json_object("NEW", &table_info.columns)?; if insert_only { // This is using the manual powersync_crud_ instead of powersync_crud because insert-only @@ -188,7 +188,6 @@ pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { let name = &table_info.name; let view_name = table_info.view_name(); let local_only = table_info.options.flags.local_only(); - let as_schema_table = SchemaTable::from(table_info); let mut sql = SqlBuffer::new(); sql.create_trigger("ps_view_update_", view_name); @@ -202,8 +201,8 @@ pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { sql.push_str("BEGIN\n"); sql.check_id_not_changed(); - let json_fragment_new = table_columns_to_json_object("NEW", &as_schema_table)?; - let json_fragment_old = table_columns_to_json_object("OLD", &as_schema_table)?; + let json_fragment_new = table_columns_to_json_object("NEW", &table_info.columns)?; + let json_fragment_old = table_columns_to_json_object("OLD", &table_info.columns)?; // UPDATE {internal_name} SET data = {json_fragment_new} WHERE id = NEW.id; sql.push_str("UPDATE "); @@ -218,7 +217,7 @@ pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { sql.insert_into_powersync_crud(InsertIntoCrud { op: WriteType::Update, id_expr: "NEW.id", - table: &as_schema_table, + table: &SchemaTable::Json(table_info), type_name: name, data: Some(&from_fn(|f| { write!( @@ -241,16 +240,13 @@ pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { /// Given a query returning column names, return a JSON object fragment for a trigger. /// /// Example output with prefix "NEW": "json_object('id', NEW.id, 'name', NEW.name, 'age', NEW.age)". -pub fn table_columns_to_json_object<'a>( - prefix: &str, - table: &'a SchemaTable<'a>, -) -> Result { - table_columns_to_json_object_with_filter(prefix, table, None) +pub fn table_columns_to_json_object(prefix: &str, columns: &[Column]) -> Result { + table_columns_to_json_object_with_filter(prefix, columns, None) } pub fn table_columns_to_json_object_with_filter<'a>( prefix: &str, - table: &'a SchemaTable<'a>, + columns: &[Column], filter: Option<&'a ColumnFilter>, ) -> Result { // floor(SQLITE_MAX_FUNCTION_ARG / 2). @@ -274,8 +270,7 @@ pub fn table_columns_to_json_object_with_filter<'a>( buffer.sql } - let mut columns = table.column_names(); - while let Some(name) = columns.next() { + for Column { name, type_name: _ } in columns { if let Some(filter) = filter && !filter.matches(name) { @@ -377,8 +372,8 @@ mod test { #[test] fn test_json_object_fragment() { - let fragment = - table_columns_to_json_object("NEW", &(&test_table()).into()).expect("should generate"); + let columns = &test_table().columns; + let fragment = table_columns_to_json_object("NEW", columns).expect("should generate"); assert_eq!( fragment, diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index f51106f..c14659e 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -355,7 +355,7 @@ END''', { 'id': 'user-id', 'name': 'Name', - '__data': '{"name":"Name","other":3}' + '_rest': '{"other":3}', }, ]); @@ -365,7 +365,7 @@ END''', )[0].columnAt(0); expect( createTable, - 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL, __data TEXT/* ps-managed */,"name" text)', + 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL, _rest TEXT /* ps-managed */,"name" text)', ); final triggers = db @@ -388,7 +388,6 @@ END''', r''' CREATE TRIGGER "users_trigger_UPDATE" AFTER UPDATE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN SELECT CASE WHEN (OLD.id != NEW.id) THEN RAISE (FAIL, 'Cannot update id') END; -UPDATE users SET __data = json_object('name', powersync_strip_subtype(NEW."name")) WHERE id = NEW.id; INSERT INTO powersync_crud(op,id,type,data,options) VALUES ('PATCH', NEW.id, 'users', json(powersync_diff(json_object('name', powersync_strip_subtype(OLD."name")), json_object('name', powersync_strip_subtype(NEW."name")))), 0); END''' ]); @@ -416,6 +415,11 @@ END''' expect(db.select('SELECT * FROM ps_untyped'), [ {'type': 'users', 'id': 'id', 'data': '{"name":"name"}'} ]); + + expect( + db.select( + 'SELECT * FROM sqlite_schema WHERE type = ?', ['trigger']), + isEmpty); }); }); }); diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 253ad66..211a205 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -2218,7 +2218,7 @@ CREATE TRIGGER users_ref_delete { 'id': 'my_user', 'name': 'First user', - '__data': '{"name":"First user"}' + '_rest': null, } ]); From 0cf8f52da17fb76b61ee32ef0ff64688d75a92f7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 10 Sep 2026 16:48:17 +0200 Subject: [PATCH 07/19] Support migrating from JSON tables --- crates/core/src/migrations.rs | 9 ++- crates/core/src/schema/inspection.rs | 6 +- crates/core/src/schema/management.rs | 105 +++++++++++++++++++-------- crates/core/src/schema/table_info.rs | 62 +++++++++++----- crates/core/src/view_admin.rs | 7 +- dart/test/schema_test.dart | 72 +++++++++++++++--- 6 files changed, 195 insertions(+), 66 deletions(-) diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index 3a8a58e..b1c72ff 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -4,7 +4,6 @@ use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; -use powersync_sqlite_nostd::Context; use powersync_sqlite_nostd::{self as sqlite, Destructor}; use serde::Serialize; use serde_json::json; @@ -15,12 +14,16 @@ use crate::fix_data::apply_v035_fix; use crate::schema::inspection::ExistingView; use crate::sync::BucketPriority; use crate::utils::database::Database; +use crate::utils::verify_in_transaction; pub const LATEST_VERSION: i32 = 14; -pub fn powersync_migrate(ctx: *mut sqlite::context, target_version: i32) -> Result<()> { - let local_db = Database::from(ctx.db_handle()); +pub fn initialize_database(db: Database) -> Result<()> { + verify_in_transaction(db)?; + powersync_migrate(db, LATEST_VERSION) +} +pub fn powersync_migrate(local_db: Database, target_version: i32) -> Result<()> { // language=SQLite local_db.exec_safe( c"\ diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 181c60d..3e76857 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -17,7 +17,7 @@ pub struct ExistingView { pub name: String, /// SQL contents of the `CREATE VIEW` statement. /// - /// This is not set for as_raw_table tables, which don't have a view. + /// This is not set for direct tables, which don't have a view. pub sql: Option, /// SQL contents of all triggers implementing deletes by forwarding to /// `ps_data` and `ps_crud`. @@ -74,6 +74,10 @@ SELECT Ok(()) } + pub fn delete_from_db(&self, db: Database) -> Result<()> { + Self::drop_by_name(db, &self.name) + } + pub fn create(&self, db: Database) -> Result<()> { if let Some(create_view) = &self.sql { Self::drop_by_name(db, &self.name)?; diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 03e3eb5..75aa052 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -15,6 +15,7 @@ use sqlite::{Connection, ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; +use crate::migrations::initialize_database; use crate::schema::inspection::{ExistingTable, ExistingView}; use crate::schema::table_info::Index; use crate::state::DatabaseState; @@ -27,7 +28,11 @@ use crate::views::{ use super::Schema; -fn update_tables(db: Database, schema: &Schema) -> Result<()> { +fn update_tables( + db: Database, + schema: &Schema, + existing_views: &mut BTreeMap<&str, &ExistingView>, +) -> Result<()> { let existing_tables = ExistingTable::list(db)?; let mut existing_tables = { let mut map = BTreeMap::new(); @@ -38,19 +43,57 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { }; for table in &schema.tables { + let mut move_data_from = None::<&str>; + if let Some(existing) = existing_tables.remove(&*table.name) { - if !table.direct && existing.local_only != table.local_only() { - // Migrating between local-only and synced tables. This works by deleting - // existing and re-creating the table from scratch. We can re-create first and - // delete the old table afterwards because they have a different name - // (local-only tables have a ps_data_local prefix). - // Direct tables are the same whether they're local or not. - - // To delete the old existing table in the end. - existing_tables.insert(&existing.name, existing); - } else { - // Compatible table exists already, nothing to do. - continue; + match (&existing.direct, table.direct) { + (None, false) => { + // JSON-based table before and now. We might have to migrate between synced and + // local-only tables. + if existing.local_only != table.local_only() { + // Migrating between local-only and synced tables. This works by deleting + // existing and re-creating the table from scratch. We can re-create first + // and delete the old table afterwards because they have a different name + // (local-only tables have a ps_data_local prefix). + + // To delete the old existing table in the end. + existing_tables.insert(&existing.name, existing); + } else { + // Compatible table exists already, nothing to do. + continue; + } + } + (None, true) => { + // When migrating from JSON-based to direct tables, there are four cases to + // consider: + // 1. Local-only to direct local-only: We copy data; delete the old table. + // 2. Local-only to synced: Delete old table, copy from ps_untyped for new. + // 3. Synced to local-only: Move old into ps_untyped; create new from scratch. + // 4. Synced to synced: Copy data; delete old table. + if existing.local_only == table.local_only() { + // Case 1 or 4. + move_data_from = Some(&existing.internal_name); + } else { + // Case 2 and 3 is the default, we'll delete the old table in the end which + // moves to ps_untyped if necessary. + } + + // To delete the existing table in the end. + existing_tables.insert(&existing.name, existing); + + // The direct table we create conflicts with the view. So delete that one first. + if let Some(old_view) = existing_views.remove(&*existing.name) { + old_view.delete_from_db(db)?; + } + } + (Some(_), false) => { + return Err(PowerSyncError::argument_error( + "Switching from direct to json-based tables is not yet implemented.", + )); + } + (Some(_), true) => { + // TODO: Consider migrations in schema tables. + } } } @@ -79,7 +122,9 @@ fn update_tables(db: Database, schema: &Schema) -> Result<()> { create_table.push_str(");"); db.exec_safe_str(&create_table.sql)?; - if !table.local_only() { + if let Some(old_json_table) = move_data_from { + table.direct_move_from_json(db, old_json_table)?; + } else if !table.local_only() { // MOVE data if any table.move_from_ps_untyped(db)?; } @@ -203,17 +248,11 @@ SELECT Ok(()) } -fn update_views(db: Database, schema: &Schema) -> Result<()> { - // First, find all existing views and index them by name. - let existing = ExistingView::list(db)?; - let mut existing = { - let mut map = BTreeMap::new(); - for entry in &existing { - map.insert(&*entry.name, entry); - } - map - }; - +fn update_views( + db: Database, + schema: &Schema, + existing: &mut BTreeMap<&str, &ExistingView>, +) -> Result<()> { for table in &schema.tables { let view_sql = if table.direct { None @@ -267,12 +306,20 @@ fn powersync_replace_schema_impl( let parsed_schema = serde_json::from_str::(schema).map_err(PowerSyncError::as_argument_error)?; - // language=SQLite - db.exec_safe(c"SELECT powersync_init()")?; + initialize_database(db)?; + + let views = ExistingView::list(db)?; + let mut existing_views = { + let mut map = BTreeMap::new(); + for entry in &views { + map.insert(&*entry.name, entry); + } + map + }; - update_tables(db, &parsed_schema)?; + update_tables(db, &parsed_schema, &mut existing_views)?; update_indexes(db, &parsed_schema)?; - update_views(db, &parsed_schema)?; + update_views(db, &parsed_schema, &mut existing_views)?; state.set_schema(parsed_schema); Ok(String::from("")) diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 8de9984..f745958 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -11,7 +11,7 @@ use crate::error::PowerSyncError; use crate::schema::raw_table::generate_schema_table_trigger; use crate::schema::{ColumnFilter, SchemaTable}; use crate::sync::PreparedPendingStatement; -use crate::utils::database::Database; +use crate::utils::database::{Database, Statement}; use crate::utils::{SqlBuffer, WriteType}; #[derive(Deserialize)] @@ -95,28 +95,11 @@ impl Table { delete_stmt.push_str("DELETE FROM ps_untyped WHERE type = ?"); if direct { - // Copying into direct tables reqires extracting from JSON. This essentially replays a - // sync_local step for the table, using ps_untyped as source. - let stmt = Rc::new(SchemaTable::Json(self).infer_put_stmt(&self.name)); - let stmt = PreparedPendingStatement::prepare(db, stmt)?; - let _ = delete_stmt.write_str(" RETURNING id, data"); let source = db.prepare_v2(&delete_stmt.sql)?; source.bind_text(1, &self.name, Destructor::STATIC)?; - while source.step()? { - let id = source.column_text(0)?; - let data = source.column_text(1)?; - - let parsed: serde_json::Value = - serde_json::from_str(data).map_err(PowerSyncError::json_local_error)?; - let json_object = parsed.as_object().ok_or_else(|| { - PowerSyncError::argument_error("expected oplog data to be an object") - })?; - let rest = stmt.render_rest_object(json_object)?; - stmt.bind_for_put(id, data, Some(json_object), rest.as_ref())?; - stmt.exec(&self.name, id, Some(&data))?; - } + self.direct_move_from_stmt(db, source)?; } else { let mut stmt = SqlBuffer::default(); stmt.push_str("INSERT INTO "); @@ -129,6 +112,47 @@ impl Table { Ok(()) } + pub fn direct_move_from_json( + &self, + db: Database, + json_table: &str, + ) -> Result<(), PowerSyncError> { + debug_assert!(self.direct); + + let mut source = SqlBuffer::new(); + source.push_str("SELECT id, data FROM "); + let _ = write!(source.identifier(), "{}", json_table); + + let source = db.prepare_v2(&source.sql)?; + self.direct_move_from_stmt(db, source) + } + + /// For direct tables, copies data from a prepared statement returning id and data. + fn direct_move_from_stmt(&self, db: Database, source: Statement) -> Result<(), PowerSyncError> { + debug_assert!(self.direct); + + // Copying into direct tables reqires extracting from JSON. This essentially replays a + // sync_local step for the table, using a custom source. + let stmt = Rc::new(SchemaTable::Json(self).infer_put_stmt(&self.name)); + let stmt = PreparedPendingStatement::prepare(db, stmt)?; + + while source.step()? { + let id = source.column_text(0)?; + let data = source.column_text(1)?; + + let parsed: serde_json::Value = + serde_json::from_str(data).map_err(PowerSyncError::json_local_error)?; + let json_object = parsed.as_object().ok_or_else(|| { + PowerSyncError::argument_error("expected oplog data to be an object") + })?; + let rest = stmt.render_rest_object(json_object)?; + stmt.bind_for_put(id, data, Some(json_object), rest.as_ref())?; + stmt.exec(&self.name, id, Some(&data))?; + } + + Ok(()) + } + pub fn write_name(&self, buffer: &mut SqlBuffer) { if self.direct { // Direct tables don't have views, so use the name of the table directly. diff --git a/crates/core/src/view_admin.rs b/crates/core/src/view_admin.rs index cfe8b93..702fa16 100644 --- a/crates/core/src/view_admin.rs +++ b/crates/core/src/view_admin.rs @@ -12,7 +12,7 @@ use sqlite::{ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; -use crate::migrations::{LATEST_VERSION, powersync_migrate}; +use crate::migrations::{initialize_database, powersync_migrate}; use crate::schema::inspection::ExistingView; use crate::state::DatabaseState; use crate::utils::database::Database; @@ -34,8 +34,7 @@ extern "C" fn powersync_drop_view( fn powersync_init_impl(ctx: *mut sqlite::context, _args: &[*mut sqlite::value]) -> Result { let db = Database::from(ctx.db_handle()); - verify_in_transaction(db)?; - powersync_migrate(ctx, LATEST_VERSION)?; + initialize_database(db)?; Ok(String::from("")) } @@ -50,7 +49,7 @@ fn powersync_test_migration_impl( verify_in_transaction(db)?; let target_version = args[0].int(); - powersync_migrate(ctx, target_version)?; + powersync_migrate(db, target_version)?; Ok(String::from("")) } diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index c14659e..fd395f5 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -339,17 +339,19 @@ END''', }; } + void replaceSchema(Object schema) { + db.executeInTx( + 'SELECT powersync_replace_schema(?)', [json.encode(schema)]); + } + test('create', () { - db.executeInTx('SELECT powersync_replace_schema(?)', [ - json.encode({'tables': []}) - ]); + replaceSchema({'tables': []}); db.execute('INSERT INTO ps_untyped (type, id, data) VALUES (?, ?, ?)', [ 'users', 'user-id', json.encode({'name': 'Name', 'other': 3}) ]); - db.executeInTx( - 'SELECT powersync_replace_schema(?)', [json.encode(schema())]); + replaceSchema(schema()); expect(db.select('SELECT * FROM users'), [ { @@ -394,9 +396,7 @@ END''' }); test('local-only', () { - db.executeInTx('SELECT powersync_replace_schema(?)', [ - json.encode(schema(additionalOptions: {'local_only': true})) - ]); + replaceSchema(schema(additionalOptions: {'local_only': true})); db.execute( 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); @@ -404,8 +404,7 @@ END''' }); test('remove from schema', () { - db.executeInTx( - 'SELECT powersync_replace_schema(?)', [json.encode(schema())]); + replaceSchema(schema()); db.execute( 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); db.executeInTx('SELECT powersync_replace_schema(?)', [ @@ -421,6 +420,59 @@ END''' 'SELECT * FROM sqlite_schema WHERE type = ?', ['trigger']), isEmpty); }); + + group('migrate', () { + group('from json to direct', () { + test('local-only', () { + replaceSchema(schema( + additionalOptions: {'local_only': true, 'direct': false})); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + replaceSchema(schema(additionalOptions: {'local_only': true})); + expect(db.select('SELECT * FROM users'), hasLength(1)); + }); + + test('local-only to synced', () { + replaceSchema(schema( + additionalOptions: {'local_only': true, 'direct': false})); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + replaceSchema(schema(additionalOptions: {})); + + // Migrating from local-only to synced tables deletes data + expect(db.select('SELECT * FROM users'), isEmpty); + }); + + test('synced', () { + replaceSchema(schema(additionalOptions: {'direct': false})); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + replaceSchema(schema(additionalOptions: {})); + expect(db.select('SELECT * FROM users'), hasLength(1)); + expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); + }); + + test('synced to local-only', () { + replaceSchema(schema(additionalOptions: {'direct': false})); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + + replaceSchema(schema(additionalOptions: {'local_only': true})); + // Data should be deleted when changing to a local-only table, + // previous crud entry is still there. + expect(db.select('SELECT * FROM users'), isEmpty); + expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); + }); + }); + + // todo: from json to direct + // todo: from direct to json + + // todo: add column + // todo: change column type + // todo: remove column + // todo: split columns + }); }); }); } From 8786fc55d0355c3ddc2e290b241a74cad8571a7d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 14 Sep 2026 15:09:56 +0200 Subject: [PATCH 08/19] Remove rest column --- crates/core/src/schema/common.rs | 27 +++------------------------ crates/core/src/schema/inspection.rs | 3 --- crates/core/src/schema/management.rs | 2 +- crates/core/src/schema/raw_table.rs | 8 +++----- crates/core/src/schema/table_info.rs | 16 ++++++++++++---- crates/core/src/views.rs | 6 +++--- dart/test/schema_test.dart | 4 +--- dart/test/sync_test.dart | 1 - 8 files changed, 23 insertions(+), 44 deletions(-) diff --git a/crates/core/src/schema/common.rs b/crates/core/src/schema/common.rs index c0d93d5..592869e 100644 --- a/crates/core/src/schema/common.rs +++ b/crates/core/src/schema/common.rs @@ -10,7 +10,7 @@ use serde::Deserialize; use crate::{ schema::{ Column, CommonTableOptions, PendingStatement, PendingStatementValue, RawTable, Table, - raw_table::InferredTableStructure, table_info::RestColumnIndex, + raw_table::InferredTableStructure, }, utils::SqlBuffer, }; @@ -67,17 +67,10 @@ impl<'a> SchemaTable<'a> { pub fn infer_put_stmt(&self, table_name: &str) -> PendingStatement { let mut buffer = SqlBuffer::new(); let mut params = vec![]; - let mut rest = match self { - SchemaTable::Json(_) => Some(("_rest", RestColumnIndex::default())), - SchemaTable::Raw { .. } => None, - }; buffer.push_str("INSERT INTO "); let _ = buffer.identifier().write_str(table_name); buffer.push_str(" (id"); - if let Some((column, _)) = rest { - let _ = write!(&mut buffer, ", {column}"); - } for column in self.column_names() { buffer.comma(); @@ -85,30 +78,16 @@ impl<'a> SchemaTable<'a> { } buffer.push_str(") VALUES (?1"); params.push(PendingStatementValue::Id); - if let Some((_, ref mut rest_index)) = rest { - params.push(PendingStatementValue::Rest); - buffer.push_str(", ?2"); - rest_index.rest_parameter_positions.push(1); // this is zero-indexed - } - let data_start_index = if rest.is_some() { 3 } else { 2 }; + let data_start_index = 2; for (i, column) in self.column_names().enumerate() { buffer.comma(); let _ = write!(&mut buffer, "?{}", i + data_start_index); params.push(PendingStatementValue::Column(column.to_string())); - - if let Some((_, ref mut index)) = rest { - index.named_parameters.insert(column.to_string()); - } } buffer.push_str(") ON CONFLICT (id) DO UPDATE SET "); let mut do_update = buffer.comma_separated(); - if let Some((column, _)) = rest { - let entry = do_update.element(); - let _ = write!(entry, "{column} = ?2"); - } - // Generate an "x" = ? for all synced columns to update them without affecting local-only // columns. for (i, column) in self.column_names().enumerate() { @@ -120,7 +99,7 @@ impl<'a> SchemaTable<'a> { PendingStatement { sql: buffer.sql, params, - named_parameters_index: rest.map(|e| e.1), + named_parameters_index: None, } } diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 3e76857..1af212a 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -129,7 +129,6 @@ impl ExistingTable { internal_name, db, &None, - true, )?), }); } @@ -163,12 +162,10 @@ impl ExistingTable { buffer.push_str("INSERT INTO ps_untyped(type, id, data) SELECT ?, id, "); if let Some(ref schema) = self.direct { - buffer.push_str("powersync_json_merge("); buffer.push_str(&table_columns_to_json_object( &self.internal_name, &schema.columns, )?); - buffer.push_str(", _rest)"); } else { buffer.push_str("data"); } diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 75aa052..3de7bb8 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -105,7 +105,7 @@ fn update_tables( _ = write!(&mut create_table, "(id TEXT PRIMARY KEY NOT NULL"); if table.direct { - create_table.push_str(", _rest TEXT /* ps-managed "); + create_table.push_str(" /* ps-managed "); if table.local_only() { create_table.push_str("local-only "); } diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index 57631b7..4f8f2c0 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -25,7 +25,6 @@ impl InferredTableStructure { table_name: &str, db: Database, synced_columns: &Option, - is_direct: bool, ) -> Result { let stmt = db.prepare_v2("select name, type from pragma_table_info(?)")?; stmt.bind_text(1, table_name, Destructor::STATIC)?; @@ -43,8 +42,6 @@ impl InferredTableStructure { && !filter.matches(name) { // This column isn't part of the synced columns, skip. - } else if is_direct && name == "_rest" { - // _rest column is an artifact of direct tables, skip. } else { columns.push(Column { name: name.to_owned(), @@ -139,7 +136,6 @@ impl SchemaCacheEntry { local_table_name, db, &table.schema.synced_columns, - false, )?; let schema_table = SchemaTable::Raw { definition: table, @@ -165,7 +161,7 @@ pub fn generate_raw_table_trigger( let local_table_name = table.require_table_name()?; let synced_columns = &table.schema.synced_columns; let resolved_table = - InferredTableStructure::read_from_database(local_table_name, db, synced_columns, false)?; + InferredTableStructure::read_from_database(local_table_name, db, synced_columns)?; let as_schema_table = SchemaTable::Raw { definition: table, @@ -220,10 +216,12 @@ pub fn generate_schema_table_trigger( if write != WriteType::Insert { // Prevent illegal writes to a table marked as insert-only by raising errors here. buffer.push_str("SELECT RAISE(FAIL, 'Unexpected update on insert-only table');\n"); + has_stmt = true; } else if !flags.local_only() { // Insert-only tables use manual CRUD writes so they don't block incoming data. let fragment = table_columns_to_json_object("NEW", table.columns())?; buffer.powersync_crud_manual_put(table.name(), &fragment); + has_stmt = true; } } else { if write == WriteType::Update { diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index f745958..b49ddb0 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -145,8 +145,7 @@ impl Table { let json_object = parsed.as_object().ok_or_else(|| { PowerSyncError::argument_error("expected oplog data to be an object") })?; - let rest = stmt.render_rest_object(json_object)?; - stmt.bind_for_put(id, data, Some(json_object), rest.as_ref())?; + stmt.bind_for_put(id, data, Some(json_object), None)?; stmt.exec(&self.name, id, Some(&data))?; } @@ -162,16 +161,25 @@ impl Table { } } - pub fn generate_direct_trigger(&self, write: WriteType) -> Result { + pub fn generate_direct_trigger( + &self, + mut trigger_name: Option, + write: WriteType, + ) -> Result { debug_assert!(self.direct); + generate_schema_table_trigger( &self.name, SchemaTable::Json(self), None, - &format!("{}_trigger_{}", self.name, write), + trigger_name.get_or_insert_with(|| self.direct_trigger_name(write)), write, ) } + + pub fn direct_trigger_name(&self, write: WriteType) -> String { + format!("{}_trigger_{}", self.name, write) + } } impl RawTable { diff --git a/crates/core/src/views.rs b/crates/core/src/views.rs index ffe2e3e..3a10035 100644 --- a/crates/core/src/views.rs +++ b/crates/core/src/views.rs @@ -60,7 +60,7 @@ pub fn powersync_view_sql(table_info: &Table) -> String { pub fn powersync_trigger_delete_sql(table_info: &Table) -> Result { if table_info.direct { - return table_info.generate_direct_trigger(WriteType::Delete); + return table_info.generate_direct_trigger(None, WriteType::Delete); } if table_info.options.flags.insert_only() { @@ -122,7 +122,7 @@ pub fn powersync_trigger_delete_sql(table_info: &Table) -> Result { pub fn powersync_trigger_insert_sql(table_info: &Table) -> Result { if table_info.direct { - return table_info.generate_direct_trigger(WriteType::Insert); + return table_info.generate_direct_trigger(None, WriteType::Insert); } let name = &table_info.name; @@ -177,7 +177,7 @@ pub fn powersync_trigger_insert_sql(table_info: &Table) -> Result { pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { if table_info.direct { - return table_info.generate_direct_trigger(WriteType::Update); + return table_info.generate_direct_trigger(None, WriteType::Update); } if table_info.options.flags.insert_only() { diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index fd395f5..c7f670c 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -357,7 +357,6 @@ END''', { 'id': 'user-id', 'name': 'Name', - '_rest': '{"other":3}', }, ]); @@ -367,7 +366,7 @@ END''', )[0].columnAt(0); expect( createTable, - 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL, _rest TEXT /* ps-managed */,"name" text)', + 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL /* ps-managed */,"name" text)', ); final triggers = db @@ -465,7 +464,6 @@ END''' }); }); - // todo: from json to direct // todo: from direct to json // todo: add column diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 211a205..e693cca 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -2218,7 +2218,6 @@ CREATE TRIGGER users_ref_delete { 'id': 'my_user', 'name': 'First user', - '_rest': null, } ]); From f1f5d645ee47426e8afce1d8318358d181399a74 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 14 Sep 2026 15:47:30 +0200 Subject: [PATCH 09/19] Support index --- crates/core/src/schema/management.rs | 63 ++++++++++++++++++---------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 3de7bb8..b8b74cd 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -1,11 +1,12 @@ extern crate alloc; use alloc::borrow::ToOwned; +use alloc::collections::BTreeSet; use alloc::collections::btree_map::BTreeMap; +use alloc::format; use alloc::rc::Rc; use alloc::string::String; use alloc::vec::Vec; -use alloc::{format, vec}; use core::ffi::c_int; use core::fmt::Write; @@ -16,6 +17,7 @@ use sqlite::{Connection, ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; use crate::migrations::initialize_database; +use crate::schema::Table; use crate::schema::inspection::{ExistingTable, ExistingView}; use crate::schema::table_info::Index; use crate::state::DatabaseState; @@ -149,18 +151,28 @@ fn update_tables( Ok(()) } -fn create_index_stmt(table_name: &str, index_name: &str, index: &Index) -> String { +fn create_index_stmt(table: &Table, index_name: &str, index: &Index) -> String { let mut sql = SqlBuffer::new(); sql.push_str("CREATE INDEX "); let _ = sql.identifier().write_str(&index_name); + if table.direct { + // We use this to identify old indexes to remove them. This is only required for direct + // tables, for json tables we use the ps_data prefix. + sql.push_str("/* ps-managed */"); + } sql.push_str(" ON "); - let _ = sql.identifier().write_str(&table_name); + table.write_name(&mut sql); sql.push_char('('); { let mut sql = sql.comma_separated(); for indexed_column in &index.columns { let sql = sql.element(); - sql.json_extract_and_cast("data", &indexed_column.name, &indexed_column.type_name); + + if table.direct { + let _ = sql.identifier().write_str(&indexed_column.name); + } else { + sql.json_extract_and_cast("data", &indexed_column.name, &indexed_column.type_name); + } if !indexed_column.ascending { sql.push_str(" DESC"); @@ -174,7 +186,7 @@ fn create_index_stmt(table_name: &str, index_name: &str, index: &Index) -> Strin fn update_indexes(db: Database, schema: &Schema) -> Result<()> { let mut statements: Vec = alloc::vec![]; - let mut expected_index_names: Vec = vec![]; + let mut expected_index_names: BTreeSet = Default::default(); { // In a block so that the statement is finalized before dropping indexes @@ -201,7 +213,7 @@ fn update_indexes(db: Database, schema: &Schema) -> Result<()> { result }; - let sql = create_index_stmt(&table_name, &index_name, index); + let sql = create_index_stmt(&table, &index_name, index); if existing_sql.is_none() { statements.push(sql); } else if existing_sql != Some(&sql) { @@ -212,30 +224,23 @@ fn update_indexes(db: Database, schema: &Schema) -> Result<()> { statements.push(sql); } - expected_index_names.push(index_name); + expected_index_names.insert(index_name); } } - // In a block so that the statement is finalized before dropping indexes // language=SQLite let statement = db.prepare_v2( "\ -SELECT - sqlite_master.name as index_name - FROM sqlite_master - WHERE sqlite_master.type = 'index' - AND sqlite_master.name GLOB 'ps_data_*' - AND sqlite_master.name NOT IN (SELECT value FROM json_each(?)) -", +SELECT name FROM sqlite_master + WHERE type = 'index' + AND (name GLOB 'ps_data_*' OR sqlite_master.sql GLOB '* ps-managed *')", )?; - let json_names = serde_json::to_string(&expected_index_names) - .map_err(PowerSyncError::as_argument_error)?; - statement.bind_text(1, &json_names, sqlite::Destructor::STATIC)?; while statement.step()? { let name = statement.column_text(0)?; - - statements.push(format!("DROP INDEX {}", SqlBuffer::quote_identifier(name))); + if !expected_index_names.contains(name) { + statements.push(format!("DROP INDEX {}", SqlBuffer::quote_identifier(name))); + } } } @@ -353,14 +358,26 @@ pub fn register( mod test { use alloc::{string::ToString, vec}; - use crate::schema::table_info::{Index, IndexedColumn}; + use crate::schema::{ + Table, + table_info::{Index, IndexedColumn}, + }; use super::create_index_stmt; #[test] fn test_create_index() { + let table = Table { + name: "table".to_string(), + view_name_override: None, + columns: Default::default(), + indexes: Default::default(), + options: Default::default(), + direct: false, + }; + let stmt = create_index_stmt( - "table", + &table, "index", &Index { name: "unused".to_string(), @@ -381,7 +398,7 @@ mod test { assert_eq!( stmt, - r#"CREATE INDEX "index" ON "table"(CAST(json_extract(data, '$.a') as text), CAST(json_extract(data, '$.b') as integer) DESC)"# + r#"CREATE INDEX "index" ON "ps_data__table"(CAST(json_extract(data, '$.a') as text), CAST(json_extract(data, '$.b') as integer) DESC)"# ) } } From 437ffe391bc1277a4ac51d6c1713f98ab7a6870c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 15 Sep 2026 18:11:27 +0200 Subject: [PATCH 10/19] Start migrating tables --- crates/core/src/schema/management.rs | 169 ++++++++++++++++++++++----- crates/core/src/schema/table_info.rs | 38 ++++++ crates/core/src/utils/database.rs | 33 +++++- crates/core/src/utils/sql_buffer.rs | 41 ++++++- crates/sqlite_nostd/src/capi.rs | 24 +++- dart/test/schema_test.dart | 16 ++- 6 files changed, 281 insertions(+), 40 deletions(-) diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index b8b74cd..5b8d7ff 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -3,23 +3,24 @@ extern crate alloc; use alloc::borrow::ToOwned; use alloc::collections::BTreeSet; use alloc::collections::btree_map::BTreeMap; -use alloc::format; use alloc::rc::Rc; use alloc::string::String; use alloc::vec::Vec; +use alloc::{format, vec}; use core::ffi::c_int; use core::fmt::Write; -use powersync_sqlite_nostd as sqlite; use powersync_sqlite_nostd::Context; +use powersync_sqlite_nostd::{self as sqlite, Destructor}; use sqlite::{Connection, ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; use crate::migrations::initialize_database; -use crate::schema::Table; use crate::schema::inspection::{ExistingTable, ExistingView}; -use crate::schema::table_info::Index; +use crate::schema::raw_table::InferredTableStructure; +use crate::schema::table_info::{CreateTableStatement, Index}; +use crate::schema::{Column, Table}; use crate::state::DatabaseState; use crate::utils::database::Database; use crate::utils::{SqlBuffer, verify_in_transaction}; @@ -93,35 +94,24 @@ fn update_tables( "Switching from direct to json-based tables is not yet implemented.", )); } - (Some(_), true) => { - // TODO: Consider migrations in schema tables. + (Some(previous), true) => { + direct_table_migration(db, previous, table)?; + continue; } } } // New table. - let mut create_table = SqlBuffer::default(); - - create_table.push_str("CREATE TABLE "); - table.write_name(&mut create_table); - _ = write!(&mut create_table, "(id TEXT PRIMARY KEY NOT NULL"); - - if table.direct { - create_table.push_str(" /* ps-managed "); - if table.local_only() { - create_table.push_str("local-only "); + let create_table = { + let mut create = CreateTableStatement::from(table); + if table.direct { + for column in &table.columns { + create.push_column(&column.name, &column.type_name); + } } - create_table.push_str("*/"); - for column in &table.columns { - create_table.push_char(','); - let _ = create_table.identifier().write_str(&column.name); - let _ = write!(&mut create_table, " {}", column.type_name); - } - } else { - create_table.push_str(", data TEXT"); - } - create_table.push_str(");"); + create.finish() + }; db.exec_safe_str(&create_table.sql)?; if let Some(old_json_table) = move_data_from { @@ -151,6 +141,123 @@ fn update_tables( Ok(()) } +fn direct_table_migration(db: Database, old: &InferredTableStructure, new: &Table) -> Result<()> { + debug_assert!(new.direct); + + struct ExistingColumn<'a> { + column: &'a Column, + index_in_table: usize, + found_in_old: bool, + } + + let mut new_columns: Vec<_> = new + .columns + .iter() + .enumerate() + .map(|(i, column)| ExistingColumn { + column, + index_in_table: i, + found_in_old: false, + }) + .collect(); + new_columns.sort_by(|a, b| a.column.name.cmp(&b.column.name)); + + let mut deleted_columns = vec![]; + let mut changed_column_types = vec![]; + + for old_column in &old.columns { + let Ok(new_column_index) = + new_columns.binary_search_by(|probe| probe.column.name.cmp(&old_column.name)) + else { + deleted_columns.push(old_column); + continue; + }; + + let new_column = &mut new_columns[new_column_index]; + new_column.found_in_old = true; + + if new_column.column.type_name != old_column.type_name { + changed_column_types.push((new_column.index_in_table, &new_column.column.type_name)); + } + } + + new_columns.retain(|c| !c.found_in_old); + + if new_columns.is_empty() && deleted_columns.is_empty() && changed_column_types.is_empty() { + return Ok(()); // Nothing to migrate. + } + + // Migrate the direct table. First, we delete every index on it (they will be re-created + // by update_indexes afterwards). + { + let stmt = + db.prepare_v2("SELECT name FROM sqlite_schema WHERE type = 'index' AND sql IS NOT NULL AND tbl_name = ?")?; + stmt.bind_text(1, &new.name, Destructor::STATIC)?; + + while stmt.step()? { + let index_name = stmt.column_text(0)?; + + let mut stmt = SqlBuffer::new(); + stmt.drop_index(&index_name); + db.exec_safe_str(&stmt.sql)?; + } + } + + if !changed_column_types.is_empty() { + // To change column types, we change the CREATE TABLE statement for the table. As long as + // we do this in a way that doesn't alter the order of existing columns, this doesn't + // corrupt data (column types in non-strict tables only affects type affinity for inserts + // and updates). The proper way to run this migration requires copying data, which we want + // to avoid. + let schema_writable_before = db.has_writable_schema(); + if !schema_writable_before { + db.set_writable_schema(true)?; + } + + let mut new_create_table = CreateTableStatement::from(new); + let mut changed_column_types = changed_column_types.iter().peekable(); + + for (i, column) in old.columns.iter().enumerate() { + let changed_type = changed_column_types + .next_if(|(index, _)| *index == i) + .map(|(_, type_name)| type_name.as_str()); + + new_create_table.push_column(&column.name, changed_type.unwrap_or(&column.type_name)); + } + + let new_create_table = new_create_table.finish(); + + { + let stmt = db + .prepare_v2("UPDATE sqlite_schema SET sql = ? WHERE type = 'table' AND name = ?")?; + stmt.bind_text(1, &new_create_table.sql, Destructor::STATIC)?; + stmt.bind_text(2, &new.name, Destructor::STATIC)?; + stmt.exec()?; + } + + if !schema_writable_before { + db.set_writable_schema(false)?; + } + } + + // Add new columns, drop old ones + for new_column in new_columns { + let mut stmt = SqlBuffer::new(); + stmt.alter_table(&new.name); + stmt.add_column(new_column.column); + db.exec_safe_str(&stmt.sql)?; + } + + for dropped_column in deleted_columns { + let mut stmt = SqlBuffer::new(); + stmt.alter_table(&new.name); + stmt.drop_column(&dropped_column.name); + db.exec_safe_str(&stmt.sql)?; + } + + Ok(()) +} + fn create_index_stmt(table: &Table, index_name: &str, index: &Index) -> String { let mut sql = SqlBuffer::new(); sql.push_str("CREATE INDEX "); @@ -217,10 +324,10 @@ fn update_indexes(db: Database, schema: &Schema) -> Result<()> { if existing_sql.is_none() { statements.push(sql); } else if existing_sql != Some(&sql) { - statements.push(format!( - "DROP INDEX {}", - SqlBuffer::quote_identifier(&index_name) - )); + let mut drop_stmt = SqlBuffer::new(); + drop_stmt.drop_index(&index_name); + + statements.push(drop_stmt.sql); statements.push(sql); } @@ -313,7 +420,7 @@ fn powersync_replace_schema_impl( initialize_database(db)?; - let views = ExistingView::list(db)?; + let views: Vec = ExistingView::list(db)?; let mut existing_views = { let mut map = BTreeMap::new(); for entry in &views { diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index b49ddb0..fc06794 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -473,3 +473,41 @@ pub enum PendingStatementValue { /// The full JSON object for the row, as received from the PowerSync service. Row, } + +pub struct CreateTableStatement { + create_table: SqlBuffer, +} + +impl From<&Table> for CreateTableStatement { + fn from(value: &Table) -> Self { + let mut create_table = SqlBuffer::new(); + + create_table.push_str("CREATE TABLE "); + value.write_name(&mut create_table); + create_table.push_str("(id TEXT PRIMARY KEY NOT NULL"); + + if value.direct { + create_table.push_str(" /* ps-managed "); + if value.local_only() { + create_table.push_str("local-only "); + } + create_table.push_str("*/"); + } else { + create_table.push_str(", data TEXT"); + } + + Self { create_table } + } +} + +impl CreateTableStatement { + pub fn push_column(&mut self, name: &str, type_name: &str) { + self.create_table.push_char(','); + self.create_table.column_definition(name, type_name); + } + + pub fn finish(mut self) -> SqlBuffer { + self.create_table.push_str(");"); + self.create_table + } +} diff --git a/crates/core/src/utils/database.rs b/crates/core/src/utils/database.rs index 8f7eb53..d5f568b 100644 --- a/crates/core/src/utils/database.rs +++ b/crates/core/src/utils/database.rs @@ -1,4 +1,4 @@ -use core::ffi::{CStr, c_char}; +use core::ffi::{CStr, c_char, c_int}; use alloc::ffi::CString; use num_traits::FromPrimitive; @@ -83,6 +83,37 @@ impl Database { statement.bind_text(1, param, Destructor::STATIC)?; statement.exec() } + + pub fn has_writable_schema(self) -> bool { + let mut result: c_int = 0; + let _ = sqlite::db_config( + self.sqlite, + sqlite::DBCONFIG_WRITABLE_SCHEMA, + -1, // negative to leave the setting unchanged. + &mut result, + ); + result != 0 + } + + pub fn set_writable_schema(self, enable: bool) -> Result<()> { + let mut result = 0; + let expected_result = if enable { 1 } else { 0 }; + + let _ = sqlite::db_config( + self.sqlite, + sqlite::DBCONFIG_WRITABLE_SCHEMA, + expected_result, + &mut result, + ); + + if expected_result != result { + return Err(PowerSyncError::state_error( + "Could not apply writable schema", + )); + } + + Ok(()) + } } pub struct Statement { diff --git a/crates/core/src/utils/sql_buffer.rs b/crates/core/src/utils/sql_buffer.rs index 3c68e24..62d8e7e 100644 --- a/crates/core/src/utils/sql_buffer.rs +++ b/crates/core/src/utils/sql_buffer.rs @@ -6,7 +6,9 @@ use core::{ use alloc::{format, string::String}; use crate::{ - error::PowerSyncError, schema::SchemaTable, views::table_columns_to_json_object_with_filter, + error::PowerSyncError, + schema::{Column, SchemaTable}, + views::table_columns_to_json_object_with_filter, }; const DOUBLE_QUOTE: char = '"'; @@ -107,6 +109,43 @@ impl SqlBuffer { ); } + pub fn drop_index(&mut self, index_name: &str) { + self.push_str("DROP INDEX "); + let _ = self.identifier().write_str(index_name); + } + + pub fn alter_table(&mut self, table: &str) { + self.push_str("ALTER TABLE "); + let _ = self.identifier().write_str(table); + self.push_char(' '); + } + + pub fn drop_column(&mut self, name: &str) { + self.drop("COLUMN", name); + } + + pub fn drop_trigger(&mut self, name: &str) { + self.drop("TRIGGER", name); + } + + fn drop(&mut self, _type: &str, name: &str) { + self.push_str("DROP "); + self.push_str(_type); + self.push_char(' '); + let _ = self.identifier().write_str(name); + } + + pub fn add_column(&mut self, column: &Column) { + self.push_str("ADD COLUMN "); + self.column_definition(&column.name, &column.type_name); + } + + pub fn column_definition(&mut self, name: &str, type_name: &str) { + let _ = self.identifier().write_str(&name); + self.push_char(' '); + self.push_str(&type_name); + } + /// Writes an `INSERT INTO powersync_crud` statement. pub fn insert_into_powersync_crud( &mut self, diff --git a/crates/sqlite_nostd/src/capi.rs b/crates/sqlite_nostd/src/capi.rs index df9ad05..765ecf6 100644 --- a/crates/sqlite_nostd/src/capi.rs +++ b/crates/sqlite_nostd/src/capi.rs @@ -7,6 +7,7 @@ use alloc::borrow::ToOwned; use alloc::ffi::CString; pub use crate::bindings::{ + SQLITE_DBCONFIG_WRITABLE_SCHEMA as DBCONFIG_WRITABLE_SCHEMA, SQLITE_DETERMINISTIC as DETERMINISTIC, SQLITE_DIRECTONLY as DIRECTONLY, SQLITE_INDEX_CONSTRAINT_EQ as INDEX_CONSTRAINT_EQ, SQLITE_INDEX_CONSTRAINT_GE as INDEX_CONSTRAINT_GE, @@ -49,12 +50,13 @@ mod aliased { sqlite3_column_value as column_value, sqlite3_commit_hook as commit_hook, sqlite3_context_db_handle as context_db_handle, sqlite3_create_function_v2 as create_function_v2, - sqlite3_create_module_v2 as create_module_v2, sqlite3_declare_vtab as declare_vtab, - sqlite3_errcode as errcode, sqlite3_errmsg as errmsg, sqlite3_error_offset as error_offset, - sqlite3_exec as exec, sqlite3_finalize as finalize, sqlite3_free as free, - sqlite3_get_autocommit as get_autocommit, sqlite3_get_auxdata as get_auxdata, - sqlite3_libversion as libversion, sqlite3_libversion_number as libversion_number, - sqlite3_malloc as malloc, sqlite3_malloc64 as malloc64, sqlite3_mutex_alloc as mutex_alloc, + sqlite3_create_module_v2 as create_module_v2, sqlite3_db_config as db_config, + sqlite3_declare_vtab as declare_vtab, sqlite3_errcode as errcode, sqlite3_errmsg as errmsg, + sqlite3_error_offset as error_offset, sqlite3_exec as exec, sqlite3_finalize as finalize, + sqlite3_free as free, sqlite3_get_autocommit as get_autocommit, + sqlite3_get_auxdata as get_auxdata, sqlite3_libversion as libversion, + sqlite3_libversion_number as libversion_number, sqlite3_malloc as malloc, + sqlite3_malloc64 as malloc64, sqlite3_mutex_alloc as mutex_alloc, sqlite3_mutex_enter as mutex_enter, sqlite3_mutex_free as mutex_free, sqlite3_mutex_leave as mutex_leave, sqlite3_mutex_try as mutex_try, sqlite3_next_stmt as next_stmt, sqlite3_open as open, sqlite3_prepare_v2 as prepare_v2, @@ -225,6 +227,16 @@ pub fn close(db: *mut sqlite3) -> c_int { unsafe { invoke_sqlite!(close, db) } } +pub extern "C" fn db_config( + db: *mut sqlite3, + op: c_uint, + option: c_int, + result: &mut c_int, +) -> c_int { + let ptr = core::ptr::from_mut(result); + unsafe { invoke_sqlite!(db_config, db, op as c_int, option, ptr) } +} + pub fn vtab_config(db: *mut sqlite3, options: u32) -> c_int { unsafe { invoke_sqlite!(vtab_config, db, options as i32) } } diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index c7f670c..f08cb96 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -324,7 +324,10 @@ END''', }); group('direct tables', () { - Object schema({Map additionalOptions = const {}}) { + Object schema({ + Map additionalOptions = const {}, + List additionalColumns = const [], + }) { return { 'tables': [ { @@ -467,6 +470,17 @@ END''' // todo: from direct to json // todo: add column + test('adding columns', () { + replaceSchema(schema()); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + + replaceSchema(schema(additionalColumns: [ + {'name': 'new-1', 'type': 'text'}, + {'name': 'new-2', 'type': 'integer'}, + ])); + }); + // todo: change column type // todo: remove column // todo: split columns From d3642276a112c982031d95d250db1bd6e5759194 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 15 Sep 2026 20:39:34 +0200 Subject: [PATCH 11/19] Adding columns works!! --- crates/core/src/migrations.rs | 5 +- crates/core/src/schema/inspection.rs | 155 +++++++++++++++++++-------- crates/core/src/schema/management.rs | 40 +++---- crates/core/src/schema/table_info.rs | 6 +- crates/core/src/utils/sql_buffer.rs | 20 ++-- dart/test/schema_test.dart | 12 ++- 6 files changed, 158 insertions(+), 80 deletions(-) diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index b1c72ff..504dc20 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -11,7 +11,7 @@ use sqlite::ResultCode; use crate::error::{PowerSyncError, Result}; use crate::fix_data::apply_v035_fix; -use crate::schema::inspection::ExistingView; +use crate::schema::inspection::{ExistingTable, ExistingView}; use crate::sync::BucketPriority; use crate::utils::database::Database; use crate::utils::verify_in_transaction; @@ -173,7 +173,8 @@ VALUES(4, // Down migrations are less common, so we're okay about that breaking // in some cases. - for mut view in ExistingView::list(local_db)? { + let tables = ExistingTable::list(local_db)?; + for mut view in ExistingView::list(local_db, &tables)? { view.delete_trigger_sql = String::default(); view.update_trigger_sql = String::default(); view.insert_trigger_sql = String::default(); diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 1af212a..082ca97 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -1,24 +1,22 @@ use core::fmt::Write; use alloc::borrow::ToOwned; -use alloc::{format, vec}; +use alloc::vec; use alloc::{string::String, vec::Vec}; +use powersync_sqlite_nostd::Destructor; use crate::error::Result; +use crate::schema::Table; use crate::schema::raw_table::InferredTableStructure; -use crate::utils::SqlBuffer; use crate::utils::database::Database; +use crate::utils::{SqlBuffer, WriteType}; use crate::views::table_columns_to_json_object; /// An existing PowerSync-managed view that was found in the schema. #[derive(PartialEq)] pub struct ExistingView { /// The name of the view itself. - pub name: String, - /// SQL contents of the `CREATE VIEW` statement. - /// - /// This is not set for direct tables, which don't have a view. - pub sql: Option, + pub key: ViewKey, /// SQL contents of all triggers implementing deletes by forwarding to /// `ps_data` and `ps_crud`. pub delete_trigger_sql: String, @@ -28,60 +26,133 @@ pub struct ExistingView { pub update_trigger_sql: String, } +#[derive(PartialEq)] +pub enum ViewKey { + JsonTable { + /// The name of the view itself. + name: String, + /// SQL contents of the `CREATE VIEW` statement. + sql: String, + }, + DirectTable { + /// The name of the direct table for which this view has been created. + table_name: String, + }, +} + impl ExistingView { - pub fn list(db: Database) -> Result> { + pub fn list(db: Database, existing_tables: &[ExistingTable]) -> Result> { let mut results = vec![]; - let stmt = db.prepare_v2(" -SELECT - view.name, - view.sql, - ifnull(group_concat(trigger1.sql, ';\n' ORDER BY trigger1.name DESC), ''), - ifnull(trigger2.sql, ''), - ifnull(trigger3.sql, '') - FROM sqlite_master view - LEFT JOIN sqlite_master trigger1 - ON trigger1.tbl_name = view.name AND trigger1.type = 'trigger' AND trigger1.name GLOB 'ps_view_delete*' - LEFT JOIN sqlite_master trigger2 - ON trigger2.tbl_name = view.name AND trigger2.type = 'trigger' AND trigger2.name GLOB 'ps_view_insert*' - LEFT JOIN sqlite_master trigger3 - ON trigger3.tbl_name = view.name AND trigger3.type = 'trigger' AND trigger3.name GLOB 'ps_view_update*' - WHERE view.type = 'view' AND view.sql GLOB '*-- powersync-auto-generated' - GROUP BY view.name; - ")?; - while stmt.step()? { - let name = stmt.column_text(0)?.to_owned(); - let sql = stmt.column_text(1)?.to_owned(); - let delete = stmt.column_text(2)?.to_owned(); - let insert = stmt.column_text(3)?.to_owned(); - let update = stmt.column_text(4)?.to_owned(); + let find_triggers = db.prepare_v2( + "SELECT name, sql FROM sqlite_schema WHERE type = 'trigger' AND tbl_name = ? ORDER BY name DESC", + )?; + let find_view = db.prepare_v2( + "SELECT sql FROM sqlite_schema WHERE type = 'view' AND name = ? AND sql GLOB '*-- powersync-auto-generated'", + )?; + + for table in existing_tables { + find_triggers.bind_text(1, &table.name, Destructor::STATIC)?; + + let mut insert_trigger_sql = String::new(); + let mut update_trigger_sql = String::new(); + let mut delete_trigger_sql = String::new(); + + while find_triggers.step()? { + let trigger_name = find_triggers.column_text(0)?; + let trigger_sql = find_triggers.column_text(1)?; + + let stmt = if trigger_name.starts_with("ps_view_delete") { + &mut delete_trigger_sql + } else if trigger_name.starts_with("ps_view_insert") { + &mut insert_trigger_sql + } else if trigger_name.starts_with("ps_view_update") { + &mut update_trigger_sql + } else { + continue; + }; + + if !stmt.is_empty() { + stmt.push_str(";\n"); + } + + stmt.push_str(trigger_sql); + } + + find_triggers.reset()?; + + let key = if table.direct.is_some() { + ViewKey::DirectTable { + table_name: table.name.clone(), + } + } else { + find_view.bind_text(1, &table.name, Destructor::STATIC)?; + let sql = if find_view.step()? { + find_view.column_text(0)?.to_owned() + } else { + String::new() + }; + find_view.reset()?; + + ViewKey::JsonTable { + name: table.name.clone(), + sql, + } + }; results.push(ExistingView { - name, - sql: Some(sql), - delete_trigger_sql: delete, - insert_trigger_sql: insert, - update_trigger_sql: update, + key, + delete_trigger_sql, + insert_trigger_sql, + update_trigger_sql, }); } Ok(results) } + pub fn name(&self) -> &str { + match &self.key { + ViewKey::JsonTable { name, .. } => &*name, + ViewKey::DirectTable { table_name } => &*table_name, + } + } + pub fn drop_by_name(db: Database, name: &str) -> Result<()> { - let q = format!("DROP VIEW IF EXISTS {:}", SqlBuffer::quote_identifier(name)); - db.exec_safe_str(&q)?; + let mut buffer = SqlBuffer::new(); + buffer.drop("VIEW", true, name); + + db.exec_safe_str(&buffer.sql)?; Ok(()) } pub fn delete_from_db(&self, db: Database) -> Result<()> { - Self::drop_by_name(db, &self.name) + match &self.key { + ViewKey::JsonTable { name, .. } => { + Self::drop_by_name(db, &name)?; + } + ViewKey::DirectTable { table_name } => { + for write in WriteType::VALUES { + let mut buffer = SqlBuffer::new(); + buffer.drop( + "TRIGGER", + true, + &Table::direct_trigger_name(table_name, *write), + ); + + db.exec_safe_str(&buffer.sql)?; + } + } + } + + Ok(()) } pub fn create(&self, db: Database) -> Result<()> { - if let Some(create_view) = &self.sql { - Self::drop_by_name(db, &self.name)?; - db.exec_safe_str(create_view)?; + self.delete_from_db(db)?; + + if let ViewKey::JsonTable { sql, .. } = &self.key { + db.exec_safe_str(sql)?; } db.exec_safe_str(&self.delete_trigger_sql)?; db.exec_safe_str(&self.insert_trigger_sql)?; diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 5b8d7ff..5530405 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -4,7 +4,7 @@ use alloc::borrow::ToOwned; use alloc::collections::BTreeSet; use alloc::collections::btree_map::BTreeMap; use alloc::rc::Rc; -use alloc::string::String; +use alloc::string::{String, ToString}; use alloc::vec::Vec; use alloc::{format, vec}; use core::ffi::c_int; @@ -17,7 +17,7 @@ use sqlite::{Connection, ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; use crate::migrations::initialize_database; -use crate::schema::inspection::{ExistingTable, ExistingView}; +use crate::schema::inspection::{ExistingTable, ExistingView, ViewKey}; use crate::schema::raw_table::InferredTableStructure; use crate::schema::table_info::{CreateTableStatement, Index}; use crate::schema::{Column, Table}; @@ -34,12 +34,12 @@ use super::Schema; fn update_tables( db: Database, schema: &Schema, + existing_tables: &[ExistingTable], existing_views: &mut BTreeMap<&str, &ExistingView>, ) -> Result<()> { - let existing_tables = ExistingTable::list(db)?; let mut existing_tables = { let mut map = BTreeMap::new(); - for table in &existing_tables { + for table in existing_tables { map.insert(&*table.name, table); } map @@ -198,7 +198,7 @@ fn direct_table_migration(db: Database, old: &InferredTableStructure, new: &Tabl let index_name = stmt.column_text(0)?; let mut stmt = SqlBuffer::new(); - stmt.drop_index(&index_name); + stmt.drop("INDEX", false, &index_name); db.exec_safe_str(&stmt.sql)?; } } @@ -251,7 +251,7 @@ fn direct_table_migration(db: Database, old: &InferredTableStructure, new: &Tabl for dropped_column in deleted_columns { let mut stmt = SqlBuffer::new(); stmt.alter_table(&new.name); - stmt.drop_column(&dropped_column.name); + stmt.drop("COLUMN", false, &dropped_column.name); db.exec_safe_str(&stmt.sql)?; } @@ -325,7 +325,7 @@ fn update_indexes(db: Database, schema: &Schema) -> Result<()> { statements.push(sql); } else if existing_sql != Some(&sql) { let mut drop_stmt = SqlBuffer::new(); - drop_stmt.drop_index(&index_name); + drop_stmt.drop("INDEX", false, &index_name); statements.push(drop_stmt.sql); statements.push(sql); @@ -366,18 +366,21 @@ fn update_views( existing: &mut BTreeMap<&str, &ExistingView>, ) -> Result<()> { for table in &schema.tables { - let view_sql = if table.direct { - None - } else { - Some(powersync_view_sql(table)) - }; let delete_trigger_sql = powersync_trigger_delete_sql(table)?; let insert_trigger_sql = powersync_trigger_insert_sql(table)?; let update_trigger_sql = powersync_trigger_update_sql(table)?; let wanted_view = ExistingView { - name: table.view_name().to_owned(), - sql: view_sql, + key: if table.direct { + ViewKey::DirectTable { + table_name: table.name.to_string(), + } + } else { + ViewKey::JsonTable { + name: table.view_name().to_owned(), + sql: powersync_view_sql(table), + } + }, delete_trigger_sql, insert_trigger_sql, update_trigger_sql, @@ -396,7 +399,7 @@ fn update_views( // Delete old views. for remaining in existing.values() { - ExistingView::drop_by_name(db, &remaining.name)?; + remaining.delete_from_db(db)?; } Ok(()) @@ -420,16 +423,17 @@ fn powersync_replace_schema_impl( initialize_database(db)?; - let views: Vec = ExistingView::list(db)?; + let existing_tables = ExistingTable::list(db)?; + let views: Vec = ExistingView::list(db, &existing_tables)?; let mut existing_views = { let mut map = BTreeMap::new(); for entry in &views { - map.insert(&*entry.name, entry); + map.insert(entry.name(), entry); } map }; - update_tables(db, &parsed_schema, &mut existing_views)?; + update_tables(db, &parsed_schema, &existing_tables, &mut existing_views)?; update_indexes(db, &parsed_schema)?; update_views(db, &parsed_schema, &mut existing_views)?; diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index fc06794..33151cf 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -172,13 +172,13 @@ impl Table { &self.name, SchemaTable::Json(self), None, - trigger_name.get_or_insert_with(|| self.direct_trigger_name(write)), + trigger_name.get_or_insert_with(|| Self::direct_trigger_name(&self.name, write)), write, ) } - pub fn direct_trigger_name(&self, write: WriteType) -> String { - format!("{}_trigger_{}", self.name, write) + pub fn direct_trigger_name(name: &str, write: WriteType) -> String { + format!("{}_trigger_{}", name, write) } } diff --git a/crates/core/src/utils/sql_buffer.rs b/crates/core/src/utils/sql_buffer.rs index 62d8e7e..518dc9a 100644 --- a/crates/core/src/utils/sql_buffer.rs +++ b/crates/core/src/utils/sql_buffer.rs @@ -109,28 +109,18 @@ impl SqlBuffer { ); } - pub fn drop_index(&mut self, index_name: &str) { - self.push_str("DROP INDEX "); - let _ = self.identifier().write_str(index_name); - } - pub fn alter_table(&mut self, table: &str) { self.push_str("ALTER TABLE "); let _ = self.identifier().write_str(table); self.push_char(' '); } - pub fn drop_column(&mut self, name: &str) { - self.drop("COLUMN", name); - } - - pub fn drop_trigger(&mut self, name: &str) { - self.drop("TRIGGER", name); - } - - fn drop(&mut self, _type: &str, name: &str) { + pub fn drop(&mut self, _type: &str, if_exists: bool, name: &str) { self.push_str("DROP "); self.push_str(_type); + if if_exists { + self.push_str(" IF EXISTS"); + } self.push_char(' '); let _ = self.identifier().write_str(name); } @@ -359,6 +349,8 @@ pub enum WriteType { } impl WriteType { + pub const VALUES: &[WriteType] = &[WriteType::Insert, WriteType::Update, WriteType::Delete]; + pub fn ps_crud_op_type(&self) -> &'static str { match self { WriteType::Insert => "PUT", diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index f08cb96..9a18b97 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -333,7 +333,8 @@ END''', { 'name': 'users', 'columns': [ - {'name': 'name', 'type': 'text'} + {'name': 'name', 'type': 'text'}, + ...additionalColumns, ], 'direct': true, ...additionalOptions, @@ -479,6 +480,15 @@ END''' {'name': 'new-1', 'type': 'text'}, {'name': 'new-2', 'type': 'integer'}, ])); + + expect(db.select('SELECT * FROM users'), [ + { + 'id': 'id', + 'name': 'name', + 'new-1': null, + 'new-2': null, + } + ]); }); // todo: change column type From 5abc27ceda4ec6300859befbb9694fa5686d3a32 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 16 Sep 2026 10:57:10 +0200 Subject: [PATCH 12/19] More migration tests --- crates/core/src/schema/management.rs | 18 ++++++-- dart/test/schema_test.dart | 66 ++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 5530405..dacd514 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -95,7 +95,7 @@ fn update_tables( )); } (Some(previous), true) => { - direct_table_migration(db, previous, table)?; + direct_table_migration(db, previous, table, existing_views)?; continue; } } @@ -141,7 +141,12 @@ fn update_tables( Ok(()) } -fn direct_table_migration(db: Database, old: &InferredTableStructure, new: &Table) -> Result<()> { +fn direct_table_migration( + db: Database, + old: &InferredTableStructure, + new: &Table, + existing_views: &mut BTreeMap<&str, &ExistingView>, +) -> Result<()> { debug_assert!(new.direct); struct ExistingColumn<'a> { @@ -187,8 +192,9 @@ fn direct_table_migration(db: Database, old: &InferredTableStructure, new: &Tabl return Ok(()); // Nothing to migrate. } - // Migrate the direct table. First, we delete every index on it (they will be re-created - // by update_indexes afterwards). + // Migrate the direct table. SQLite validates associated triggers and indexes on ALTER TABLE + // statements, so we drop those first. A subsequent update_indexes and update_views call will + // create them again. { let stmt = db.prepare_v2("SELECT name FROM sqlite_schema WHERE type = 'index' AND sql IS NOT NULL AND tbl_name = ?")?; @@ -201,6 +207,10 @@ fn direct_table_migration(db: Database, old: &InferredTableStructure, new: &Tabl stmt.drop("INDEX", false, &index_name); db.exec_safe_str(&stmt.sql)?; } + + if let Some(old_triggers) = existing_views.remove(new.name.as_str()) { + old_triggers.delete_from_db(db)?; + } } if !changed_column_types.is_empty() { diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index 9a18b97..3afdd26 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -470,7 +470,6 @@ END''' // todo: from direct to json - // todo: add column test('adding columns', () { replaceSchema(schema()); db.execute( @@ -491,9 +490,68 @@ END''' ]); }); - // todo: change column type - // todo: remove column - // todo: split columns + test('change column type', () { + replaceSchema(schema(additionalColumns: [ + {'name': 'additional', 'type': 'text'} + ])); + db.execute( + 'INSERT INTO users (id, name, additional) VALUES (?, ?, ?)', + ['id', 'name', 'text']); + + replaceSchema(schema(additionalColumns: [ + {'name': 'additional', 'type': 'integer'} + ])); + + expect(db.select('SELECT * FROM users'), [ + { + 'id': 'id', + 'name': 'name', + 'additional': 'text', + } + ]); + }); + + test('remove column', () { + replaceSchema(schema(additionalColumns: [ + {'name': 'additional', 'type': 'text'} + ])); + db.execute( + 'INSERT INTO users (id, name, additional) VALUES (?, ?, ?)', + ['id', 'name', 'text']); + replaceSchema(schema(additionalColumns: [])); + + expect(db.select('SELECT * FROM users'), [ + { + 'id': 'id', + 'name': 'name', + } + ]); + }); + + test('multiple column migrations at once', () { + replaceSchema(schema(additionalColumns: [ + {'name': 'removed', 'type': 'text'}, + {'name': 'changed-type', 'type': 'text'}, + ])); + db.execute( + 'INSERT INTO users (id, name, removed, "changed-type") VALUES (?, ?, ?, ?)', + ['id', 'name', 'removed', 'changed-type'], + ); + + replaceSchema(schema(additionalColumns: [ + {'name': 'added', 'type': 'text'}, + {'name': 'changed-type', 'type': 'integer'}, + ])); + + expect(db.select('SELECT * FROM users'), [ + { + 'id': 'id', + 'name': 'name', + 'changed-type': 'changed-type', + 'added': null, + } + ]); + }); }); }); }); From 032262b87eb84cf3146eb422bb7ad0ab6b02a2f7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 16 Sep 2026 12:01:25 +0200 Subject: [PATCH 13/19] Fix existing schema tests --- crates/core/src/schema/inspection.rs | 67 ++++++++++++++-------------- crates/core/src/schema/management.rs | 2 +- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index 082ca97..f3384de 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -47,12 +47,10 @@ impl ExistingView { let find_triggers = db.prepare_v2( "SELECT name, sql FROM sqlite_schema WHERE type = 'trigger' AND tbl_name = ? ORDER BY name DESC", )?; - let find_view = db.prepare_v2( - "SELECT sql FROM sqlite_schema WHERE type = 'view' AND name = ? AND sql GLOB '*-- powersync-auto-generated'", - )?; + let find_views = db.prepare_v2("SELECT name, sql FROM sqlite_schema WHERE type = 'view' AND sql GLOB '*-- powersync-auto-generated'")?; - for table in existing_tables { - find_triggers.bind_text(1, &table.name, Destructor::STATIC)?; + let complete_triggers = |key: ViewKey| -> Result { + find_triggers.bind_text(1, &key.name(), Destructor::STATIC)?; let mut insert_trigger_sql = String::new(); let mut update_trigger_sql = String::new(); @@ -80,42 +78,34 @@ impl ExistingView { } find_triggers.reset()?; - - let key = if table.direct.is_some() { - ViewKey::DirectTable { - table_name: table.name.clone(), - } - } else { - find_view.bind_text(1, &table.name, Destructor::STATIC)?; - let sql = if find_view.step()? { - find_view.column_text(0)?.to_owned() - } else { - String::new() - }; - find_view.reset()?; - - ViewKey::JsonTable { - name: table.name.clone(), - sql, - } - }; - - results.push(ExistingView { + Ok(ExistingView { key, delete_trigger_sql, insert_trigger_sql, update_trigger_sql, - }); - } + }) + }; - Ok(results) - } + while find_views.step()? { + let name = find_views.column_text(0)?.to_owned(); + let sql = find_views.column_text(1)?.to_owned(); - pub fn name(&self) -> &str { - match &self.key { - ViewKey::JsonTable { name, .. } => &*name, - ViewKey::DirectTable { table_name } => &*table_name, + let key = ViewKey::JsonTable { name, sql }; + results.push(complete_triggers(key)?); + } + + for table in existing_tables { + if table.direct.is_some() { + // Direct tables don't have a view, but we still want to collect associated + // triggers. + let key = ViewKey::DirectTable { + table_name: table.name.clone(), + }; + results.push(complete_triggers(key)?); + } } + + Ok(results) } pub fn drop_by_name(db: Database, name: &str) -> Result<()> { @@ -162,6 +152,15 @@ impl ExistingView { } } +impl ViewKey { + pub fn name(&self) -> &str { + match &self { + ViewKey::JsonTable { name, .. } => name, + ViewKey::DirectTable { table_name } => table_name, + } + } +} + pub struct ExistingTable { pub name: String, pub internal_name: String, diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index dacd514..3e3f806 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -438,7 +438,7 @@ fn powersync_replace_schema_impl( let mut existing_views = { let mut map = BTreeMap::new(); for entry in &views { - map.insert(entry.name(), entry); + map.insert(entry.key.name(), entry); } map }; From 35973df11e4f6dc39d027884a91a5298b915be98 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 16 Sep 2026 15:25:22 +0200 Subject: [PATCH 14/19] Rename triggers --- crates/core/src/schema/inspection.rs | 3 +- crates/core/src/schema/management.rs | 45 +++++++++++++++++++++++----- crates/core/src/schema/raw_table.rs | 2 +- crates/core/src/schema/table_info.rs | 13 +++++--- crates/core/src/utils/mod.rs | 2 +- crates/core/src/utils/sql_buffer.rs | 26 ++++++++++++++-- crates/core/src/views.rs | 14 +++++---- dart/test/schema_test.dart | 44 +++++++++++++++++++++++++-- 8 files changed, 124 insertions(+), 25 deletions(-) diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index f3384de..ba1864b 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -1,6 +1,7 @@ use core::fmt::Write; use alloc::borrow::ToOwned; +use alloc::string::ToString; use alloc::vec; use alloc::{string::String, vec::Vec}; use powersync_sqlite_nostd::Destructor; @@ -127,7 +128,7 @@ impl ExistingView { buffer.drop( "TRIGGER", true, - &Table::direct_trigger_name(table_name, *write), + &Table::crud_trigger_name(table_name, *write).to_string(), ); db.exec_safe_str(&buffer.sql)?; diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 3e3f806..e5788f9 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -49,6 +49,7 @@ fn update_tables( let mut move_data_from = None::<&str>; if let Some(existing) = existing_tables.remove(&*table.name) { + // Migrate between JSON-based and direct tables. match (&existing.direct, table.direct) { (None, false) => { // JSON-based table before and now. We might have to migrate between synced and @@ -95,8 +96,24 @@ fn update_tables( )); } (Some(previous), true) => { - direct_table_migration(db, previous, table, existing_views)?; - continue; + if existing.local_only != table.local_only() { + // Unlike with json-based tables where local and synced tables have + // different names, here we need to drop the old table first. + existing_views.remove(existing.name.as_str()); + + if !existing.local_only { + existing.move_into_ps_untyped(db)?; + } + + let mut buffer = SqlBuffer::new(); + buffer.drop("TABLE", false, &existing.internal_name); + db.exec_safe_str(&buffer.sql)?; + } else { + // Otherwise compatible tables might still have different columns, which + // requires a migration for direct tables. + direct_table_migration(db, previous, table, existing_views)?; + continue; + } } } } @@ -131,11 +148,9 @@ fn update_tables( // We cannot have any open queries on sqlite_master at the point that we drop tables, otherwise // we get "table is locked" errors. for remaining in existing_tables.values() { - let q = format!( - "DROP TABLE {:}", - SqlBuffer::quote_identifier(&remaining.internal_name) - ); - db.exec_safe_str(&q)?; + let mut buffer = SqlBuffer::new(); + buffer.drop("TABLE", false, &remaining.internal_name); + db.exec_safe_str(&buffer.sql)?; } Ok(()) @@ -181,7 +196,11 @@ fn direct_table_migration( let new_column = &mut new_columns[new_column_index]; new_column.found_in_old = true; - if new_column.column.type_name != old_column.type_name { + if !new_column + .column + .type_name + .eq_ignore_ascii_case(&old_column.type_name) + { changed_column_types.push((new_column.index_in_table, &new_column.column.type_name)); } } @@ -397,6 +416,16 @@ fn update_views( }; if let Some(actual_view) = existing.remove(table.view_name()) { + if wanted_view.key == actual_view.key { + if wanted_view.delete_trigger_sql == actual_view.delete_trigger_sql { + if wanted_view.update_trigger_sql == actual_view.update_trigger_sql { + if wanted_view.insert_trigger_sql == actual_view.insert_trigger_sql { + continue; + } + } + } + } + if *actual_view == wanted_view { // View exists with identical definition, don't re-create. continue; diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index 4f8f2c0..d15f2c8 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -185,7 +185,7 @@ pub fn generate_schema_table_trigger( write: WriteType, ) -> Result { let mut buffer = SqlBuffer::new(); - buffer.create_trigger("", trigger_name); + buffer.create_trigger(trigger_name); buffer.trigger_after(write, local_table_name); // Skip the trigger for writes during sync_local, these aren't crud writes. buffer.push_str("WHEN NOT powersync_in_sync_operation()"); diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 33151cf..712c7b8 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -12,7 +12,7 @@ use crate::schema::raw_table::generate_schema_table_trigger; use crate::schema::{ColumnFilter, SchemaTable}; use crate::sync::PreparedPendingStatement; use crate::utils::database::{Database, Statement}; -use crate::utils::{SqlBuffer, WriteType}; +use crate::utils::{CrudTriggerName, SqlBuffer, WriteType}; #[derive(Deserialize)] pub struct Table { @@ -172,13 +172,18 @@ impl Table { &self.name, SchemaTable::Json(self), None, - trigger_name.get_or_insert_with(|| Self::direct_trigger_name(&self.name, write)), + trigger_name + .get_or_insert_with(|| Self::crud_trigger_name(&self.name, write).to_string()), write, ) } - pub fn direct_trigger_name(name: &str, write: WriteType) -> String { - format!("{}_trigger_{}", name, write) + pub fn crud_trigger_name<'a>(name: &'a str, write: WriteType) -> CrudTriggerName<'a> { + CrudTriggerName { + write, + name_suffix: "", + view_name: name, + } } } diff --git a/crates/core/src/utils/mod.rs b/crates/core/src/utils/mod.rs index 6a11893..62b1aef 100644 --- a/crates/core/src/utils/mod.rs +++ b/crates/core/src/utils/mod.rs @@ -6,7 +6,7 @@ use core::{cmp::Ordering, fmt::Display, hash::Hash}; use alloc::{boxed::Box, string::String}; use serde::Serialize; use serde_json::value::RawValue; -pub use sql_buffer::{InsertIntoCrud, SqlBuffer, WriteType}; +pub use sql_buffer::{CrudTriggerName, InsertIntoCrud, SqlBuffer, WriteType}; use crate::{ error::{PowerSyncError, RawPowerSyncError}, diff --git a/crates/core/src/utils/sql_buffer.rs b/crates/core/src/utils/sql_buffer.rs index 518dc9a..07c1d2f 100644 --- a/crates/core/src/utils/sql_buffer.rs +++ b/crates/core/src/utils/sql_buffer.rs @@ -71,9 +71,9 @@ impl SqlBuffer { let _ = write!(str, "$.{s}"); } - pub fn create_trigger(&mut self, prefix: &str, view_name: &str) { + pub fn create_trigger(&mut self, name: impl Display) { self.push_str("CREATE TRIGGER "); - self.quote_identifier_prefixed(prefix, view_name); + let _ = write!(self.identifier(), "{}", name); self.push_char(' '); } @@ -388,6 +388,28 @@ impl FromStr for WriteType { } } +pub struct CrudTriggerName<'a> { + pub write: WriteType, + pub name_suffix: &'a str, + pub view_name: &'a str, +} + +impl<'a> Display for CrudTriggerName<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "{}{}_{}", + match self.write { + WriteType::Insert => "ps_view_insert", + WriteType::Update => "ps_view_update", + WriteType::Delete => "ps_view_delete", + }, + self.name_suffix, + self.view_name + ) + } +} + #[cfg(test)] mod test { use super::SqlBuffer; diff --git a/crates/core/src/views.rs b/crates/core/src/views.rs index 3a10035..71f92c1 100644 --- a/crates/core/src/views.rs +++ b/crates/core/src/views.rs @@ -7,7 +7,7 @@ use core::mem; use crate::error::{PowerSyncError, Result}; use crate::schema::{Column, ColumnFilter, SchemaTable, Table}; -use crate::utils::{InsertIntoCrud, SqlBuffer, WriteType}; +use crate::utils::{CrudTriggerName, InsertIntoCrud, SqlBuffer, WriteType}; pub fn powersync_view_sql(table_info: &Table) -> String { let name = &table_info.name; @@ -74,7 +74,7 @@ pub fn powersync_trigger_delete_sql(table_info: &Table) -> Result { let as_schema_table = SchemaTable::from(table_info); let mut sql = SqlBuffer::new(); - sql.create_trigger("ps_view_delete_", view_name); + sql.create_trigger(Table::crud_trigger_name(view_name, WriteType::Delete)); sql.trigger_instead_of(WriteType::Delete, view_name); sql.push_str("BEGIN\n"); // First, forward to internal data table. @@ -99,7 +99,11 @@ pub fn powersync_trigger_delete_sql(table_info: &Table) -> Result { sql.trigger_end(); sql.push_str(";\n"); - sql.create_trigger("ps_view_delete2_", view_name); + sql.create_trigger(CrudTriggerName { + write: WriteType::Delete, + name_suffix: "2", + view_name, + }); sql.trigger_instead_of(WriteType::Update, view_name); sql.push_str("WHEN NEW._deleted IS TRUE BEGIN DELETE FROM "); sql.quote_internal_name(name, local_only); @@ -132,7 +136,7 @@ pub fn powersync_trigger_insert_sql(table_info: &Table) -> Result { let as_schema_table = SchemaTable::from(table_info); let mut sql = SqlBuffer::new(); - sql.create_trigger("ps_view_insert_", view_name); + sql.create_trigger(Table::crud_trigger_name(view_name, WriteType::Insert)); sql.trigger_instead_of(WriteType::Insert, view_name); sql.push_str("BEGIN\n"); @@ -190,7 +194,7 @@ pub fn powersync_trigger_update_sql(table_info: &Table) -> Result { let local_only = table_info.options.flags.local_only(); let mut sql = SqlBuffer::new(); - sql.create_trigger("ps_view_update_", view_name); + sql.create_trigger(Table::crud_trigger_name(view_name, WriteType::Update)); sql.trigger_instead_of(WriteType::Update, view_name); // If we're supposed to include metadata, we support UPDATE ... SET _deleted = TRUE with diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index 3afdd26..a3165e4 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -383,15 +383,15 @@ END''', expect(triggers, [ r''' -CREATE TRIGGER "users_trigger_DELETE" AFTER DELETE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN +CREATE TRIGGER "ps_view_delete_users" AFTER DELETE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN INSERT INTO powersync_crud(op,id,type) VALUES ('DELETE', OLD.id, 'users'); END''', r''' -CREATE TRIGGER "users_trigger_INSERT" AFTER INSERT ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN +CREATE TRIGGER "ps_view_insert_users" AFTER INSERT ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN INSERT INTO powersync_crud(op,id,type,data) VALUES ('PUT', NEW.id, 'users', json(powersync_diff('{}', json_object('name', powersync_strip_subtype(NEW."name"))))); END''', r''' -CREATE TRIGGER "users_trigger_UPDATE" AFTER UPDATE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN +CREATE TRIGGER "ps_view_update_users" AFTER UPDATE ON "users" FOR EACH ROW WHEN NOT powersync_in_sync_operation() BEGIN SELECT CASE WHEN (OLD.id != NEW.id) THEN RAISE (FAIL, 'Cannot update id') END; INSERT INTO powersync_crud(op,id,type,data,options) VALUES ('PATCH', NEW.id, 'users', json(powersync_diff(json_object('name', powersync_strip_subtype(OLD."name")), json_object('name', powersync_strip_subtype(NEW."name")))), 0); END''' @@ -425,6 +425,16 @@ END''' }); group('migrate', () { + test('unchanged', () { + replaceSchema(schema()); + + final [versionBefore] = db.select('PRAGMA schema_version'); + replaceSchema(schema()); + final [versionAfter] = db.select('PRAGMA schema_version'); + + expect(versionAfter, versionBefore); + }); + group('from json to direct', () { test('local-only', () { replaceSchema(schema( @@ -470,6 +480,34 @@ END''' // todo: from direct to json + test('from synced to local', () { + replaceSchema(schema()); + db.execute('INSERT INTO users (id, name) VALUES (?, ?)', + ['synced-id', 'name']); + + replaceSchema(schema(additionalOptions: {'local_only': true})); + + expect(db.select('SELECT * FROM ps_untyped'), hasLength(1)); + expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); + expect(db.select('SELECT * FROM users'), isEmpty); + + // A second write on the now local-only table should not be recorded. + db.execute( + 'INSERT INTO users (id, name) VALUES (uuid(), ?)', ['name']); + expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); + }); + + test('from local to synced', () { + replaceSchema(schema(additionalOptions: {'local_only': true})); + db.execute( + 'INSERT INTO users (id, name) VALUES (uuid(), ?)', ['local']); + + // Migrate to synced table. Because the previous local write would + // never get uploaded, this clears local data. + replaceSchema(schema()); + expect(db.select('SELECT * FROM users'), isEmpty); + }); + test('adding columns', () { replaceSchema(schema()); db.execute( From 5bc4fd37651d6081d4f39eb88dbea72b945f196e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 17 Sep 2026 10:38:31 +0200 Subject: [PATCH 15/19] Support migrating back to json tables --- crates/core/src/schema/management.rs | 36 +++++++--- crates/core/src/schema/table_info.rs | 37 +++++++++-- dart/test/schema_test.dart | 99 ++++++++++++++++------------ 3 files changed, 116 insertions(+), 56 deletions(-) diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index e5788f9..3de9efd 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -19,14 +19,14 @@ use crate::error::{PowerSyncError, Result}; use crate::migrations::initialize_database; use crate::schema::inspection::{ExistingTable, ExistingView, ViewKey}; use crate::schema::raw_table::InferredTableStructure; -use crate::schema::table_info::{CreateTableStatement, Index}; +use crate::schema::table_info::{CreateTableStatement, Index, JsonDataSource}; use crate::schema::{Column, Table}; use crate::state::DatabaseState; use crate::utils::database::Database; use crate::utils::{SqlBuffer, verify_in_transaction}; use crate::views::{ powersync_trigger_delete_sql, powersync_trigger_insert_sql, powersync_trigger_update_sql, - powersync_view_sql, + powersync_view_sql, table_columns_to_json_object, }; use super::Schema; @@ -46,7 +46,7 @@ fn update_tables( }; for table in &schema.tables { - let mut move_data_from = None::<&str>; + let mut move_data_from = None::; if let Some(existing) = existing_tables.remove(&*table.name) { // Migrate between JSON-based and direct tables. @@ -76,7 +76,10 @@ fn update_tables( // 4. Synced to synced: Copy data; delete old table. if existing.local_only == table.local_only() { // Case 1 or 4. - move_data_from = Some(&existing.internal_name); + move_data_from = Some(JsonDataSource { + table: &existing.internal_name, + fragment: None, + }); } else { // Case 2 and 3 is the default, we'll delete the old table in the end which // moves to ps_untyped if necessary. @@ -90,10 +93,25 @@ fn update_tables( old_view.delete_from_db(db)?; } } - (Some(_), false) => { - return Err(PowerSyncError::argument_error( - "Switching from direct to json-based tables is not yet implemented.", - )); + (Some(old_direct), false) => { + // The four cases to consider here match those from the other direction. + if existing.local_only == table.local_only() { + let json = table_columns_to_json_object( + &existing.internal_name, + &old_direct.columns, + )?; + + move_data_from = Some(JsonDataSource { + table: &existing.name, + fragment: Some(json), + }) + } else { + // Also switching synced / local-only state. We'll delete data for this, no + // need to copy. + } + + // To delete the old table in the end. + existing_tables.insert(&existing.name, existing); } (Some(previous), true) => { if existing.local_only != table.local_only() { @@ -131,7 +149,7 @@ fn update_tables( }; db.exec_safe_str(&create_table.sql)?; - if let Some(old_json_table) = move_data_from { + if let Some(ref old_json_table) = move_data_from { table.direct_move_from_json(db, old_json_table)?; } else if !table.local_only() { // MOVE data if any diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 712c7b8..50174c7 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -115,16 +115,36 @@ impl Table { pub fn direct_move_from_json( &self, db: Database, - json_table: &str, + json: &JsonDataSource, ) -> Result<(), PowerSyncError> { - debug_assert!(self.direct); - let mut source = SqlBuffer::new(); - source.push_str("SELECT id, data FROM "); - let _ = write!(source.identifier(), "{}", json_table); + // For direct tables, create a SELECT statement returning id and json data we then parse via + // direct_move_from_stmt. For json tables, we directly generate an INSERT INTO SELECT + // statement. + let direct = self.direct; + + if !direct { + source.push_str("INSERT INTO "); + source.quote_internal_name(&self.name, self.local_only()); + source.push_char(' '); + } + + source.push_str("SELECT id, "); + if let Some(ref json_fragment) = json.fragment { + source.push_str(json_fragment); + } else { + source.push_str("data "); + } + source.push_str("FROM "); + let _ = write!(source.identifier(), "{}", json.table); let source = db.prepare_v2(&source.sql)?; - self.direct_move_from_stmt(db, source) + + if direct { + self.direct_move_from_stmt(db, source) + } else { + source.exec() + } } /// For direct tables, copies data from a prepared statement returning id and data. @@ -199,6 +219,11 @@ impl RawTable { } } +pub struct JsonDataSource<'a> { + pub table: &'a str, + pub fragment: Option, +} + #[derive(Deserialize)] pub struct Column { pub name: String, diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index a3165e4..5828592 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -435,48 +435,65 @@ END''' expect(versionAfter, versionBefore); }); - group('from json to direct', () { - test('local-only', () { - replaceSchema(schema( - additionalOptions: {'local_only': true, 'direct': false})); - db.execute( - 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); - replaceSchema(schema(additionalOptions: {'local_only': true})); - expect(db.select('SELECT * FROM users'), hasLength(1)); + // Test migrating from json to direct tables (and vice versa). + for (final startDirect in [false, true]) { + final fromDesc = startDirect ? 'direct' : 'json'; + final toDesc = startDirect ? 'json' : 'direct'; + final endDirect = !startDirect; + + group('from $fromDesc to $toDesc', () { + test('local-only', () { + replaceSchema(schema(additionalOptions: { + 'local_only': true, + 'direct': startDirect + })); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + replaceSchema(schema(additionalOptions: { + 'local_only': true, + 'direct': endDirect + })); + expect(db.select('SELECT * FROM users'), hasLength(1)); + }); + + test('local-only to synced', () { + replaceSchema(schema(additionalOptions: { + 'local_only': true, + 'direct': startDirect + })); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + replaceSchema(schema(additionalOptions: {'direct': endDirect})); + + // Migrating from local-only to synced tables deletes data + expect(db.select('SELECT * FROM users'), isEmpty); + }); + + test('synced', () { + replaceSchema(schema(additionalOptions: {'direct': startDirect})); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + replaceSchema(schema(additionalOptions: {'direct': endDirect})); + expect(db.select('SELECT * FROM users'), hasLength(1)); + expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); + }); + + test('synced to local-only', () { + replaceSchema(schema(additionalOptions: {'direct': startDirect})); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + + replaceSchema(schema(additionalOptions: { + 'local_only': true, + 'direct': endDirect + })); + // Data should be deleted when changing to a local-only table, + // previous crud entry is still there. + expect(db.select('SELECT * FROM users'), isEmpty); + expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); + }); }); - - test('local-only to synced', () { - replaceSchema(schema( - additionalOptions: {'local_only': true, 'direct': false})); - db.execute( - 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); - replaceSchema(schema(additionalOptions: {})); - - // Migrating from local-only to synced tables deletes data - expect(db.select('SELECT * FROM users'), isEmpty); - }); - - test('synced', () { - replaceSchema(schema(additionalOptions: {'direct': false})); - db.execute( - 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); - replaceSchema(schema(additionalOptions: {})); - expect(db.select('SELECT * FROM users'), hasLength(1)); - expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); - }); - - test('synced to local-only', () { - replaceSchema(schema(additionalOptions: {'direct': false})); - db.execute( - 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); - - replaceSchema(schema(additionalOptions: {'local_only': true})); - // Data should be deleted when changing to a local-only table, - // previous crud entry is still there. - expect(db.select('SELECT * FROM users'), isEmpty); - expect(db.select('SELECT * FROM ps_crud'), hasLength(1)); - }); - }); + } // todo: from direct to json From 530f0621439b4715029b0968d2faf945bec30034 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 17 Sep 2026 10:58:31 +0200 Subject: [PATCH 16/19] Test index creation and dropping --- dart/test/schema_test.dart | 62 +++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index 5828592..c3221aa 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -426,10 +426,21 @@ END''' group('migrate', () { test('unchanged', () { - replaceSchema(schema()); + final usedSchema = schema(additionalOptions: { + 'indexes': [ + { + 'name': 'test', + 'columns': [ + {'name': 'name', 'type': 'text', 'ascending': true}, + ] + } + ] + }); + + replaceSchema(usedSchema); final [versionBefore] = db.select('PRAGMA schema_version'); - replaceSchema(schema()); + replaceSchema(usedSchema); final [versionAfter] = db.select('PRAGMA schema_version'); expect(versionAfter, versionBefore); @@ -495,8 +506,6 @@ END''' }); } - // todo: from direct to json - test('from synced to local', () { replaceSchema(schema()); db.execute('INSERT INTO users (id, name) VALUES (?, ?)', @@ -545,6 +554,51 @@ END''' ]); }); + group('index', () { + final indexes = { + 'indexes': [ + { + 'name': 'test', + 'columns': [ + {'name': 'name', 'type': 'text', 'ascending': true}, + ] + } + ] + }; + + test('add', () { + replaceSchema(schema()); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + + replaceSchema(schema(additionalOptions: indexes)); + expect( + db.select( + 'SELECT sql FROM sqlite_schema WHERE type = ? AND tbl_name = ? AND sql IS NOT NULL', + ['index', 'users']), + [ + { + 'sql': + 'CREATE INDEX "ps_data__users__test"/* ps-managed */ ON "users"("name")' + } + ], + ); + }); + + test('remove', () { + replaceSchema(schema(additionalOptions: indexes)); + db.execute( + 'INSERT INTO users (id, name) VALUES (?, ?)', ['id', 'name']); + + replaceSchema(schema()); + expect( + db.select( + 'SELECT sql FROM sqlite_schema WHERE type = ? AND tbl_name = ? AND sql IS NOT NULL', + ['index', 'users']), + isEmpty); + }); + }); + test('change column type', () { replaceSchema(schema(additionalColumns: [ {'name': 'additional', 'type': 'text'} From 98cebcffdf1e6f1a930dcc845b9ac76f85e387d0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 17 Sep 2026 11:24:33 +0200 Subject: [PATCH 17/19] Use strict tables, any --- crates/core/src/schema/management.rs | 57 +++------------------------- crates/core/src/schema/table_info.rs | 17 +++++++-- crates/core/src/utils/database.rs | 33 +--------------- crates/core/src/utils/sql_buffer.rs | 8 ++-- crates/sqlite_nostd/src/capi.rs | 24 +++--------- dart/test/schema_test.dart | 2 +- 6 files changed, 29 insertions(+), 112 deletions(-) diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 3de9efd..8b396b3 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -141,7 +141,7 @@ fn update_tables( let mut create = CreateTableStatement::from(table); if table.direct { for column in &table.columns { - create.push_column(&column.name, &column.type_name); + create.push_any_column(&column.name); } } @@ -184,24 +184,20 @@ fn direct_table_migration( struct ExistingColumn<'a> { column: &'a Column, - index_in_table: usize, found_in_old: bool, } let mut new_columns: Vec<_> = new .columns .iter() - .enumerate() - .map(|(i, column)| ExistingColumn { + .map(|column| ExistingColumn { column, - index_in_table: i, found_in_old: false, }) .collect(); new_columns.sort_by(|a, b| a.column.name.cmp(&b.column.name)); let mut deleted_columns = vec![]; - let mut changed_column_types = vec![]; for old_column in &old.columns { let Ok(new_column_index) = @@ -214,18 +210,12 @@ fn direct_table_migration( let new_column = &mut new_columns[new_column_index]; new_column.found_in_old = true; - if !new_column - .column - .type_name - .eq_ignore_ascii_case(&old_column.type_name) - { - changed_column_types.push((new_column.index_in_table, &new_column.column.type_name)); - } + // For found columns, the type doesn't matter as we generate ANY types for all of them. } new_columns.retain(|c| !c.found_in_old); - if new_columns.is_empty() && deleted_columns.is_empty() && changed_column_types.is_empty() { + if new_columns.is_empty() && deleted_columns.is_empty() { return Ok(()); // Nothing to migrate. } @@ -250,48 +240,11 @@ fn direct_table_migration( } } - if !changed_column_types.is_empty() { - // To change column types, we change the CREATE TABLE statement for the table. As long as - // we do this in a way that doesn't alter the order of existing columns, this doesn't - // corrupt data (column types in non-strict tables only affects type affinity for inserts - // and updates). The proper way to run this migration requires copying data, which we want - // to avoid. - let schema_writable_before = db.has_writable_schema(); - if !schema_writable_before { - db.set_writable_schema(true)?; - } - - let mut new_create_table = CreateTableStatement::from(new); - let mut changed_column_types = changed_column_types.iter().peekable(); - - for (i, column) in old.columns.iter().enumerate() { - let changed_type = changed_column_types - .next_if(|(index, _)| *index == i) - .map(|(_, type_name)| type_name.as_str()); - - new_create_table.push_column(&column.name, changed_type.unwrap_or(&column.type_name)); - } - - let new_create_table = new_create_table.finish(); - - { - let stmt = db - .prepare_v2("UPDATE sqlite_schema SET sql = ? WHERE type = 'table' AND name = ?")?; - stmt.bind_text(1, &new_create_table.sql, Destructor::STATIC)?; - stmt.bind_text(2, &new.name, Destructor::STATIC)?; - stmt.exec()?; - } - - if !schema_writable_before { - db.set_writable_schema(false)?; - } - } - // Add new columns, drop old ones for new_column in new_columns { let mut stmt = SqlBuffer::new(); stmt.alter_table(&new.name); - stmt.add_column(new_column.column); + stmt.add_column(&new_column.column.name, "ANY"); db.exec_safe_str(&stmt.sql)?; } diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 50174c7..5d11cc9 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -506,6 +506,7 @@ pub enum PendingStatementValue { pub struct CreateTableStatement { create_table: SqlBuffer, + direct: bool, } impl From<&Table> for CreateTableStatement { @@ -526,18 +527,26 @@ impl From<&Table> for CreateTableStatement { create_table.push_str(", data TEXT"); } - Self { create_table } + Self { + create_table, + direct: value.direct, + } } } impl CreateTableStatement { - pub fn push_column(&mut self, name: &str, type_name: &str) { + pub fn push_any_column(&mut self, name: &str) { self.create_table.push_char(','); - self.create_table.column_definition(name, type_name); + self.create_table.column_definition(name, "ANY"); } pub fn finish(mut self) -> SqlBuffer { - self.create_table.push_str(");"); + self.create_table.push_char(')'); + if self.direct { + self.create_table.push_str(" STRICT"); + } + + self.create_table.push_char(';'); self.create_table } } diff --git a/crates/core/src/utils/database.rs b/crates/core/src/utils/database.rs index d5f568b..8f7eb53 100644 --- a/crates/core/src/utils/database.rs +++ b/crates/core/src/utils/database.rs @@ -1,4 +1,4 @@ -use core::ffi::{CStr, c_char, c_int}; +use core::ffi::{CStr, c_char}; use alloc::ffi::CString; use num_traits::FromPrimitive; @@ -83,37 +83,6 @@ impl Database { statement.bind_text(1, param, Destructor::STATIC)?; statement.exec() } - - pub fn has_writable_schema(self) -> bool { - let mut result: c_int = 0; - let _ = sqlite::db_config( - self.sqlite, - sqlite::DBCONFIG_WRITABLE_SCHEMA, - -1, // negative to leave the setting unchanged. - &mut result, - ); - result != 0 - } - - pub fn set_writable_schema(self, enable: bool) -> Result<()> { - let mut result = 0; - let expected_result = if enable { 1 } else { 0 }; - - let _ = sqlite::db_config( - self.sqlite, - sqlite::DBCONFIG_WRITABLE_SCHEMA, - expected_result, - &mut result, - ); - - if expected_result != result { - return Err(PowerSyncError::state_error( - "Could not apply writable schema", - )); - } - - Ok(()) - } } pub struct Statement { diff --git a/crates/core/src/utils/sql_buffer.rs b/crates/core/src/utils/sql_buffer.rs index 07c1d2f..0b4a51e 100644 --- a/crates/core/src/utils/sql_buffer.rs +++ b/crates/core/src/utils/sql_buffer.rs @@ -6,9 +6,7 @@ use core::{ use alloc::{format, string::String}; use crate::{ - error::PowerSyncError, - schema::{Column, SchemaTable}, - views::table_columns_to_json_object_with_filter, + error::PowerSyncError, schema::SchemaTable, views::table_columns_to_json_object_with_filter, }; const DOUBLE_QUOTE: char = '"'; @@ -125,9 +123,9 @@ impl SqlBuffer { let _ = self.identifier().write_str(name); } - pub fn add_column(&mut self, column: &Column) { + pub fn add_column(&mut self, name: &str, type_name: &str) { self.push_str("ADD COLUMN "); - self.column_definition(&column.name, &column.type_name); + self.column_definition(name, type_name); } pub fn column_definition(&mut self, name: &str, type_name: &str) { diff --git a/crates/sqlite_nostd/src/capi.rs b/crates/sqlite_nostd/src/capi.rs index 765ecf6..df9ad05 100644 --- a/crates/sqlite_nostd/src/capi.rs +++ b/crates/sqlite_nostd/src/capi.rs @@ -7,7 +7,6 @@ use alloc::borrow::ToOwned; use alloc::ffi::CString; pub use crate::bindings::{ - SQLITE_DBCONFIG_WRITABLE_SCHEMA as DBCONFIG_WRITABLE_SCHEMA, SQLITE_DETERMINISTIC as DETERMINISTIC, SQLITE_DIRECTONLY as DIRECTONLY, SQLITE_INDEX_CONSTRAINT_EQ as INDEX_CONSTRAINT_EQ, SQLITE_INDEX_CONSTRAINT_GE as INDEX_CONSTRAINT_GE, @@ -50,13 +49,12 @@ mod aliased { sqlite3_column_value as column_value, sqlite3_commit_hook as commit_hook, sqlite3_context_db_handle as context_db_handle, sqlite3_create_function_v2 as create_function_v2, - sqlite3_create_module_v2 as create_module_v2, sqlite3_db_config as db_config, - sqlite3_declare_vtab as declare_vtab, sqlite3_errcode as errcode, sqlite3_errmsg as errmsg, - sqlite3_error_offset as error_offset, sqlite3_exec as exec, sqlite3_finalize as finalize, - sqlite3_free as free, sqlite3_get_autocommit as get_autocommit, - sqlite3_get_auxdata as get_auxdata, sqlite3_libversion as libversion, - sqlite3_libversion_number as libversion_number, sqlite3_malloc as malloc, - sqlite3_malloc64 as malloc64, sqlite3_mutex_alloc as mutex_alloc, + sqlite3_create_module_v2 as create_module_v2, sqlite3_declare_vtab as declare_vtab, + sqlite3_errcode as errcode, sqlite3_errmsg as errmsg, sqlite3_error_offset as error_offset, + sqlite3_exec as exec, sqlite3_finalize as finalize, sqlite3_free as free, + sqlite3_get_autocommit as get_autocommit, sqlite3_get_auxdata as get_auxdata, + sqlite3_libversion as libversion, sqlite3_libversion_number as libversion_number, + sqlite3_malloc as malloc, sqlite3_malloc64 as malloc64, sqlite3_mutex_alloc as mutex_alloc, sqlite3_mutex_enter as mutex_enter, sqlite3_mutex_free as mutex_free, sqlite3_mutex_leave as mutex_leave, sqlite3_mutex_try as mutex_try, sqlite3_next_stmt as next_stmt, sqlite3_open as open, sqlite3_prepare_v2 as prepare_v2, @@ -227,16 +225,6 @@ pub fn close(db: *mut sqlite3) -> c_int { unsafe { invoke_sqlite!(close, db) } } -pub extern "C" fn db_config( - db: *mut sqlite3, - op: c_uint, - option: c_int, - result: &mut c_int, -) -> c_int { - let ptr = core::ptr::from_mut(result); - unsafe { invoke_sqlite!(db_config, db, op as c_int, option, ptr) } -} - pub fn vtab_config(db: *mut sqlite3, options: u32) -> c_int { unsafe { invoke_sqlite!(vtab_config, db, options as i32) } } diff --git a/dart/test/schema_test.dart b/dart/test/schema_test.dart index c3221aa..ad7d09f 100644 --- a/dart/test/schema_test.dart +++ b/dart/test/schema_test.dart @@ -370,7 +370,7 @@ END''', )[0].columnAt(0); expect( createTable, - 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL /* ps-managed */,"name" text)', + 'CREATE TABLE "users"(id TEXT PRIMARY KEY NOT NULL /* ps-managed */,"name" ANY) STRICT', ); final triggers = db From 60ded39ae6d88016462f691584b2feff5f521701 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 17 Sep 2026 11:49:34 +0200 Subject: [PATCH 18/19] Minor cleanup --- crates/core/src/schema/inspection.rs | 2 ++ crates/core/src/schema/management.rs | 12 +----------- crates/core/src/schema/table_info.rs | 2 +- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/crates/core/src/schema/inspection.rs b/crates/core/src/schema/inspection.rs index ba1864b..286825f 100644 --- a/crates/core/src/schema/inspection.rs +++ b/crates/core/src/schema/inspection.rs @@ -123,6 +123,8 @@ impl ExistingView { Self::drop_by_name(db, &name)?; } ViewKey::DirectTable { table_name } => { + // For json tables, dropping the view also drops the triggers. For direct tables + // where we only want to remove triggers, we need to drop them by name manually. for write in WriteType::VALUES { let mut buffer = SqlBuffer::new(); buffer.drop( diff --git a/crates/core/src/schema/management.rs b/crates/core/src/schema/management.rs index 8b396b3..c109a58 100644 --- a/crates/core/src/schema/management.rs +++ b/crates/core/src/schema/management.rs @@ -150,7 +150,7 @@ fn update_tables( db.exec_safe_str(&create_table.sql)?; if let Some(ref old_json_table) = move_data_from { - table.direct_move_from_json(db, old_json_table)?; + table.move_from_json(db, old_json_table)?; } else if !table.local_only() { // MOVE data if any table.move_from_ps_untyped(db)?; @@ -387,16 +387,6 @@ fn update_views( }; if let Some(actual_view) = existing.remove(table.view_name()) { - if wanted_view.key == actual_view.key { - if wanted_view.delete_trigger_sql == actual_view.delete_trigger_sql { - if wanted_view.update_trigger_sql == actual_view.update_trigger_sql { - if wanted_view.insert_trigger_sql == actual_view.insert_trigger_sql { - continue; - } - } - } - } - if *actual_view == wanted_view { // View exists with identical definition, don't re-create. continue; diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs index 5d11cc9..a292c6a 100644 --- a/crates/core/src/schema/table_info.rs +++ b/crates/core/src/schema/table_info.rs @@ -112,7 +112,7 @@ impl Table { Ok(()) } - pub fn direct_move_from_json( + pub fn move_from_json( &self, db: Database, json: &JsonDataSource, From 09f191b40b8c0ce92ffbfd1f25bbbe6a09f8392d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 17 Sep 2026 18:09:07 +0200 Subject: [PATCH 19/19] Support clearing managed raw tables --- crates/core/src/view_admin.rs | 37 ++++++++++++----------------------- dart/test/crud_test.dart | 23 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/crates/core/src/view_admin.rs b/crates/core/src/view_admin.rs index 702fa16..11bda86 100644 --- a/crates/core/src/view_admin.rs +++ b/crates/core/src/view_admin.rs @@ -2,8 +2,7 @@ extern crate alloc; use alloc::format; use alloc::rc::Rc; -use alloc::string::{String, ToString}; -use alloc::vec::Vec; +use alloc::string::String; use core::ffi::{c_int, c_void}; use powersync_sqlite_nostd as sqlite; @@ -13,7 +12,7 @@ use sqlite::{ResultCode, Value}; use crate::create_sqlite_text_fn; use crate::error::{PowerSyncError, Result}; use crate::migrations::{initialize_database, powersync_migrate}; -use crate::schema::inspection::ExistingView; +use crate::schema::inspection::{ExistingTable, ExistingView}; use crate::state::DatabaseState; use crate::utils::database::Database; use crate::utils::{SqlBuffer, verify_in_transaction}; @@ -88,25 +87,18 @@ DELETE FROM ps_stream_subscriptions; )?; clear_has_synced(local_db)?; - let table_glob = if flags.clear_local() { - "ps_data_*" - } else { - "ps_data__*" - }; - - let tables_stmt = local_db - .prepare_v2("SELECT name FROM sqlite_master WHERE type='table' AND name GLOB ?1")?; - tables_stmt.bind_text(1, table_glob, sqlite::Destructor::STATIC)?; - - let mut tables: Vec = alloc::vec![]; + // Pretend to be in a sync_local step when clearing raw and direct tables. For json-based tables + // we delete from underlying ps_data__ tables to sidestep crud triggers, but some tables have + // triggers directly on the table. + let _skip_crud = state.sync_local_guard(); - while tables_stmt.step()? { - let name = tables_stmt.column_text(0)?; - tables.push(name.to_string()); - } + let existing_tables = ExistingTable::list(local_db)?; + for table in &existing_tables { + if !flags.clear_local() && table.local_only { + continue; + } - for name in tables { - let quoted = SqlBuffer::quote_identifier(&name); + let quoted = SqlBuffer::quote_identifier(&table.internal_name); // The first delete statement deletes a single row, to trigger an update notification for the table. // The second delete statement uses the truncate optimization to delete the remainder of the data. let delete_sql = format!( @@ -119,11 +111,6 @@ DELETE FROM {table};", } if let Some(schema) = state.view_schema() { - // Pretend to be in a sync_local step when clearing raw tables. Similar to the case above - // where we delete from the underlying table to sidestep the CRUD trigger, we don't want - // triggers on raw tables to record this delete in ps_crud. - let _skip_crud = state.sync_local_guard(); - for raw_table in &schema.raw_tables { if let Some(stmt) = &raw_table.clear { local_db diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart index d8a9b1b..ba92c4f 100644 --- a/dart/test/crud_test.dart +++ b/dart/test/crud_test.dart @@ -981,6 +981,29 @@ INSERT INTO ps_kv(key, value) VALUES }); }); + test('can clear direct tables', () { + db.executeInTx('SELECT powersync_replace_schema(?)', [ + json.encode({ + 'tables': [ + { + 'name': 'users', + 'columns': [ + {'name': 'name', 'type': 'text'}, + ], + 'direct': true, + } + ] + }) + ]); + + db.execute( + 'INSERT INTO users (id, name) VALUES (uuid(), ?)', ['test user']); + + db.executeInTx('SELECT powersync_clear(0)'); + expect(db.select('SELECT * FROM users'), isEmpty); + expect(db.select('SELECT * FROM ps_crud'), isEmpty); + }); + group('transaction ids', () { setUp(() { db.executeInTx('select powersync_replace_schema(?)', [