@@ -38,6 +38,12 @@ service Plugin {
3838 // Assess how the destination plugin would apply table schema changes, without writing anything.
3939 // Plugins that do not support assessment return Unimplemented.
4040 rpc AssessTables (AssessTables.Request ) returns (AssessTables.Response );
41+ // Write resources, like Write, and also accept Write.MessageFlush.
42+ // The plugin sends one Write.Response per Write.MessageFlush, once it has tried to commit every message sent before
43+ // the flush. Write.Response.flush_error is set if some messages sent since the previous flush may not have been
44+ // committed.
45+ // Only use it with plugins that set Init.Response.supports_flush.
46+ rpc WriteStream (stream Write.Request ) returns (stream Write.Response );
4147}
4248
4349message GetName {
@@ -69,7 +75,10 @@ message Init {
6975 bool no_connection = 2 ; // A flag to indicate plugins should skip establishing a connection
7076 string invocation_id = 3 ; // unique execution_id that will identify the invocation (sync, migrate etc)
7177 }
72- message Response {}
78+ message Response {
79+ // The plugin supports WriteStream and Write.MessageFlush
80+ bool supports_flush = 1 ;
81+ }
7382}
7483
7584message GetTables {
@@ -102,6 +111,19 @@ message Sync {
102111 string table_name = 1 ;
103112 string error = 2 ;
104113 }
114+ // State backend rows to persist once every message sent before the checkpoint is committed to all destinations.
115+ // Only sent when Sync.Request.supports_checkpoints is set.
116+ //
117+ // The record must:
118+ // - have the cq:table_name schema metadata set to Sync.BackendOptions.table_name
119+ // - have a "key" column and a "value" column, both of the arrow utf8 type (not large_utf8)
120+ // - have a "version" column of the arrow uint64 type, if the state table has one
121+ // - hold only the keys that changed. Each row is an upsert of its key; keys not in the record keep their state.
122+ // When the same key is in several checkpoints, the row with the highest version wins, or the last row without a version column.
123+ message MessageCheckpoint {
124+ // marshalled arrow.Record
125+ bytes record = 1 ;
126+ }
105127
106128
107129 message BackendOptions {
@@ -122,13 +144,17 @@ message Sync {
122144 BackendOptions backend = 5 ;
123145 optional Shard shard = 6 ;
124146 bool withErrorMessages = 7 ; // If true, the plugin will send error messages in the response stream
147+ // If true, the client persists state backend writes from Sync.MessageCheckpoint,
148+ // so the plugin sends them on the stream instead of writing to the state backend
149+ bool supports_checkpoints = 8 ;
125150 }
126151 message Response {
127152 oneof message {
128153 Sync.MessageMigrateTable migrate_table = 1 ;
129154 Sync.MessageInsert insert = 2 ;
130155 Sync.MessageDeleteRecord delete_record = 3 ;
131156 Sync.MessageError error = 4 ;
157+ Sync.MessageCheckpoint checkpoint = 5 ;
132158 }
133159 }
134160}
@@ -198,15 +224,30 @@ message Write {
198224 repeated PredicatesGroup where_clause = 2 ;
199225 repeated TableRelation table_relations = 3 ;
200226 }
227+ // Commit every message sent before this one. Only valid on WriteStream.
228+ message MessageFlush {
229+ uint64 id = 1 ;
230+ }
201231 message Request {
202232 oneof message {
203233 Write.MessageMigrateTable migrate_table = 1 ;
204234 Write.MessageInsert insert = 2 ;
205235 Write.MessageDeleteStale delete = 3 ;
206- Write.MessageDeleteRecord delete_record = 4 ;
236+ Write.MessageDeleteRecord delete_record = 4 ;
237+ Write.MessageFlush flush = 5 ;
207238 }
208239 }
209- message Response {}
240+ message Response {
241+ // WriteStream only: id of the Write.MessageFlush that completed
242+ uint64 flush_id = 1 ;
243+ // WriteStream only: why some messages sent since the previous flush may not have been committed. A failure may be
244+ // reported only once, so a later flush can succeed although earlier messages were lost: clients must treat the
245+ // first failure as final. A plugin may end the stream after reporting a failure, or instead of reporting it.
246+ string flush_error = 2 ;
247+ // WriteStream only: with flush_error, the tables whose messages may not have been committed. Empty if the plugin
248+ // cannot tell, in which case messages of any table may have been lost.
249+ repeated string failed_tables = 3 ;
250+ }
210251}
211252
212253message Transform {
0 commit comments