From f5a0c38a84d6852e9af52bf163acc0b4756db70c Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 1 Feb 2026 19:22:17 -0800 Subject: [PATCH 01/15] add rough impl --- .../architecture/rpc/controlcommands.proto | 21 +- .../architecture/rpc/controllerservice.proto | 1 + .../architecture/rpc/workerservice.proto | 1 + .../control/update_executor_handler.py | 29 +- .../amber/engine/architecture/rpc/__init__.py | 1527 +++++++++-------- .../texera/amber/engine/common/__init__.py | 42 +- ...ControllerAsyncRPCHandlerInitializer.scala | 3 +- .../ReconfigurationHandler.scala | 105 ++ .../DataProcessorRPCHandlerInitializer.scala | 27 +- .../InitializeExecutorHandler.scala | 1 - .../UpdateExecutorHandler.scala | 63 + .../FriesReconfigurationAlgorithm.scala | 56 +- common/config/src/main/resources/storage.conf | 4 +- common/config/src/main/resources/udf.conf | 2 +- 14 files changed, 1041 insertions(+), 841 deletions(-) create mode 100644 amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala create mode 100644 amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala rename amber/src/main/scala/org/apache/texera/{web/service => amber/engine/common}/FriesReconfigurationAlgorithm.scala (71%) diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto index ed17be236dc..2ce7f9be8e5 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto @@ -25,7 +25,6 @@ import "org/apache/texera/amber/engine/architecture/worker/statistics.proto"; import "org/apache/texera/amber/engine/architecture/sendsemantics/partitionings.proto"; import "scalapb/scalapb.proto"; import "google/protobuf/timestamp.proto"; -import "google/protobuf/any.proto"; option (scalapb.options) = { scope: FILE, @@ -40,12 +39,11 @@ message ControlRequest { TakeGlobalCheckpointRequest takeGlobalCheckpointRequest = 2; DebugCommandRequest debugCommandRequest = 3; EvaluatePythonExpressionRequest evaluatePythonExpressionRequest = 4; - ModifyLogicRequest modifyLogicRequest = 5; - RetryWorkflowRequest retryWorkflowRequest = 6; - ConsoleMessageTriggeredRequest consoleMessageTriggeredRequest = 8; - PortCompletedRequest portCompletedRequest = 9; - WorkerStateUpdatedRequest workerStateUpdatedRequest = 10; - LinkWorkersRequest linkWorkersRequest = 11; + RetryWorkflowRequest retryWorkflowRequest = 5; + ConsoleMessageTriggeredRequest consoleMessageTriggeredRequest = 6; + PortCompletedRequest portCompletedRequest = 7; + WorkerStateUpdatedRequest workerStateUpdatedRequest = 8; + LinkWorkersRequest linkWorkersRequest = 9; // request for worker AddInputChannelRequest addInputChannelRequest = 50; @@ -119,7 +117,7 @@ message TakeGlobalCheckpointRequest { } message WorkflowReconfigureRequest{ - ModifyLogicRequest reconfiguration = 1 [(scalapb.field).no_box = true]; + repeated UpdateExecutorRequest reconfiguration = 1; string reconfigurationId = 2; } @@ -134,10 +132,6 @@ message EvaluatePythonExpressionRequest { string operatorId = 2; } -message ModifyLogicRequest { - repeated UpdateExecutorRequest updateRequest = 1; -} - message RetryWorkflowRequest { repeated core.ActorVirtualIdentity workers = 1; } @@ -260,8 +254,7 @@ message InitializeExecutorRequest { message UpdateExecutorRequest { core.PhysicalOpIdentity targetOpId = 1 [(scalapb.field).no_box = true]; - google.protobuf.Any newExecutor = 2 [(scalapb.field).no_box = true]; - google.protobuf.Any stateTransferFunc = 3; + core.OpExecInitInfo newExecInitInfo = 2; } message PrepareCheckpointRequest{ diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto index 70d189a3411..27b4727ee98 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto @@ -45,4 +45,5 @@ service ControllerService { rpc LinkWorkers(LinkWorkersRequest) returns (EmptyReturn); rpc ControllerInitiateQueryStatistics(QueryStatisticsRequest) returns (EmptyReturn); rpc RetryWorkflow(RetryWorkflowRequest) returns (EmptyReturn); + rpc ReconfigureWorkflow(WorkflowReconfigureRequest) returns (EmptyReturn); } \ No newline at end of file diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto index dbcd6d8a5e0..21944ffefc6 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/workerservice.proto @@ -50,4 +50,5 @@ service WorkerService { rpc DebugCommand(DebugCommandRequest) returns (EmptyReturn); rpc EvaluatePythonExpression(EvaluatePythonExpressionRequest) returns (EvaluatedValue); rpc NoOperation(EmptyRequest) returns (EmptyReturn); + rpc UpdateExecutor(UpdateExecutorRequest) returns (EmptyReturn); } \ No newline at end of file diff --git a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py index fe33b2dec0d..a5bc1a6c0fc 100644 --- a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py +++ b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py @@ -1,28 +1 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# from proto.org.apache.texera.amber.engine.architecture.worker import UpdateExecutorV2 -# from core.architecture.handlers.control.control_handler_base import ControlHandler -# from core.architecture.managers.context import Context -# -# -# class UpdateExecutorHandler(ControlHandler): -# cmd = UpdateExecutorV2 -# -# def __call__(self, context: Context, command: cmd, *args, **kwargs): -# context.executor_manager.update_executor(command.code, command.is_source) -# return None +# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from core.architecture.handlers.control.control_handler_base import ControlHandler from proto.org.apache.texera.amber.engine.architecture.rpc import ( EmptyReturn, UpdateExecutorRequest, ) class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: self.context.executor_manager.update_executor(req.code, req.is_source) return EmptyReturn() \ No newline at end of file diff --git a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py index ea6ddc5e43f..f7b89f13e63 100644 --- a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py +++ b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py @@ -13,7 +13,6 @@ ) import betterproto -import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf import grpclib from betterproto.grpc.grpclib_server import ServiceBase @@ -23,7 +22,6 @@ worker as _worker__, ) - if TYPE_CHECKING: import grpclib.server from betterproto.grpc.grpclib_client import MetadataLike @@ -78,23 +76,20 @@ class ControlRequest(betterproto.Message): evaluate_python_expression_request: "EvaluatePythonExpressionRequest" = ( betterproto.message_field(4, group="sealed_value") ) - modify_logic_request: "ModifyLogicRequest" = betterproto.message_field( - 5, group="sealed_value" - ) retry_workflow_request: "RetryWorkflowRequest" = betterproto.message_field( - 6, group="sealed_value" + 5, group="sealed_value" ) console_message_triggered_request: "ConsoleMessageTriggeredRequest" = ( - betterproto.message_field(8, group="sealed_value") + betterproto.message_field(6, group="sealed_value") ) port_completed_request: "PortCompletedRequest" = betterproto.message_field( - 9, group="sealed_value" + 7, group="sealed_value" ) worker_state_updated_request: "WorkerStateUpdatedRequest" = ( - betterproto.message_field(10, group="sealed_value") + betterproto.message_field(8, group="sealed_value") ) link_workers_request: "LinkWorkersRequest" = betterproto.message_field( - 11, group="sealed_value" + 9, group="sealed_value" ) add_input_channel_request: "AddInputChannelRequest" = betterproto.message_field( 50, group="sealed_value" @@ -192,7 +187,7 @@ class TakeGlobalCheckpointRequest(betterproto.Message): @dataclass(eq=False, repr=False) class WorkflowReconfigureRequest(betterproto.Message): - reconfiguration: "ModifyLogicRequest" = betterproto.message_field(1) + reconfiguration: List["UpdateExecutorRequest"] = betterproto.message_field(1) reconfiguration_id: str = betterproto.string_field(2) @@ -208,11 +203,6 @@ class EvaluatePythonExpressionRequest(betterproto.Message): operator_id: str = betterproto.string_field(2) -@dataclass(eq=False, repr=False) -class ModifyLogicRequest(betterproto.Message): - update_request: List["UpdateExecutorRequest"] = betterproto.message_field(1) - - @dataclass(eq=False, repr=False) class RetryWorkflowRequest(betterproto.Message): workers: List["___core__.ActorVirtualIdentity"] = betterproto.message_field(1) @@ -366,10 +356,7 @@ class InitializeExecutorRequest(betterproto.Message): @dataclass(eq=False, repr=False) class UpdateExecutorRequest(betterproto.Message): target_op_id: "___core__.PhysicalOpIdentity" = betterproto.message_field(1) - new_executor: "betterproto_lib_google_protobuf.Any" = betterproto.message_field(2) - state_transfer_func: "betterproto_lib_google_protobuf.Any" = ( - betterproto.message_field(3) - ) + new_exec_init_info: "___core__.OpExecInitInfo" = betterproto.message_field(2) @dataclass(eq=False, repr=False) @@ -517,503 +504,522 @@ class WorkerMetricsResponse(betterproto.Message): metrics: "_worker__.WorkerMetrics" = betterproto.message_field(1) -class RpcTesterStub(betterproto.ServiceStub): - async def send_ping( +class ControllerServiceStub(betterproto.ServiceStub): + async def retrieve_workflow_state( self, - ping: "Ping", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "RetrieveWorkflowStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing", - ping, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState", + empty_request, + RetrieveWorkflowStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_pong( + async def propagate_embedded_control_message( self, - pong: "Pong", + propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "PropagateEmbeddedControlMessageResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong", - pong, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage", + propagate_embedded_control_message_request, + PropagateEmbeddedControlMessageResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_nested( + async def take_global_checkpoint( self, - nested: "Nested", + take_global_checkpoint_request: "TakeGlobalCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "TakeGlobalCheckpointResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested", - nested, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint", + take_global_checkpoint_request, + TakeGlobalCheckpointResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_pass( + async def debug_command( self, - pass_: "Pass", + debug_command_request: "DebugCommandRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass", - pass_, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand", + debug_command_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_error_command( + async def evaluate_python_expression( self, - error_command: "ErrorCommand", + evaluate_python_expression_request: "EvaluatePythonExpressionRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EvaluatePythonExpressionResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand", - error_command, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression", + evaluate_python_expression_request, + EvaluatePythonExpressionResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_recursion( + async def console_message_triggered( self, - recursion: "Recursion", + console_message_triggered_request: "ConsoleMessageTriggeredRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion", - recursion, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered", + console_message_triggered_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_collect( + async def port_completed( self, - collect: "Collect", + port_completed_request: "PortCompletedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect", - collect, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted", + port_completed_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_generate_number( + async def start_workflow( self, - generate_number: "GenerateNumber", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "StartWorkflowResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber", - generate_number, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow", + empty_request, + StartWorkflowResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_multi_call( + async def resume_workflow( self, - multi_call: "MultiCall", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall", - multi_call, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_chain( + async def pause_workflow( self, - chain: "Chain", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain", - chain, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - -class WorkerServiceStub(betterproto.ServiceStub): - async def add_input_channel( + async def worker_state_updated( self, - add_input_channel_request: "AddInputChannelRequest", + worker_state_updated_request: "WorkerStateUpdatedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel", - add_input_channel_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated", + worker_state_updated_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def add_partitioning( + async def worker_execution_completed( self, - add_partitioning_request: "AddPartitioningRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning", - add_partitioning_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def assign_port( + async def link_workers( self, - assign_port_request: "AssignPortRequest", + link_workers_request: "LinkWorkersRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort", - assign_port_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers", + link_workers_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def finalize_checkpoint( + async def controller_initiate_query_statistics( self, - finalize_checkpoint_request: "FinalizeCheckpointRequest", + query_statistics_request: "QueryStatisticsRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "FinalizeCheckpointResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint", - finalize_checkpoint_request, - FinalizeCheckpointResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics", + query_statistics_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def flush_network_buffer( + async def retry_workflow( self, - empty_request: "EmptyRequest", + retry_workflow_request: "RetryWorkflowRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow", + retry_workflow_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def initialize_executor( + async def reconfigure_workflow( self, - initialize_executor_request: "InitializeExecutorRequest", + workflow_reconfigure_request: "WorkflowReconfigureRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor", - initialize_executor_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ReconfigureWorkflow", + workflow_reconfigure_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def open_executor( + +class RpcTesterStub(betterproto.ServiceStub): + async def send_ping( self, - empty_request: "EmptyRequest", + ping: "Ping", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing", + ping, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def pause_worker( + async def send_pong( self, - empty_request: "EmptyRequest", + pong: "Pong", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong", + pong, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def prepare_checkpoint( + async def send_nested( self, - prepare_checkpoint_request: "PrepareCheckpointRequest", + nested: "Nested", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint", - prepare_checkpoint_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested", + nested, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def query_statistics( + async def send_pass( self, - empty_request: "EmptyRequest", + pass_: "Pass", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerMetricsResponse": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics", - empty_request, - WorkerMetricsResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass", + pass_, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def resume_worker( + async def send_error_command( self, - empty_request: "EmptyRequest", + error_command: "ErrorCommand", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand", + error_command, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retrieve_state( + async def send_recursion( self, - empty_request: "EmptyRequest", + recursion: "Recursion", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState", - empty_request, - EmptyReturn, - timeout=timeout, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion", + recursion, + StringResponse, + timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retry_current_tuple( + async def send_collect( self, - empty_request: "EmptyRequest", + collect: "Collect", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect", + collect, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_worker( + async def send_generate_number( self, - empty_request: "EmptyRequest", + generate_number: "GenerateNumber", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber", + generate_number, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def end_worker( + async def send_multi_call( self, - empty_request: "EmptyRequest", + multi_call: "MultiCall", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall", + multi_call, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_channel( + async def send_chain( self, - empty_request: "EmptyRequest", + chain: "Chain", + *, + timeout: Optional[float] = None, + deadline: Optional["Deadline"] = None, + metadata: Optional["MetadataLike"] = None + ) -> "StringResponse": + return await self._unary_unary( + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain", + chain, + StringResponse, + timeout=timeout, + deadline=deadline, + metadata=metadata, + ) + + +class WorkerServiceStub(betterproto.ServiceStub): + async def add_input_channel( + self, + add_input_channel_request: "AddInputChannelRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel", + add_input_channel_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def end_channel( + async def add_partitioning( self, - empty_request: "EmptyRequest", + add_partitioning_request: "AddPartitioningRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning", + add_partitioning_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def debug_command( + async def assign_port( self, - debug_command_request: "DebugCommandRequest", + assign_port_request: "AssignPortRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand", - debug_command_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort", + assign_port_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def evaluate_python_expression( + async def finalize_checkpoint( self, - evaluate_python_expression_request: "EvaluatePythonExpressionRequest", + finalize_checkpoint_request: "FinalizeCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EvaluatedValue": + ) -> "FinalizeCheckpointResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression", - evaluate_python_expression_request, - EvaluatedValue, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint", + finalize_checkpoint_request, + FinalizeCheckpointResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def no_operation( + async def flush_network_buffer( self, empty_request: "EmptyRequest", *, @@ -1022,7 +1028,7 @@ async def no_operation( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer", empty_request, EmptyReturn, timeout=timeout, @@ -1030,145 +1036,160 @@ async def no_operation( metadata=metadata, ) + async def initialize_executor( + self, + initialize_executor_request: "InitializeExecutorRequest", + *, + timeout: Optional[float] = None, + deadline: Optional["Deadline"] = None, + metadata: Optional["MetadataLike"] = None + ) -> "EmptyReturn": + return await self._unary_unary( + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor", + initialize_executor_request, + EmptyReturn, + timeout=timeout, + deadline=deadline, + metadata=metadata, + ) -class ControllerServiceStub(betterproto.ServiceStub): - async def retrieve_workflow_state( + async def open_executor( self, empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "RetrieveWorkflowStateResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor", empty_request, - RetrieveWorkflowStateResponse, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def propagate_embedded_control_message( + async def pause_worker( self, - propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "PropagateEmbeddedControlMessageResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage", - propagate_embedded_control_message_request, - PropagateEmbeddedControlMessageResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def take_global_checkpoint( + async def prepare_checkpoint( self, - take_global_checkpoint_request: "TakeGlobalCheckpointRequest", + prepare_checkpoint_request: "PrepareCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "TakeGlobalCheckpointResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint", - take_global_checkpoint_request, - TakeGlobalCheckpointResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint", + prepare_checkpoint_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def debug_command( + async def query_statistics( self, - debug_command_request: "DebugCommandRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "WorkerMetricsResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand", - debug_command_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics", + empty_request, + WorkerMetricsResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def evaluate_python_expression( + async def resume_worker( self, - evaluate_python_expression_request: "EvaluatePythonExpressionRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EvaluatePythonExpressionResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression", - evaluate_python_expression_request, - EvaluatePythonExpressionResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def console_message_triggered( + async def retrieve_state( self, - console_message_triggered_request: "ConsoleMessageTriggeredRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered", - console_message_triggered_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def port_completed( + async def retry_current_tuple( self, - port_completed_request: "PortCompletedRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted", - port_completed_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_workflow( + async def start_worker( self, empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StartWorkflowResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker", empty_request, - StartWorkflowResponse, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def resume_workflow( + async def end_worker( self, empty_request: "EmptyRequest", *, @@ -1177,7 +1198,7 @@ async def resume_workflow( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker", empty_request, EmptyReturn, timeout=timeout, @@ -1185,7 +1206,7 @@ async def resume_workflow( metadata=metadata, ) - async def pause_workflow( + async def start_channel( self, empty_request: "EmptyRequest", *, @@ -1194,7 +1215,7 @@ async def pause_workflow( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow", + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel", empty_request, EmptyReturn, timeout=timeout, @@ -1202,85 +1223,85 @@ async def pause_workflow( metadata=metadata, ) - async def worker_state_updated( + async def end_channel( self, - worker_state_updated_request: "WorkerStateUpdatedRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated", - worker_state_updated_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def worker_execution_completed( + async def debug_command( self, - empty_request: "EmptyRequest", + debug_command_request: "DebugCommandRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand", + debug_command_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def link_workers( + async def evaluate_python_expression( self, - link_workers_request: "LinkWorkersRequest", + evaluate_python_expression_request: "EvaluatePythonExpressionRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "EvaluatedValue": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers", - link_workers_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression", + evaluate_python_expression_request, + EvaluatedValue, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def controller_initiate_query_statistics( + async def no_operation( self, - query_statistics_request: "QueryStatisticsRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics", - query_statistics_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retry_workflow( + async def update_executor( self, - retry_workflow_request: "RetryWorkflowRequest", + update_executor_request: "UpdateExecutorRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow", - retry_workflow_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/UpdateExecutor", + update_executor_request, EmptyReturn, timeout=timeout, deadline=deadline, @@ -1288,806 +1309,842 @@ async def retry_workflow( ) -class RpcTesterBase(ServiceBase): +class ControllerServiceBase(ServiceBase): - async def send_ping(self, ping: "Ping") -> "IntResponse": + async def retrieve_workflow_state( + self, empty_request: "EmptyRequest" + ) -> "RetrieveWorkflowStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_pong(self, pong: "Pong") -> "IntResponse": + async def propagate_embedded_control_message( + self, + propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", + ) -> "PropagateEmbeddedControlMessageResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_nested(self, nested: "Nested") -> "StringResponse": + async def take_global_checkpoint( + self, take_global_checkpoint_request: "TakeGlobalCheckpointRequest" + ) -> "TakeGlobalCheckpointResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_pass(self, pass_: "Pass") -> "StringResponse": + async def debug_command( + self, debug_command_request: "DebugCommandRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_error_command( - self, error_command: "ErrorCommand" - ) -> "StringResponse": + async def evaluate_python_expression( + self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" + ) -> "EvaluatePythonExpressionResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_recursion(self, recursion: "Recursion") -> "StringResponse": + async def console_message_triggered( + self, console_message_triggered_request: "ConsoleMessageTriggeredRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_collect(self, collect: "Collect") -> "StringResponse": + async def port_completed( + self, port_completed_request: "PortCompletedRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_generate_number( - self, generate_number: "GenerateNumber" - ) -> "IntResponse": + async def start_workflow( + self, empty_request: "EmptyRequest" + ) -> "StartWorkflowResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_multi_call(self, multi_call: "MultiCall") -> "StringResponse": + async def resume_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_chain(self, chain: "Chain") -> "StringResponse": + async def pause_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_send_ping( - self, stream: "grpclib.server.Stream[Ping, IntResponse]" + async def worker_state_updated( + self, worker_state_updated_request: "WorkerStateUpdatedRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def worker_execution_completed( + self, empty_request: "EmptyRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def link_workers( + self, link_workers_request: "LinkWorkersRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def controller_initiate_query_statistics( + self, query_statistics_request: "QueryStatisticsRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def retry_workflow( + self, retry_workflow_request: "RetryWorkflowRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def reconfigure_workflow( + self, workflow_reconfigure_request: "WorkflowReconfigureRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def __rpc_retrieve_workflow_state( + self, + stream: "grpclib.server.Stream[EmptyRequest, RetrieveWorkflowStateResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_ping(request) + response = await self.retrieve_workflow_state(request) await stream.send_message(response) - async def __rpc_send_pong( - self, stream: "grpclib.server.Stream[Pong, IntResponse]" + async def __rpc_propagate_embedded_control_message( + self, + stream: "grpclib.server.Stream[PropagateEmbeddedControlMessageRequest, PropagateEmbeddedControlMessageResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_pong(request) + response = await self.propagate_embedded_control_message(request) await stream.send_message(response) - async def __rpc_send_nested( - self, stream: "grpclib.server.Stream[Nested, StringResponse]" + async def __rpc_take_global_checkpoint( + self, + stream: "grpclib.server.Stream[TakeGlobalCheckpointRequest, TakeGlobalCheckpointResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_nested(request) + response = await self.take_global_checkpoint(request) await stream.send_message(response) - async def __rpc_send_pass( - self, stream: "grpclib.server.Stream[Pass, StringResponse]" + async def __rpc_debug_command( + self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_pass(request) + response = await self.debug_command(request) await stream.send_message(response) - async def __rpc_send_error_command( - self, stream: "grpclib.server.Stream[ErrorCommand, StringResponse]" + async def __rpc_evaluate_python_expression( + self, + stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatePythonExpressionResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_error_command(request) + response = await self.evaluate_python_expression(request) await stream.send_message(response) - async def __rpc_send_recursion( - self, stream: "grpclib.server.Stream[Recursion, StringResponse]" + async def __rpc_console_message_triggered( + self, + stream: "grpclib.server.Stream[ConsoleMessageTriggeredRequest, EmptyReturn]", ) -> None: request = await stream.recv_message() - response = await self.send_recursion(request) + response = await self.console_message_triggered(request) await stream.send_message(response) - async def __rpc_send_collect( - self, stream: "grpclib.server.Stream[Collect, StringResponse]" + async def __rpc_port_completed( + self, stream: "grpclib.server.Stream[PortCompletedRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_collect(request) + response = await self.port_completed(request) await stream.send_message(response) - async def __rpc_send_generate_number( - self, stream: "grpclib.server.Stream[GenerateNumber, IntResponse]" + async def __rpc_start_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, StartWorkflowResponse]" ) -> None: request = await stream.recv_message() - response = await self.send_generate_number(request) + response = await self.start_workflow(request) await stream.send_message(response) - async def __rpc_send_multi_call( - self, stream: "grpclib.server.Stream[MultiCall, StringResponse]" + async def __rpc_resume_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_multi_call(request) + response = await self.resume_workflow(request) await stream.send_message(response) - async def __rpc_send_chain( - self, stream: "grpclib.server.Stream[Chain, StringResponse]" + async def __rpc_pause_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_chain(request) + response = await self.pause_workflow(request) + await stream.send_message(response) + + async def __rpc_worker_state_updated( + self, stream: "grpclib.server.Stream[WorkerStateUpdatedRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.worker_state_updated(request) + await stream.send_message(response) + + async def __rpc_worker_execution_completed( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.worker_execution_completed(request) + await stream.send_message(response) + + async def __rpc_link_workers( + self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.link_workers(request) + await stream.send_message(response) + + async def __rpc_controller_initiate_query_statistics( + self, stream: "grpclib.server.Stream[QueryStatisticsRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.controller_initiate_query_statistics(request) + await stream.send_message(response) + + async def __rpc_retry_workflow( + self, stream: "grpclib.server.Stream[RetryWorkflowRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.retry_workflow(request) + await stream.send_message(response) + + async def __rpc_reconfigure_workflow( + self, stream: "grpclib.server.Stream[WorkflowReconfigureRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.reconfigure_workflow(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing": grpclib.const.Handler( - self.__rpc_send_ping, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState": grpclib.const.Handler( + self.__rpc_retrieve_workflow_state, grpclib.const.Cardinality.UNARY_UNARY, - Ping, - IntResponse, + EmptyRequest, + RetrieveWorkflowStateResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong": grpclib.const.Handler( - self.__rpc_send_pong, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage": grpclib.const.Handler( + self.__rpc_propagate_embedded_control_message, grpclib.const.Cardinality.UNARY_UNARY, - Pong, - IntResponse, + PropagateEmbeddedControlMessageRequest, + PropagateEmbeddedControlMessageResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested": grpclib.const.Handler( - self.__rpc_send_nested, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint": grpclib.const.Handler( + self.__rpc_take_global_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - Nested, - StringResponse, + TakeGlobalCheckpointRequest, + TakeGlobalCheckpointResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass": grpclib.const.Handler( - self.__rpc_send_pass, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand": grpclib.const.Handler( + self.__rpc_debug_command, grpclib.const.Cardinality.UNARY_UNARY, - Pass, - StringResponse, + DebugCommandRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand": grpclib.const.Handler( - self.__rpc_send_error_command, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression": grpclib.const.Handler( + self.__rpc_evaluate_python_expression, grpclib.const.Cardinality.UNARY_UNARY, - ErrorCommand, - StringResponse, + EvaluatePythonExpressionRequest, + EvaluatePythonExpressionResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion": grpclib.const.Handler( - self.__rpc_send_recursion, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered": grpclib.const.Handler( + self.__rpc_console_message_triggered, grpclib.const.Cardinality.UNARY_UNARY, - Recursion, - StringResponse, + ConsoleMessageTriggeredRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect": grpclib.const.Handler( - self.__rpc_send_collect, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted": grpclib.const.Handler( + self.__rpc_port_completed, grpclib.const.Cardinality.UNARY_UNARY, - Collect, - StringResponse, + PortCompletedRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber": grpclib.const.Handler( - self.__rpc_send_generate_number, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow": grpclib.const.Handler( + self.__rpc_start_workflow, grpclib.const.Cardinality.UNARY_UNARY, - GenerateNumber, - IntResponse, + EmptyRequest, + StartWorkflowResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall": grpclib.const.Handler( - self.__rpc_send_multi_call, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow": grpclib.const.Handler( + self.__rpc_resume_workflow, grpclib.const.Cardinality.UNARY_UNARY, - MultiCall, - StringResponse, + EmptyRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain": grpclib.const.Handler( - self.__rpc_send_chain, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow": grpclib.const.Handler( + self.__rpc_pause_workflow, grpclib.const.Cardinality.UNARY_UNARY, - Chain, - StringResponse, + EmptyRequest, + EmptyReturn, ), - } - - -class WorkerServiceBase(ServiceBase): - - async def add_input_channel( - self, add_input_channel_request: "AddInputChannelRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def add_partitioning( - self, add_partitioning_request: "AddPartitioningRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def assign_port( - self, assign_port_request: "AssignPortRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def finalize_checkpoint( - self, finalize_checkpoint_request: "FinalizeCheckpointRequest" - ) -> "FinalizeCheckpointResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def flush_network_buffer( - self, empty_request: "EmptyRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def initialize_executor( - self, initialize_executor_request: "InitializeExecutorRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def open_executor(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def pause_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated": grpclib.const.Handler( + self.__rpc_worker_state_updated, + grpclib.const.Cardinality.UNARY_UNARY, + WorkerStateUpdatedRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted": grpclib.const.Handler( + self.__rpc_worker_execution_completed, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler( + self.__rpc_link_workers, + grpclib.const.Cardinality.UNARY_UNARY, + LinkWorkersRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics": grpclib.const.Handler( + self.__rpc_controller_initiate_query_statistics, + grpclib.const.Cardinality.UNARY_UNARY, + QueryStatisticsRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow": grpclib.const.Handler( + self.__rpc_retry_workflow, + grpclib.const.Cardinality.UNARY_UNARY, + RetryWorkflowRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ReconfigureWorkflow": grpclib.const.Handler( + self.__rpc_reconfigure_workflow, + grpclib.const.Cardinality.UNARY_UNARY, + WorkflowReconfigureRequest, + EmptyReturn, + ), + } - async def prepare_checkpoint( - self, prepare_checkpoint_request: "PrepareCheckpointRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def query_statistics( - self, empty_request: "EmptyRequest" - ) -> "WorkerMetricsResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) +class RpcTesterBase(ServiceBase): - async def resume_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": + async def send_ping(self, ping: "Ping") -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retrieve_state(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_pong(self, pong: "Pong") -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retry_current_tuple(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_nested(self, nested: "Nested") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": + async def send_pass(self, pass_: "Pass") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def end_worker(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_error_command( + self, error_command: "ErrorCommand" + ) -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_recursion(self, recursion: "Recursion") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def end_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_collect(self, collect: "Collect") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def debug_command( - self, debug_command_request: "DebugCommandRequest" - ) -> "EmptyReturn": + async def send_generate_number( + self, generate_number: "GenerateNumber" + ) -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def evaluate_python_expression( - self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" - ) -> "EvaluatedValue": + async def send_multi_call(self, multi_call: "MultiCall") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def no_operation(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_chain(self, chain: "Chain") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_add_input_channel( - self, stream: "grpclib.server.Stream[AddInputChannelRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.add_input_channel(request) - await stream.send_message(response) - - async def __rpc_add_partitioning( - self, stream: "grpclib.server.Stream[AddPartitioningRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.add_partitioning(request) - await stream.send_message(response) - - async def __rpc_assign_port( - self, stream: "grpclib.server.Stream[AssignPortRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.assign_port(request) - await stream.send_message(response) - - async def __rpc_finalize_checkpoint( - self, - stream: "grpclib.server.Stream[FinalizeCheckpointRequest, FinalizeCheckpointResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.finalize_checkpoint(request) - await stream.send_message(response) - - async def __rpc_flush_network_buffer( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.flush_network_buffer(request) - await stream.send_message(response) - - async def __rpc_initialize_executor( - self, stream: "grpclib.server.Stream[InitializeExecutorRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.initialize_executor(request) - await stream.send_message(response) - - async def __rpc_open_executor( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.open_executor(request) - await stream.send_message(response) - - async def __rpc_pause_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" - ) -> None: - request = await stream.recv_message() - response = await self.pause_worker(request) - await stream.send_message(response) - - async def __rpc_prepare_checkpoint( - self, stream: "grpclib.server.Stream[PrepareCheckpointRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.prepare_checkpoint(request) - await stream.send_message(response) - - async def __rpc_query_statistics( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerMetricsResponse]" - ) -> None: - request = await stream.recv_message() - response = await self.query_statistics(request) - await stream.send_message(response) - - async def __rpc_resume_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + async def __rpc_send_ping( + self, stream: "grpclib.server.Stream[Ping, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.resume_worker(request) + response = await self.send_ping(request) await stream.send_message(response) - async def __rpc_retrieve_state( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_pong( + self, stream: "grpclib.server.Stream[Pong, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.retrieve_state(request) + response = await self.send_pong(request) await stream.send_message(response) - async def __rpc_retry_current_tuple( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_nested( + self, stream: "grpclib.server.Stream[Nested, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.retry_current_tuple(request) + response = await self.send_nested(request) await stream.send_message(response) - async def __rpc_start_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + async def __rpc_send_pass( + self, stream: "grpclib.server.Stream[Pass, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.start_worker(request) + response = await self.send_pass(request) await stream.send_message(response) - async def __rpc_end_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_error_command( + self, stream: "grpclib.server.Stream[ErrorCommand, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.end_worker(request) + response = await self.send_error_command(request) await stream.send_message(response) - async def __rpc_start_channel( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_recursion( + self, stream: "grpclib.server.Stream[Recursion, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.start_channel(request) + response = await self.send_recursion(request) await stream.send_message(response) - async def __rpc_end_channel( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_collect( + self, stream: "grpclib.server.Stream[Collect, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.end_channel(request) + response = await self.send_collect(request) await stream.send_message(response) - async def __rpc_debug_command( - self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" + async def __rpc_send_generate_number( + self, stream: "grpclib.server.Stream[GenerateNumber, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.debug_command(request) + response = await self.send_generate_number(request) await stream.send_message(response) - async def __rpc_evaluate_python_expression( - self, - stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatedValue]", + async def __rpc_send_multi_call( + self, stream: "grpclib.server.Stream[MultiCall, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.evaluate_python_expression(request) + response = await self.send_multi_call(request) await stream.send_message(response) - async def __rpc_no_operation( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_chain( + self, stream: "grpclib.server.Stream[Chain, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.no_operation(request) + response = await self.send_chain(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel": grpclib.const.Handler( - self.__rpc_add_input_channel, - grpclib.const.Cardinality.UNARY_UNARY, - AddInputChannelRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning": grpclib.const.Handler( - self.__rpc_add_partitioning, - grpclib.const.Cardinality.UNARY_UNARY, - AddPartitioningRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort": grpclib.const.Handler( - self.__rpc_assign_port, - grpclib.const.Cardinality.UNARY_UNARY, - AssignPortRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint": grpclib.const.Handler( - self.__rpc_finalize_checkpoint, - grpclib.const.Cardinality.UNARY_UNARY, - FinalizeCheckpointRequest, - FinalizeCheckpointResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer": grpclib.const.Handler( - self.__rpc_flush_network_buffer, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor": grpclib.const.Handler( - self.__rpc_initialize_executor, - grpclib.const.Cardinality.UNARY_UNARY, - InitializeExecutorRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor": grpclib.const.Handler( - self.__rpc_open_executor, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker": grpclib.const.Handler( - self.__rpc_pause_worker, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint": grpclib.const.Handler( - self.__rpc_prepare_checkpoint, - grpclib.const.Cardinality.UNARY_UNARY, - PrepareCheckpointRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics": grpclib.const.Handler( - self.__rpc_query_statistics, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerMetricsResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker": grpclib.const.Handler( - self.__rpc_resume_worker, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing": grpclib.const.Handler( + self.__rpc_send_ping, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, + Ping, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState": grpclib.const.Handler( - self.__rpc_retrieve_state, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong": grpclib.const.Handler( + self.__rpc_send_pong, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Pong, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple": grpclib.const.Handler( - self.__rpc_retry_current_tuple, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested": grpclib.const.Handler( + self.__rpc_send_nested, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Nested, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker": grpclib.const.Handler( - self.__rpc_start_worker, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass": grpclib.const.Handler( + self.__rpc_send_pass, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, + Pass, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker": grpclib.const.Handler( - self.__rpc_end_worker, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand": grpclib.const.Handler( + self.__rpc_send_error_command, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + ErrorCommand, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel": grpclib.const.Handler( - self.__rpc_start_channel, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion": grpclib.const.Handler( + self.__rpc_send_recursion, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Recursion, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel": grpclib.const.Handler( - self.__rpc_end_channel, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect": grpclib.const.Handler( + self.__rpc_send_collect, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Collect, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand": grpclib.const.Handler( - self.__rpc_debug_command, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber": grpclib.const.Handler( + self.__rpc_send_generate_number, grpclib.const.Cardinality.UNARY_UNARY, - DebugCommandRequest, - EmptyReturn, + GenerateNumber, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression": grpclib.const.Handler( - self.__rpc_evaluate_python_expression, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall": grpclib.const.Handler( + self.__rpc_send_multi_call, grpclib.const.Cardinality.UNARY_UNARY, - EvaluatePythonExpressionRequest, - EvaluatedValue, + MultiCall, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation": grpclib.const.Handler( - self.__rpc_no_operation, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain": grpclib.const.Handler( + self.__rpc_send_chain, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Chain, + StringResponse, ), } -class ControllerServiceBase(ServiceBase): +class WorkerServiceBase(ServiceBase): - async def retrieve_workflow_state( - self, empty_request: "EmptyRequest" - ) -> "RetrieveWorkflowStateResponse": + async def add_input_channel( + self, add_input_channel_request: "AddInputChannelRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def propagate_embedded_control_message( - self, - propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", - ) -> "PropagateEmbeddedControlMessageResponse": + async def add_partitioning( + self, add_partitioning_request: "AddPartitioningRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def take_global_checkpoint( - self, take_global_checkpoint_request: "TakeGlobalCheckpointRequest" - ) -> "TakeGlobalCheckpointResponse": + async def assign_port( + self, assign_port_request: "AssignPortRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def debug_command( - self, debug_command_request: "DebugCommandRequest" - ) -> "EmptyReturn": + async def finalize_checkpoint( + self, finalize_checkpoint_request: "FinalizeCheckpointRequest" + ) -> "FinalizeCheckpointResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def evaluate_python_expression( - self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" - ) -> "EvaluatePythonExpressionResponse": + async def flush_network_buffer( + self, empty_request: "EmptyRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def console_message_triggered( - self, console_message_triggered_request: "ConsoleMessageTriggeredRequest" + async def initialize_executor( + self, initialize_executor_request: "InitializeExecutorRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def port_completed( - self, port_completed_request: "PortCompletedRequest" + async def open_executor(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def pause_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def prepare_checkpoint( + self, prepare_checkpoint_request: "PrepareCheckpointRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_workflow( + async def query_statistics( self, empty_request: "EmptyRequest" - ) -> "StartWorkflowResponse": + ) -> "WorkerMetricsResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def resume_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def resume_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def pause_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def retrieve_state(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def worker_state_updated( - self, worker_state_updated_request: "WorkerStateUpdatedRequest" - ) -> "EmptyReturn": + async def retry_current_tuple(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def worker_execution_completed( + async def start_worker( self, empty_request: "EmptyRequest" - ) -> "EmptyReturn": + ) -> "WorkerStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def link_workers( - self, link_workers_request: "LinkWorkersRequest" - ) -> "EmptyReturn": + async def end_worker(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def controller_initiate_query_statistics( - self, query_statistics_request: "QueryStatisticsRequest" + async def start_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def end_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def debug_command( + self, debug_command_request: "DebugCommandRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retry_workflow( - self, retry_workflow_request: "RetryWorkflowRequest" + async def evaluate_python_expression( + self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" + ) -> "EvaluatedValue": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def no_operation(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def update_executor( + self, update_executor_request: "UpdateExecutorRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_retrieve_workflow_state( - self, - stream: "grpclib.server.Stream[EmptyRequest, RetrieveWorkflowStateResponse]", + async def __rpc_add_input_channel( + self, stream: "grpclib.server.Stream[AddInputChannelRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.retrieve_workflow_state(request) + response = await self.add_input_channel(request) await stream.send_message(response) - async def __rpc_propagate_embedded_control_message( - self, - stream: "grpclib.server.Stream[PropagateEmbeddedControlMessageRequest, PropagateEmbeddedControlMessageResponse]", + async def __rpc_add_partitioning( + self, stream: "grpclib.server.Stream[AddPartitioningRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.propagate_embedded_control_message(request) + response = await self.add_partitioning(request) await stream.send_message(response) - async def __rpc_take_global_checkpoint( + async def __rpc_assign_port( + self, stream: "grpclib.server.Stream[AssignPortRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.assign_port(request) + await stream.send_message(response) + + async def __rpc_finalize_checkpoint( self, - stream: "grpclib.server.Stream[TakeGlobalCheckpointRequest, TakeGlobalCheckpointResponse]", + stream: "grpclib.server.Stream[FinalizeCheckpointRequest, FinalizeCheckpointResponse]", ) -> None: request = await stream.recv_message() - response = await self.take_global_checkpoint(request) + response = await self.finalize_checkpoint(request) await stream.send_message(response) - async def __rpc_debug_command( - self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" + async def __rpc_flush_network_buffer( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.debug_command(request) + response = await self.flush_network_buffer(request) await stream.send_message(response) - async def __rpc_evaluate_python_expression( - self, - stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatePythonExpressionResponse]", + async def __rpc_initialize_executor( + self, stream: "grpclib.server.Stream[InitializeExecutorRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.evaluate_python_expression(request) + response = await self.initialize_executor(request) await stream.send_message(response) - async def __rpc_console_message_triggered( - self, - stream: "grpclib.server.Stream[ConsoleMessageTriggeredRequest, EmptyReturn]", + async def __rpc_open_executor( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.console_message_triggered(request) + response = await self.open_executor(request) await stream.send_message(response) - async def __rpc_port_completed( - self, stream: "grpclib.server.Stream[PortCompletedRequest, EmptyReturn]" + async def __rpc_pause_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" ) -> None: request = await stream.recv_message() - response = await self.port_completed(request) + response = await self.pause_worker(request) await stream.send_message(response) - async def __rpc_start_workflow( - self, stream: "grpclib.server.Stream[EmptyRequest, StartWorkflowResponse]" + async def __rpc_prepare_checkpoint( + self, stream: "grpclib.server.Stream[PrepareCheckpointRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.start_workflow(request) + response = await self.prepare_checkpoint(request) await stream.send_message(response) - async def __rpc_resume_workflow( + async def __rpc_query_statistics( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerMetricsResponse]" + ) -> None: + request = await stream.recv_message() + response = await self.query_statistics(request) + await stream.send_message(response) + + async def __rpc_resume_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + ) -> None: + request = await stream.recv_message() + response = await self.resume_worker(request) + await stream.send_message(response) + + async def __rpc_retrieve_state( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.resume_workflow(request) + response = await self.retrieve_state(request) await stream.send_message(response) - async def __rpc_pause_workflow( + async def __rpc_retry_current_tuple( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.pause_workflow(request) + response = await self.retry_current_tuple(request) await stream.send_message(response) - async def __rpc_worker_state_updated( - self, stream: "grpclib.server.Stream[WorkerStateUpdatedRequest, EmptyReturn]" + async def __rpc_start_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" ) -> None: request = await stream.recv_message() - response = await self.worker_state_updated(request) + response = await self.start_worker(request) await stream.send_message(response) - async def __rpc_worker_execution_completed( + async def __rpc_end_worker( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.worker_execution_completed(request) + response = await self.end_worker(request) await stream.send_message(response) - async def __rpc_link_workers( - self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]" + async def __rpc_start_channel( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.link_workers(request) + response = await self.start_channel(request) await stream.send_message(response) - async def __rpc_controller_initiate_query_statistics( - self, stream: "grpclib.server.Stream[QueryStatisticsRequest, EmptyReturn]" + async def __rpc_end_channel( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.controller_initiate_query_statistics(request) + response = await self.end_channel(request) await stream.send_message(response) - async def __rpc_retry_workflow( - self, stream: "grpclib.server.Stream[RetryWorkflowRequest, EmptyReturn]" + async def __rpc_debug_command( + self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.retry_workflow(request) + response = await self.debug_command(request) + await stream.send_message(response) + + async def __rpc_evaluate_python_expression( + self, + stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatedValue]", + ) -> None: + request = await stream.recv_message() + response = await self.evaluate_python_expression(request) + await stream.send_message(response) + + async def __rpc_no_operation( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.no_operation(request) + await stream.send_message(response) + + async def __rpc_update_executor( + self, stream: "grpclib.server.Stream[UpdateExecutorRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.update_executor(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState": grpclib.const.Handler( - self.__rpc_retrieve_workflow_state, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel": grpclib.const.Handler( + self.__rpc_add_input_channel, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - RetrieveWorkflowStateResponse, + AddInputChannelRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage": grpclib.const.Handler( - self.__rpc_propagate_embedded_control_message, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning": grpclib.const.Handler( + self.__rpc_add_partitioning, grpclib.const.Cardinality.UNARY_UNARY, - PropagateEmbeddedControlMessageRequest, - PropagateEmbeddedControlMessageResponse, + AddPartitioningRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint": grpclib.const.Handler( - self.__rpc_take_global_checkpoint, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort": grpclib.const.Handler( + self.__rpc_assign_port, grpclib.const.Cardinality.UNARY_UNARY, - TakeGlobalCheckpointRequest, - TakeGlobalCheckpointResponse, + AssignPortRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand": grpclib.const.Handler( - self.__rpc_debug_command, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint": grpclib.const.Handler( + self.__rpc_finalize_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - DebugCommandRequest, + FinalizeCheckpointRequest, + FinalizeCheckpointResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer": grpclib.const.Handler( + self.__rpc_flush_network_buffer, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression": grpclib.const.Handler( - self.__rpc_evaluate_python_expression, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor": grpclib.const.Handler( + self.__rpc_initialize_executor, grpclib.const.Cardinality.UNARY_UNARY, - EvaluatePythonExpressionRequest, - EvaluatePythonExpressionResponse, + InitializeExecutorRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered": grpclib.const.Handler( - self.__rpc_console_message_triggered, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor": grpclib.const.Handler( + self.__rpc_open_executor, grpclib.const.Cardinality.UNARY_UNARY, - ConsoleMessageTriggeredRequest, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted": grpclib.const.Handler( - self.__rpc_port_completed, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker": grpclib.const.Handler( + self.__rpc_pause_worker, grpclib.const.Cardinality.UNARY_UNARY, - PortCompletedRequest, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint": grpclib.const.Handler( + self.__rpc_prepare_checkpoint, + grpclib.const.Cardinality.UNARY_UNARY, + PrepareCheckpointRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow": grpclib.const.Handler( - self.__rpc_start_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics": grpclib.const.Handler( + self.__rpc_query_statistics, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, - StartWorkflowResponse, + WorkerMetricsResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow": grpclib.const.Handler( - self.__rpc_resume_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker": grpclib.const.Handler( + self.__rpc_resume_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState": grpclib.const.Handler( + self.__rpc_retrieve_state, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow": grpclib.const.Handler( - self.__rpc_pause_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple": grpclib.const.Handler( + self.__rpc_retry_current_tuple, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated": grpclib.const.Handler( - self.__rpc_worker_state_updated, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker": grpclib.const.Handler( + self.__rpc_start_worker, grpclib.const.Cardinality.UNARY_UNARY, - WorkerStateUpdatedRequest, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker": grpclib.const.Handler( + self.__rpc_end_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted": grpclib.const.Handler( - self.__rpc_worker_execution_completed, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel": grpclib.const.Handler( + self.__rpc_start_channel, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler( - self.__rpc_link_workers, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel": grpclib.const.Handler( + self.__rpc_end_channel, grpclib.const.Cardinality.UNARY_UNARY, - LinkWorkersRequest, + EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics": grpclib.const.Handler( - self.__rpc_controller_initiate_query_statistics, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand": grpclib.const.Handler( + self.__rpc_debug_command, grpclib.const.Cardinality.UNARY_UNARY, - QueryStatisticsRequest, + DebugCommandRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow": grpclib.const.Handler( - self.__rpc_retry_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression": grpclib.const.Handler( + self.__rpc_evaluate_python_expression, grpclib.const.Cardinality.UNARY_UNARY, - RetryWorkflowRequest, + EvaluatePythonExpressionRequest, + EvaluatedValue, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation": grpclib.const.Handler( + self.__rpc_no_operation, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/UpdateExecutor": grpclib.const.Handler( + self.__rpc_update_executor, + grpclib.const.Cardinality.UNARY_UNARY, + UpdateExecutorRequest, EmptyReturn, ), } diff --git a/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py b/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py index 55c789aa395..8c1464cc76c 100644 --- a/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py +++ b/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py @@ -18,6 +18,27 @@ ) +@dataclass(eq=False, repr=False) +class Backpressure(betterproto.Message): + enable_backpressure: bool = betterproto.bool_field(1) + + +@dataclass(eq=False, repr=False) +class CreditUpdate(betterproto.Message): + pass + + +@dataclass(eq=False, repr=False) +class ActorCommand(betterproto.Message): + backpressure: "Backpressure" = betterproto.message_field(1, group="sealed_value") + credit_update: "CreditUpdate" = betterproto.message_field(2, group="sealed_value") + + +@dataclass(eq=False, repr=False) +class PythonActorMessage(betterproto.Message): + payload: "ActorCommand" = betterproto.message_field(1) + + @dataclass(eq=False, repr=False) class DirectControlMessagePayloadV2(betterproto.Message): control_invocation: "_architecture_rpc__.ControlInvocation" = ( @@ -133,24 +154,3 @@ class ExecutionMetadataStore(betterproto.Message): fatal_errors: List["__core__.WorkflowFatalError"] = betterproto.message_field(2) execution_id: "__core__.ExecutionIdentity" = betterproto.message_field(3) is_recovering: bool = betterproto.bool_field(4) - - -@dataclass(eq=False, repr=False) -class Backpressure(betterproto.Message): - enable_backpressure: bool = betterproto.bool_field(1) - - -@dataclass(eq=False, repr=False) -class CreditUpdate(betterproto.Message): - pass - - -@dataclass(eq=False, repr=False) -class ActorCommand(betterproto.Message): - backpressure: "Backpressure" = betterproto.message_field(1, group="sealed_value") - credit_update: "CreditUpdate" = betterproto.message_field(2, group="sealed_value") - - -@dataclass(eq=False, repr=False) -class PythonActorMessage(betterproto.Message): - payload: "ActorCommand" = betterproto.message_field(1) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala index 4d9a36bab43..2902173364e 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala @@ -46,6 +46,7 @@ class ControllerAsyncRPCHandlerInitializer( with DebugCommandHandler with TakeGlobalCheckpointHandler with EmbeddedControlMessageHandler - with RetrieveWorkflowStateHandler { + with RetrieveWorkflowStateHandler + with ReconfigurationHandler { val actorId: ActorVirtualIdentity = cp.actorId } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala new file mode 100644 index 00000000000..9470336efb5 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.architecture.controller.promisehandlers + +import com.twitter.util.Future +import org.apache.texera.amber.core.virtualidentity.{ChannelIdentity, EmbeddedControlMessageIdentity} +import org.apache.texera.amber.engine.architecture.controller.{ControllerAsyncRPCHandlerInitializer, ExecutionStatsUpdate} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.ALL_ALIGNMENT +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, WorkflowReconfigureRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn +import org.apache.texera.amber.engine.common.FriesReconfigurationAlgorithm +import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER +import org.apache.texera.amber.util.VirtualIdentityUtils + +trait ReconfigurationHandler { + this: ControllerAsyncRPCHandlerInitializer => + + override def reconfigureWorkflow(msg: WorkflowReconfigureRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { + FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg).foreach{ + friesComponent => + if(friesComponent.scope.size == 1){ + val updateExecutorRequest = friesComponent.reconfigurations.head + val workerIds = cp.workflowExecution.getLatestOperatorExecution(updateExecutorRequest.targetOpId).getWorkerIds + workerIds.foreach{ + worker => + workerInterface.updateExecutor(updateExecutorRequest, mkContext(worker)) + } + }else{ + val channelScope = cp.workflowExecution.getRunningRegionExecutions + .flatMap(regionExecution => + regionExecution.getAllLinkExecutions + .map(_._2) + .flatMap(linkExecution => linkExecution.getAllChannelExecutions.map(_._1)) + ).filter(channelId => { + friesComponent.scope + .contains(VirtualIdentityUtils.getPhysicalOpId(channelId.fromWorkerId)) && + friesComponent.scope + .contains(VirtualIdentityUtils.getPhysicalOpId(channelId.toWorkerId)) + }) + val controlChannels = friesComponent.sources.flatMap { source => + cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.flatMap { worker => + Seq( + ChannelIdentity(CONTROLLER, worker, isControl = true), + ChannelIdentity(worker, CONTROLLER, isControl = true) + ) + } + } + val finalScope = channelScope ++ controlChannels + val cmdMapping = + friesComponent.reconfigurations + .flatMap { updateReq => + val workers = + cp.workflowExecution + .getLatestOperatorExecution(updateReq.targetOpId) + .getWorkerIds + + workers.map { worker => + worker.name -> + createInvocation( + METHOD_UPDATE_EXECUTOR.getBareMethodName, + updateReq.newExecInitInfo, + worker + ) + } + } + .groupBy(_._1) + .map { + case (worker, entries) => + worker -> entries.head._2 + } + friesComponent.sources.foreach { source => + cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.foreach { worker => + sendECM( + EmbeddedControlMessageIdentity(msg.reconfigurationId), + ALL_ALIGNMENT, + finalScope.toSet, + cmdMapping, + ChannelIdentity(actorId, worker, isControl = true) + ) + } + } + } + + } + EmptyReturn() + } + +} diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala index 2abcdf66975..545d0c60f14 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala @@ -20,18 +20,17 @@ package org.apache.texera.amber.engine.architecture.worker import com.twitter.util.Future +import org.apache.texera.amber.core.executor.{ExecFactory, OpExecInitInfo, OpExecSource, OpExecWithClassName, OpExecWithCode} import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ - AsyncRPCContext, - DebugCommandRequest, - EmptyRequest, - EvaluatePythonExpressionRequest -} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, DebugCommandRequest, EmptyRequest, EvaluatePythonExpressionRequest} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{EmptyReturn, EvaluatedValue} import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceFs2Grpc import org.apache.texera.amber.engine.architecture.worker.promisehandlers._ import org.apache.texera.amber.engine.common.AmberLogging import org.apache.texera.amber.engine.common.rpc.AsyncRPCHandlerInitializer +import org.apache.texera.amber.operator.source.cache.CacheSourceOpExec + +import java.net.URI class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) extends AsyncRPCHandlerInitializer(dp.asyncRPCClient, dp.asyncRPCServer) @@ -52,7 +51,8 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) with FlushNetworkBufferHandler with RetrieveStateHandler with PrepareCheckpointHandler - with FinalizeCheckpointHandler { + with FinalizeCheckpointHandler + with UpdateExecutorHandler { val actorId: ActorVirtualIdentity = dp.actorId override def debugCommand( @@ -69,4 +69,17 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) ??? override def noOperation(request: EmptyRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = ??? + + def initializeExecutor(execInitInfo: OpExecInitInfo): Unit = { + dp.executor = execInitInfo match { + case OpExecWithClassName(className, descString) => + ExecFactory.newExecFromJavaClassName(className, descString, workerIdx, workerCount) + case OpExecWithCode(code, _) => + ExecFactory.newExecFromJavaCode(code) + case OpExecSource(storageUri, _) => + new CacheSourceOpExec(URI.create(storageUri)) + case OpExecInitInfo.Empty => + throw new IllegalArgumentException("Empty executor initialization info") + } + } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index bf45d8eff9a..12047101817 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -54,5 +54,4 @@ trait InitializeExecutorHandler { } EmptyReturn() } - } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala new file mode 100644 index 00000000000..7f9e1fa31d8 --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.architecture.worker.promisehandlers + +import com.twitter.util.Future +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, UpdateExecutorRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn +import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer +import scala.reflect.runtime.universe._ + +trait UpdateExecutorHandler { + this: DataProcessorRPCHandlerInitializer => + + override def updateExecutor( + request: UpdateExecutorRequest, + ctx: AsyncRPCContext + ): Future[EmptyReturn] = { + val oldOpExecState = dp.executor + initializeExecutor(request.newExecInitInfo) + dp.executor.open() + copyMatchingFields(oldOpExecState, dp.executor) //TBD if we really need this + EmptyReturn() + } + + private[this] def copyMatchingFields[A: TypeTag, B: TypeTag](from: A, to: B): Unit = { + val mirror = runtimeMirror(from.getClass.getClassLoader) + + val fromFields = typeOf[A].members.collect { + case m: MethodSymbol if m.isGetter => m + } + + val toFields = typeOf[B].members.collect { + case m: MethodSymbol if m.isVar => m + }.map(m => m.name.toString -> m).toMap + + fromFields.foreach { f => + val name = f.name.toString + toFields.get(name).foreach { setter => + if (f.returnType =:= setter.returnType) { + val value = mirror.reflect(from).reflectMethod(f).apply() + mirror.reflect(to).reflectField(setter).set(value) + } + } + } + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/service/FriesReconfigurationAlgorithm.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala similarity index 71% rename from amber/src/main/scala/org/apache/texera/web/service/FriesReconfigurationAlgorithm.scala rename to amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala index c2a15106b19..f1c82e6343b 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/FriesReconfigurationAlgorithm.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.texera.web.service +package org.apache.texera.amber.engine.common import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity import org.apache.texera.amber.core.workflow.PhysicalPlan import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ - ModifyLogicRequest, - PropagateEmbeddedControlMessageRequest + UpdateExecutorRequest, + WorkflowReconfigureRequest } import org.apache.texera.amber.engine.architecture.scheduling.{Region, WorkflowExecutionCoordinator} import org.jgrapht.alg.connectivity.ConnectivityInspector @@ -34,28 +34,32 @@ import scala.jdk.CollectionConverters.SetHasAsScala object FriesReconfigurationAlgorithm { + case class FriesComponent( + sources: Set[PhysicalOpIdentity], + scope: Set[PhysicalOpIdentity], + reconfigurations: Set[UpdateExecutorRequest]) + private def getOneToManyOperators(region: Region): Set[PhysicalOpIdentity] = { region.getOperators.filter(op => op.isOneToManyOp).map(op => op.id) } - def scheduleReconfigurations( - workflowExecutionCoordinator: WorkflowExecutionCoordinator, - reconfiguration: ModifyLogicRequest, - epochMarkerId: String - ): Set[PropagateEmbeddedControlMessageRequest] = { + def getReconfigurations( + workflowExecutionCoordinator: WorkflowExecutionCoordinator, + reconfiguration: WorkflowReconfigureRequest + ): Set[FriesComponent] = { // independently schedule reconfigurations for each region: workflowExecutionCoordinator.getExecutingRegions - .flatMap(region => computeMCS(region, reconfiguration, epochMarkerId)) + .flatMap(region => computeMCS(region, reconfiguration, reconfiguration.reconfigurationId)) } private def computeMCS( region: Region, - reconfiguration: ModifyLogicRequest, + reconfiguration: WorkflowReconfigureRequest, epochMarkerId: String - ): List[PropagateEmbeddedControlMessageRequest] = { + ): List[FriesComponent] = { // add all reconfiguration operators to M - val reconfigOps = reconfiguration.updateRequest.map(req => req.targetOpId).toSet + val reconfigOps = reconfiguration.reconfiguration.map(req => req.targetOpId).toSet val M = mutable.Set.empty ++ reconfigOps // for each one-to-many operator, add it to M if its downstream has a reconfiguration operator @@ -101,30 +105,20 @@ object FriesReconfigurationAlgorithm { // find the MCS components, // for each component, send an epoch marker to each of its source operators - val epochMarkers = new ArrayBuffer[PropagateEmbeddedControlMessageRequest]() + val epochMarkers = new ArrayBuffer[FriesComponent]() val connectedSets = new ConnectivityInspector(mcsPlan.dag).connectedSets() connectedSets.forEach(component => { val componentSet = component.asScala.toSet val componentPlan = mcsPlan.getSubPlan(componentSet) - - // generate the reconfiguration command for this component - // val reconfigCommands = - // reconfiguration.updateRequest - // .filter(req => component.contains(req.targetOpId)) - // val reconfigTargets = reconfigCommands.map(_.targetOpId) - // - // // find the source operators of the component - // val sources = componentSet.intersect(mcsPlan.getSourceOperatorIds) - // epochMarkers += PropagateEmbeddedControlMessageRequest( - // sources.toSeq, - // EmbeddedControlMessageIdentity(epochMarkerId), - // ALL_ALIGNMENT, - // componentPlan.operators.map(_.id).toSeq, - // reconfigTargets, - // ModifyLogicRequest(reconfigCommands), - // METHOD_MODIFY_LOGIC.getBareMethodName - // ) + val reconfigCommands = + reconfiguration.reconfiguration + .filter(req => component.contains(req.targetOpId)) + .toSet + + // find the source operators of the component + val sources = componentSet.intersect(mcsPlan.getSourceOperatorIds) + epochMarkers += FriesComponent(sources, componentPlan.operators.map(_.id), reconfigCommands) }) epochMarkers.toList } diff --git a/common/config/src/main/resources/storage.conf b/common/config/src/main/resources/storage.conf index 85a62b77a3b..f8b18689417 100644 --- a/common/config/src/main/resources/storage.conf +++ b/common/config/src/main/resources/storage.conf @@ -37,7 +37,7 @@ storage { username = "texera" username = ${?STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME} - password = "password" + password = "123456" password = ${?STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD} } } @@ -134,7 +134,7 @@ storage { username = "postgres" username = ${?STORAGE_JDBC_USERNAME} - password = "postgres" + password = "123456" password = ${?STORAGE_JDBC_PASSWORD} } } diff --git a/common/config/src/main/resources/udf.conf b/common/config/src/main/resources/udf.conf index 7212d1720e5..c009aed0ded 100644 --- a/common/config/src/main/resources/udf.conf +++ b/common/config/src/main/resources/udf.conf @@ -17,7 +17,7 @@ python { # python3 executable path - path = "" + path = "C:\\Users\\12198\\IdeaProjects\\texera\\.venv\\Scripts\\python.exe" path = ${?UDF_PYTHON_PATH} log { From 97e45aeaae6294ab319d29553e9a4894cd1f09d0 Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 1 Feb 2026 20:36:49 -0800 Subject: [PATCH 02/15] bug fix --- .../ReconfigurationHandler.scala | 40 +++++++------------ .../DataProcessorRPCHandlerInitializer.scala | 4 +- .../InitializeExecutorHandler.scala | 13 +----- .../UpdateExecutorHandler.scala | 28 ++----------- 4 files changed, 23 insertions(+), 62 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index 9470336efb5..573306bc065 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -24,15 +24,19 @@ import org.apache.texera.amber.core.virtualidentity.{ChannelIdentity, EmbeddedCo import org.apache.texera.amber.engine.architecture.controller.{ControllerAsyncRPCHandlerInitializer, ExecutionStatsUpdate} import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.ALL_ALIGNMENT import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, WorkflowReconfigureRequest} -import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{EmptyReturn, PropagateEmbeddedControlMessageResponse} import org.apache.texera.amber.engine.common.FriesReconfigurationAlgorithm import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER import org.apache.texera.amber.util.VirtualIdentityUtils +import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceGrpc.METHOD_UPDATE_EXECUTOR + +import scala.collection.mutable trait ReconfigurationHandler { this: ControllerAsyncRPCHandlerInitializer => override def reconfigureWorkflow(msg: WorkflowReconfigureRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { + val futures = mutable.ArrayBuffer[Future[_]]() FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg).foreach{ friesComponent => if(friesComponent.scope.size == 1){ @@ -40,7 +44,7 @@ trait ReconfigurationHandler { val workerIds = cp.workflowExecution.getLatestOperatorExecution(updateExecutorRequest.targetOpId).getWorkerIds workerIds.foreach{ worker => - workerInterface.updateExecutor(updateExecutorRequest, mkContext(worker)) + futures.append(workerInterface.updateExecutor(updateExecutorRequest, mkContext(worker))) } }else{ val channelScope = cp.workflowExecution.getRunningRegionExecutions @@ -64,42 +68,26 @@ trait ReconfigurationHandler { } val finalScope = channelScope ++ controlChannels val cmdMapping = - friesComponent.reconfigurations - .flatMap { updateReq => - val workers = - cp.workflowExecution - .getLatestOperatorExecution(updateReq.targetOpId) - .getWorkerIds - - workers.map { worker => - worker.name -> - createInvocation( - METHOD_UPDATE_EXECUTOR.getBareMethodName, - updateReq.newExecInitInfo, - worker - ) - } - } - .groupBy(_._1) - .map { - case (worker, entries) => - worker -> entries.head._2 - } + friesComponent.reconfigurations .flatMap { updateReq => + val workers = cp.workflowExecution .getLatestOperatorExecution(updateReq.targetOpId) .getWorkerIds + workers.map(worker => worker.name -> createInvocation(METHOD_UPDATE_EXECUTOR.getBareMethodName, updateReq, worker)) } .toMap + futures += cmdMapping.map(_._2._2) friesComponent.sources.foreach { source => cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.foreach { worker => sendECM( EmbeddedControlMessageIdentity(msg.reconfigurationId), ALL_ALIGNMENT, finalScope.toSet, - cmdMapping, + cmdMapping.map(x => (x._1, x._2._1)), ChannelIdentity(actorId, worker, isControl = true) ) } } } - } - EmptyReturn() + Future.collect(futures.toList).map { _ => + EmptyReturn() + } } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala index 545d0c60f14..d2837860f5a 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala @@ -55,6 +55,8 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) with UpdateExecutorHandler { val actorId: ActorVirtualIdentity = dp.actorId + var cachedTotalWorkerCount = 0 + override def debugCommand( request: DebugCommandRequest, ctx: AsyncRPCContext @@ -70,7 +72,7 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) override def noOperation(request: EmptyRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = ??? - def initializeExecutor(execInitInfo: OpExecInitInfo): Unit = { + def initializeExecutor(execInitInfo: OpExecInitInfo, workerIdx: Int, workerCount: Int): Unit = { dp.executor = execInitInfo match { case OpExecWithClassName(className, descString) => ExecFactory.newExecFromJavaClassName(className, descString, workerIdx, workerCount) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index 12047101817..f6e7f3d8b21 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -41,17 +41,8 @@ trait InitializeExecutorHandler { ): Future[EmptyReturn] = { dp.serializationManager.setOpInitialization(req) val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) - val workerCount = req.totalWorkerCount - dp.executor = req.opExecInitInfo match { - case OpExecWithClassName(className, descString) => - ExecFactory.newExecFromJavaClassName(className, descString, workerIdx, workerCount) - case OpExecWithCode(code, _) => - ExecFactory.newExecFromJavaCode(code) - case OpExecSource(storageUri, _) => - new CacheSourceOpExec(URI.create(storageUri)) - case OpExecInitInfo.Empty => - throw new IllegalArgumentException("Empty executor initialization info") - } + cachedTotalWorkerCount = req.totalWorkerCount + initializeExecutor(req.opExecInitInfo, workerIdx, cachedTotalWorkerCount) EmptyReturn() } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala index 7f9e1fa31d8..8cb0e30be21 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala @@ -23,6 +23,8 @@ import com.twitter.util.Future import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, UpdateExecutorRequest} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer +import org.apache.texera.amber.util.VirtualIdentityUtils + import scala.reflect.runtime.universe._ trait UpdateExecutorHandler { @@ -32,32 +34,10 @@ trait UpdateExecutorHandler { request: UpdateExecutorRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { - val oldOpExecState = dp.executor - initializeExecutor(request.newExecInitInfo) + val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) + initializeExecutor(request.newExecInitInfo, workerIdx, cachedTotalWorkerCount) dp.executor.open() - copyMatchingFields(oldOpExecState, dp.executor) //TBD if we really need this EmptyReturn() } - private[this] def copyMatchingFields[A: TypeTag, B: TypeTag](from: A, to: B): Unit = { - val mirror = runtimeMirror(from.getClass.getClassLoader) - - val fromFields = typeOf[A].members.collect { - case m: MethodSymbol if m.isGetter => m - } - - val toFields = typeOf[B].members.collect { - case m: MethodSymbol if m.isVar => m - }.map(m => m.name.toString -> m).toMap - - fromFields.foreach { f => - val name = f.name.toString - toFields.get(name).foreach { setter => - if (f.returnType =:= setter.returnType) { - val value = mirror.reflect(from).reflectMethod(f).apply() - mirror.reflect(to).reflectField(setter).set(value) - } - } - } - } } From d422896fba101e8406204198c06700919c0a8306 Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sat, 14 Feb 2026 19:52:17 -0800 Subject: [PATCH 03/15] add test cases --- .../architecture/rpc/controlcommands.proto | 1 + .../engine/common/rpc/AsyncRPCServer.scala | 4 +- .../amber/engine/e2e/ModifyLogicSpec.scala | 180 ++++++++++++++++++ .../texera/amber/operator/TestOperators.scala | 27 ++- 4 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto index 2ce7f9be8e5..b4658a63d05 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto @@ -44,6 +44,7 @@ message ControlRequest { PortCompletedRequest portCompletedRequest = 7; WorkerStateUpdatedRequest workerStateUpdatedRequest = 8; LinkWorkersRequest linkWorkersRequest = 9; + WorkflowReconfigureRequest workflowReconfigureRequest = 10; // request for worker AddInputChannelRequest addInputChannelRequest = 50; diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala index e9a3e2cc455..78c1ed206e7 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala @@ -86,7 +86,9 @@ class AsyncRPCServer( } catch { case e: java.lang.reflect.InvocationTargetException => throw Option(e.getCause).getOrElse(e) - case e: Throwable => throw e + case e: Throwable => + logger.info(s"error when invoking ${method.getName} + ${requestArg} with ${contextArg}") + throw e } result .asInstanceOf[Future[ControlReturn]] diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala new file mode 100644 index 00000000000..2572a78137d --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.engine.e2e + +import com.twitter.util.{Await, Duration, Promise} +import com.typesafe.scalalogging.Logger +import org.apache.pekko.actor.{ActorSystem, Props} +import org.apache.pekko.testkit.{ImplicitSender, TestKit} +import org.apache.pekko.util.Timeout +import org.apache.texera.amber.clustering.SingleNodeListener +import org.apache.texera.amber.core.executor.{OpExecInitInfo, OpExecWithCode} +import org.apache.texera.amber.core.storage.DocumentFactory +import org.apache.texera.amber.core.storage.model.VirtualDocument +import org.apache.texera.amber.core.tuple.Tuple +import org.apache.texera.amber.core.virtualidentity.OperatorIdentity +import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} +import org.apache.texera.amber.engine.architecture.controller.{ControllerConfig, ExecutionStateUpdate} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{EmptyRequest, UpdateExecutorRequest, WorkflowReconfigureRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.COMPLETED +import org.apache.texera.amber.engine.common.AmberRuntime +import org.apache.texera.amber.engine.common.client.AmberClient +import org.apache.texera.amber.engine.e2e.TestUtils.{cleanupWorkflowExecutionData, initiateTexeraDBForTestCases, setUpWorkflowExecutionData} +import org.apache.texera.amber.operator.{LogicalOp, TestOperators} +import org.apache.texera.amber.operator.TestOperators.{pythonOpDesc, pythonSourceOpDesc} +import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource.getResultUriByLogicalPortId +import org.apache.texera.workflow.LogicalLink +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome, Retries} +import org.scalatest.flatspec.AnyFlatSpecLike + +import scala.concurrent.duration._ + +class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntime.akkaConfig)) + with ImplicitSender + with AnyFlatSpecLike + with BeforeAndAfterAll + with BeforeAndAfterEach + with Retries { + + /** + * This block retries each test once if it fails. + * In the CI environment, there is a chance that executeWorkflow does not receive "COMPLETED" status. + * Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs. + */ + override def withFixture(test: NoArgTest): Outcome = + withRetry { super.withFixture(test) } + + implicit val timeout: Timeout = Timeout(5.seconds) + + val logger = Logger("ModifyLogicSpecLogger") + + override protected def beforeEach(): Unit = { + setUpWorkflowExecutionData() + } + + override protected def afterEach(): Unit = { + cleanupWorkflowExecutionData() + } + + override def beforeAll(): Unit = { + system.actorOf(Props[SingleNodeListener](), "cluster-info") + // These test cases access postgres in CI, but occasionally the jdbc driver cannot be found during CI run. + // Explicitly load the JDBC driver to avoid flaky CI failures. + Class.forName("org.postgresql.Driver") + initiateTexeraDBForTestCases() + } + + override def afterAll(): Unit = { + TestKit.shutdownActorSystem(system) + } + + + def shouldReconfigure( + operators: List[LogicalOp], + links: List[LogicalLink], + targetOp: LogicalOp, + newOpExecInitInfo: OpExecInitInfo, + checkResultLambda: (Map[OperatorIdentity, List[Tuple]]) => Boolean + ): Unit = { + val workflow = + TestUtils.buildWorkflow(operators, links, new WorkflowContext()) + val client = + new AmberClient( + system, + workflow.context, + workflow.physicalPlan, + ControllerConfig.default, + error => {} + ) + val completion = Promise[Unit]() + client + .registerCallback[ExecutionStateUpdate](evt => { + if (evt.state == COMPLETED) { +// checkResultLambda(workflow.logicalPlan.getTerminalOperatorIds +// .filter(terminalOpId => { +// val uri = getResultUriByLogicalPortId( +// workflow.context.executionId, +// terminalOpId, +// PortIdentity() +// ) +// uri.nonEmpty +// }) +// .map(terminalOpId => { +// //TODO: remove the delay after fixing the issue of reporting "completed" status too early. +// Thread.sleep(1000) +// val uri = getResultUriByLogicalPortId( +// workflow.context.executionId, +// terminalOpId, +// PortIdentity() +// ).get +// terminalOpId -> DocumentFactory +// .openDocument(uri) +// ._1 +// .asInstanceOf[VirtualDocument[Tuple]] +// .get() +// .toList +// }) +// .toMap) + completion.setDone() + } + }) + Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ())) + Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ())) + Thread.sleep(4000) + val physicalOps = workflow.physicalPlan.getPhysicalOpsOfLogicalOp(targetOp.operatorIdentifier) + assert(physicalOps.nonEmpty && physicalOps.length == 1, + "cannot reconfigure more than one physical operator in this test") + Await.result(client.controllerInterface.reconfigureWorkflow(WorkflowReconfigureRequest( + reconfiguration = + physicalOps.map(op => UpdateExecutorRequest(op.id, newOpExecInitInfo) + ), reconfigurationId = "test-reconfigure-1"), ())) + Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ())) + Thread.sleep(400) + Await.result(completion, Duration.fromMinutes(1)) + } + + + "Engine" should "be able to modify a python UDF worker in workflow" in { + val udfOpDesc = pythonOpDesc() + val sourceOpDesc = pythonSourceOpDesc(5000) + val code = """ + |from pytexera import * + | + |class ProcessTupleOperator(UDFOperatorV2): + | @overrides + | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: + | tuple_['field_2'] = tuple_['field_2'] + '_reconfigured' + | yield tuple_ + |""".stripMargin + + shouldReconfigure(List(sourceOpDesc, udfOpDesc), List(LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + udfOpDesc.operatorIdentifier, + PortIdentity() + )), + udfOpDesc, OpExecWithCode(code, "python"), + results => results(udfOpDesc.operatorIdentifier).exists { + t => t.getField("field_2").asInstanceOf[String].contains("_reconfigured") + } + ) + } + +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index ab7e8dc2de5..0f6fcd669a3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -20,11 +20,8 @@ package org.apache.texera.amber.operator import org.apache.texera.amber.core.storage.FileResolver -import org.apache.texera.amber.operator.aggregate.{ - AggregateOpDesc, - AggregationFunction, - AggregationOperation -} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType} +import org.apache.texera.amber.operator.aggregate.{AggregateOpDesc, AggregationFunction, AggregationOperation} import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc @@ -32,6 +29,7 @@ import org.apache.texera.amber.operator.source.scan.json.JSONLScanSourceOpDesc import org.apache.texera.amber.operator.source.sql.asterixdb.AsterixDBSourceOpDesc import org.apache.texera.amber.operator.source.sql.mysql.MySQLSourceOpDesc import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 +import org.apache.texera.amber.operator.udf.python.source.PythonUDFSourceOpDescV2 import java.nio.file.Path @@ -171,14 +169,33 @@ object TestOperators { def pythonOpDesc(): PythonUDFOpDescV2 = { val udf = new PythonUDFOpDescV2() udf.workers = 1 + udf.retainInputColumns = true udf.code = """ |from pytexera import * | |class ProcessTupleOperator(UDFOperatorV2): | @overrides | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: + | print(tuple_) | yield tuple_ |""".stripMargin udf } + + def pythonSourceOpDesc(num_tuple: Int): PythonUDFSourceOpDescV2 = { + val udf = new PythonUDFSourceOpDescV2() + udf.workers = 1 + udf.columns = List(new Attribute("field_1", AttributeType.INTEGER), new Attribute("field_2", AttributeType.STRING)) + udf.code = + s""" + |from pytexera import * + | + |class ProcessTupleOperator(UDFSourceOperator): + | @overrides + | def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]: + | for i in range($num_tuple): + | yield {'field_1': i, 'field_2': str(i)} + |""".stripMargin + udf + } } From 4e59c4d285b28e0567e1041b9e55c3d594d873eb Mon Sep 17 00:00:00 2001 From: Shengquan Ni <121989255@qq.com> Date: Sat, 14 Feb 2026 19:56:52 -0800 Subject: [PATCH 04/15] fix python proto --- .../amber/engine/architecture/rpc/__init__.py | 1526 +++++++++-------- .../texera/amber/engine/common/__init__.py | 42 +- 2 files changed, 786 insertions(+), 782 deletions(-) diff --git a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py index f7b89f13e63..b341f700814 100644 --- a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py +++ b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py @@ -22,6 +22,7 @@ worker as _worker__, ) + if TYPE_CHECKING: import grpclib.server from betterproto.grpc.grpclib_client import MetadataLike @@ -91,6 +92,9 @@ class ControlRequest(betterproto.Message): link_workers_request: "LinkWorkersRequest" = betterproto.message_field( 9, group="sealed_value" ) + workflow_reconfigure_request: "WorkflowReconfigureRequest" = ( + betterproto.message_field(10, group="sealed_value") + ) add_input_channel_request: "AddInputChannelRequest" = betterproto.message_field( 50, group="sealed_value" ) @@ -504,692 +508,692 @@ class WorkerMetricsResponse(betterproto.Message): metrics: "_worker__.WorkerMetrics" = betterproto.message_field(1) -class ControllerServiceStub(betterproto.ServiceStub): - async def retrieve_workflow_state( +class RpcTesterStub(betterproto.ServiceStub): + async def send_ping( self, - empty_request: "EmptyRequest", + ping: "Ping", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "RetrieveWorkflowStateResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState", - empty_request, - RetrieveWorkflowStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing", + ping, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def propagate_embedded_control_message( + async def send_pong( self, - propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", + pong: "Pong", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "PropagateEmbeddedControlMessageResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage", - propagate_embedded_control_message_request, - PropagateEmbeddedControlMessageResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong", + pong, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def take_global_checkpoint( + async def send_nested( self, - take_global_checkpoint_request: "TakeGlobalCheckpointRequest", + nested: "Nested", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "TakeGlobalCheckpointResponse": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint", - take_global_checkpoint_request, - TakeGlobalCheckpointResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested", + nested, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def debug_command( + async def send_pass( self, - debug_command_request: "DebugCommandRequest", + pass_: "Pass", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand", - debug_command_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass", + pass_, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def evaluate_python_expression( + async def send_error_command( self, - evaluate_python_expression_request: "EvaluatePythonExpressionRequest", + error_command: "ErrorCommand", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EvaluatePythonExpressionResponse": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression", - evaluate_python_expression_request, - EvaluatePythonExpressionResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand", + error_command, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def console_message_triggered( + async def send_recursion( self, - console_message_triggered_request: "ConsoleMessageTriggeredRequest", + recursion: "Recursion", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered", - console_message_triggered_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion", + recursion, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def port_completed( + async def send_collect( self, - port_completed_request: "PortCompletedRequest", + collect: "Collect", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted", - port_completed_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect", + collect, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_workflow( + async def send_generate_number( self, - empty_request: "EmptyRequest", + generate_number: "GenerateNumber", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StartWorkflowResponse": + ) -> "IntResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow", - empty_request, - StartWorkflowResponse, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber", + generate_number, + IntResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def resume_workflow( + async def send_multi_call( self, - empty_request: "EmptyRequest", + multi_call: "MultiCall", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall", + multi_call, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def pause_workflow( + async def send_chain( self, - empty_request: "EmptyRequest", + chain: "Chain", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StringResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain", + chain, + StringResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def worker_state_updated( + +class WorkerServiceStub(betterproto.ServiceStub): + async def add_input_channel( self, - worker_state_updated_request: "WorkerStateUpdatedRequest", + add_input_channel_request: "AddInputChannelRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated", - worker_state_updated_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel", + add_input_channel_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def worker_execution_completed( + async def add_partitioning( self, - empty_request: "EmptyRequest", + add_partitioning_request: "AddPartitioningRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning", + add_partitioning_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def link_workers( + async def assign_port( self, - link_workers_request: "LinkWorkersRequest", + assign_port_request: "AssignPortRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers", - link_workers_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort", + assign_port_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def controller_initiate_query_statistics( + async def finalize_checkpoint( self, - query_statistics_request: "QueryStatisticsRequest", + finalize_checkpoint_request: "FinalizeCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "FinalizeCheckpointResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics", - query_statistics_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint", + finalize_checkpoint_request, + FinalizeCheckpointResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retry_workflow( + async def flush_network_buffer( self, - retry_workflow_request: "RetryWorkflowRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow", - retry_workflow_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def reconfigure_workflow( + async def initialize_executor( self, - workflow_reconfigure_request: "WorkflowReconfigureRequest", + initialize_executor_request: "InitializeExecutorRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ReconfigureWorkflow", - workflow_reconfigure_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor", + initialize_executor_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - -class RpcTesterStub(betterproto.ServiceStub): - async def send_ping( + async def open_executor( self, - ping: "Ping", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing", - ping, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_pong( + async def pause_worker( self, - pong: "Pong", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong", - pong, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_nested( + async def prepare_checkpoint( self, - nested: "Nested", + prepare_checkpoint_request: "PrepareCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested", - nested, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint", + prepare_checkpoint_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_pass( + async def query_statistics( self, - pass_: "Pass", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "WorkerMetricsResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass", - pass_, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics", + empty_request, + WorkerMetricsResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_error_command( + async def resume_worker( self, - error_command: "ErrorCommand", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand", - error_command, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_recursion( + async def retrieve_state( self, - recursion: "Recursion", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion", - recursion, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_collect( + async def retry_current_tuple( self, - collect: "Collect", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect", - collect, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_generate_number( + async def start_worker( self, - generate_number: "GenerateNumber", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "IntResponse": + ) -> "WorkerStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber", - generate_number, - IntResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker", + empty_request, + WorkerStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_multi_call( + async def end_worker( self, - multi_call: "MultiCall", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall", - multi_call, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def send_chain( + async def start_channel( self, - chain: "Chain", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "StringResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain", - chain, - StringResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - -class WorkerServiceStub(betterproto.ServiceStub): - async def add_input_channel( + async def end_channel( self, - add_input_channel_request: "AddInputChannelRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel", - add_input_channel_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel", + empty_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def add_partitioning( + async def debug_command( self, - add_partitioning_request: "AddPartitioningRequest", + debug_command_request: "DebugCommandRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning", - add_partitioning_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand", + debug_command_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def assign_port( + async def evaluate_python_expression( self, - assign_port_request: "AssignPortRequest", + evaluate_python_expression_request: "EvaluatePythonExpressionRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "EvaluatedValue": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort", - assign_port_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression", + evaluate_python_expression_request, + EvaluatedValue, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def finalize_checkpoint( + async def no_operation( self, - finalize_checkpoint_request: "FinalizeCheckpointRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "FinalizeCheckpointResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint", - finalize_checkpoint_request, - FinalizeCheckpointResponse, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation", + empty_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def flush_network_buffer( + async def update_executor( self, - empty_request: "EmptyRequest", + update_executor_request: "UpdateExecutorRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/UpdateExecutor", + update_executor_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def initialize_executor( + +class ControllerServiceStub(betterproto.ServiceStub): + async def retrieve_workflow_state( self, - initialize_executor_request: "InitializeExecutorRequest", + empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "RetrieveWorkflowStateResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor", - initialize_executor_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState", + empty_request, + RetrieveWorkflowStateResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def open_executor( + async def propagate_embedded_control_message( self, - empty_request: "EmptyRequest", + propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "PropagateEmbeddedControlMessageResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor", - empty_request, - EmptyReturn, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage", + propagate_embedded_control_message_request, + PropagateEmbeddedControlMessageResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def pause_worker( + async def take_global_checkpoint( self, - empty_request: "EmptyRequest", + take_global_checkpoint_request: "TakeGlobalCheckpointRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "TakeGlobalCheckpointResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint", + take_global_checkpoint_request, + TakeGlobalCheckpointResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def prepare_checkpoint( + async def debug_command( self, - prepare_checkpoint_request: "PrepareCheckpointRequest", + debug_command_request: "DebugCommandRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint", - prepare_checkpoint_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand", + debug_command_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def query_statistics( + async def evaluate_python_expression( self, - empty_request: "EmptyRequest", + evaluate_python_expression_request: "EvaluatePythonExpressionRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerMetricsResponse": + ) -> "EvaluatePythonExpressionResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics", - empty_request, - WorkerMetricsResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression", + evaluate_python_expression_request, + EvaluatePythonExpressionResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def resume_worker( + async def console_message_triggered( self, - empty_request: "EmptyRequest", + console_message_triggered_request: "ConsoleMessageTriggeredRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker", - empty_request, - WorkerStateResponse, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered", + console_message_triggered_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retrieve_state( + async def port_completed( self, - empty_request: "EmptyRequest", + port_completed_request: "PortCompletedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted", + port_completed_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def retry_current_tuple( + async def start_workflow( self, empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EmptyReturn": + ) -> "StartWorkflowResponse": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple", + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow", empty_request, - EmptyReturn, + StartWorkflowResponse, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def start_worker( + async def resume_workflow( self, empty_request: "EmptyRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "WorkerStateResponse": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker", + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow", empty_request, - WorkerStateResponse, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def end_worker( + async def pause_workflow( self, empty_request: "EmptyRequest", *, @@ -1198,7 +1202,7 @@ async def end_worker( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker", + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow", empty_request, EmptyReturn, timeout=timeout, @@ -1206,24 +1210,24 @@ async def end_worker( metadata=metadata, ) - async def start_channel( + async def worker_state_updated( self, - empty_request: "EmptyRequest", + worker_state_updated_request: "WorkerStateUpdatedRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated", + worker_state_updated_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def end_channel( + async def worker_execution_completed( self, empty_request: "EmptyRequest", *, @@ -1232,7 +1236,7 @@ async def end_channel( metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel", + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted", empty_request, EmptyReturn, timeout=timeout, @@ -1240,68 +1244,68 @@ async def end_channel( metadata=metadata, ) - async def debug_command( + async def link_workers( self, - debug_command_request: "DebugCommandRequest", + link_workers_request: "LinkWorkersRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand", - debug_command_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers", + link_workers_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def evaluate_python_expression( + async def controller_initiate_query_statistics( self, - evaluate_python_expression_request: "EvaluatePythonExpressionRequest", + query_statistics_request: "QueryStatisticsRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None - ) -> "EvaluatedValue": + ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression", - evaluate_python_expression_request, - EvaluatedValue, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics", + query_statistics_request, + EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def no_operation( + async def retry_workflow( self, - empty_request: "EmptyRequest", + retry_workflow_request: "RetryWorkflowRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation", - empty_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow", + retry_workflow_request, EmptyReturn, timeout=timeout, deadline=deadline, metadata=metadata, ) - async def update_executor( + async def reconfigure_workflow( self, - update_executor_request: "UpdateExecutorRequest", + workflow_reconfigure_request: "WorkflowReconfigureRequest", *, timeout: Optional[float] = None, deadline: Optional["Deadline"] = None, metadata: Optional["MetadataLike"] = None ) -> "EmptyReturn": return await self._unary_unary( - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/UpdateExecutor", - update_executor_request, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ReconfigureWorkflow", + workflow_reconfigure_request, EmptyReturn, timeout=timeout, deadline=deadline, @@ -1309,842 +1313,842 @@ async def update_executor( ) -class ControllerServiceBase(ServiceBase): +class RpcTesterBase(ServiceBase): - async def retrieve_workflow_state( - self, empty_request: "EmptyRequest" - ) -> "RetrieveWorkflowStateResponse": + async def send_ping(self, ping: "Ping") -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def propagate_embedded_control_message( - self, - propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", - ) -> "PropagateEmbeddedControlMessageResponse": + async def send_pong(self, pong: "Pong") -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def take_global_checkpoint( - self, take_global_checkpoint_request: "TakeGlobalCheckpointRequest" - ) -> "TakeGlobalCheckpointResponse": + async def send_nested(self, nested: "Nested") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def debug_command( - self, debug_command_request: "DebugCommandRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def evaluate_python_expression( - self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" - ) -> "EvaluatePythonExpressionResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def console_message_triggered( - self, console_message_triggered_request: "ConsoleMessageTriggeredRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def port_completed( - self, port_completed_request: "PortCompletedRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def start_workflow( - self, empty_request: "EmptyRequest" - ) -> "StartWorkflowResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def resume_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def pause_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def send_pass(self, pass_: "Pass") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def worker_state_updated( - self, worker_state_updated_request: "WorkerStateUpdatedRequest" - ) -> "EmptyReturn": + async def send_error_command( + self, error_command: "ErrorCommand" + ) -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def worker_execution_completed( - self, empty_request: "EmptyRequest" - ) -> "EmptyReturn": + async def send_recursion(self, recursion: "Recursion") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def link_workers( - self, link_workers_request: "LinkWorkersRequest" - ) -> "EmptyReturn": + async def send_collect(self, collect: "Collect") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def controller_initiate_query_statistics( - self, query_statistics_request: "QueryStatisticsRequest" - ) -> "EmptyReturn": + async def send_generate_number( + self, generate_number: "GenerateNumber" + ) -> "IntResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retry_workflow( - self, retry_workflow_request: "RetryWorkflowRequest" - ) -> "EmptyReturn": + async def send_multi_call(self, multi_call: "MultiCall") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def reconfigure_workflow( - self, workflow_reconfigure_request: "WorkflowReconfigureRequest" - ) -> "EmptyReturn": + async def send_chain(self, chain: "Chain") -> "StringResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_retrieve_workflow_state( - self, - stream: "grpclib.server.Stream[EmptyRequest, RetrieveWorkflowStateResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.retrieve_workflow_state(request) - await stream.send_message(response) - - async def __rpc_propagate_embedded_control_message( - self, - stream: "grpclib.server.Stream[PropagateEmbeddedControlMessageRequest, PropagateEmbeddedControlMessageResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.propagate_embedded_control_message(request) - await stream.send_message(response) - - async def __rpc_take_global_checkpoint( - self, - stream: "grpclib.server.Stream[TakeGlobalCheckpointRequest, TakeGlobalCheckpointResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.take_global_checkpoint(request) - await stream.send_message(response) - - async def __rpc_debug_command( - self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.debug_command(request) - await stream.send_message(response) - - async def __rpc_evaluate_python_expression( - self, - stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatePythonExpressionResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.evaluate_python_expression(request) - await stream.send_message(response) - - async def __rpc_console_message_triggered( - self, - stream: "grpclib.server.Stream[ConsoleMessageTriggeredRequest, EmptyReturn]", - ) -> None: - request = await stream.recv_message() - response = await self.console_message_triggered(request) - await stream.send_message(response) - - async def __rpc_port_completed( - self, stream: "grpclib.server.Stream[PortCompletedRequest, EmptyReturn]" + async def __rpc_send_ping( + self, stream: "grpclib.server.Stream[Ping, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.port_completed(request) + response = await self.send_ping(request) await stream.send_message(response) - async def __rpc_start_workflow( - self, stream: "grpclib.server.Stream[EmptyRequest, StartWorkflowResponse]" + async def __rpc_send_pong( + self, stream: "grpclib.server.Stream[Pong, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.start_workflow(request) + response = await self.send_pong(request) await stream.send_message(response) - async def __rpc_resume_workflow( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_nested( + self, stream: "grpclib.server.Stream[Nested, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.resume_workflow(request) + response = await self.send_nested(request) await stream.send_message(response) - async def __rpc_pause_workflow( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_pass( + self, stream: "grpclib.server.Stream[Pass, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.pause_workflow(request) + response = await self.send_pass(request) await stream.send_message(response) - async def __rpc_worker_state_updated( - self, stream: "grpclib.server.Stream[WorkerStateUpdatedRequest, EmptyReturn]" + async def __rpc_send_error_command( + self, stream: "grpclib.server.Stream[ErrorCommand, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.worker_state_updated(request) + response = await self.send_error_command(request) await stream.send_message(response) - async def __rpc_worker_execution_completed( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_send_recursion( + self, stream: "grpclib.server.Stream[Recursion, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.worker_execution_completed(request) + response = await self.send_recursion(request) await stream.send_message(response) - async def __rpc_link_workers( - self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]" + async def __rpc_send_collect( + self, stream: "grpclib.server.Stream[Collect, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.link_workers(request) + response = await self.send_collect(request) await stream.send_message(response) - async def __rpc_controller_initiate_query_statistics( - self, stream: "grpclib.server.Stream[QueryStatisticsRequest, EmptyReturn]" + async def __rpc_send_generate_number( + self, stream: "grpclib.server.Stream[GenerateNumber, IntResponse]" ) -> None: request = await stream.recv_message() - response = await self.controller_initiate_query_statistics(request) + response = await self.send_generate_number(request) await stream.send_message(response) - async def __rpc_retry_workflow( - self, stream: "grpclib.server.Stream[RetryWorkflowRequest, EmptyReturn]" + async def __rpc_send_multi_call( + self, stream: "grpclib.server.Stream[MultiCall, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.retry_workflow(request) + response = await self.send_multi_call(request) await stream.send_message(response) - async def __rpc_reconfigure_workflow( - self, stream: "grpclib.server.Stream[WorkflowReconfigureRequest, EmptyReturn]" + async def __rpc_send_chain( + self, stream: "grpclib.server.Stream[Chain, StringResponse]" ) -> None: request = await stream.recv_message() - response = await self.reconfigure_workflow(request) + response = await self.send_chain(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState": grpclib.const.Handler( - self.__rpc_retrieve_workflow_state, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - RetrieveWorkflowStateResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage": grpclib.const.Handler( - self.__rpc_propagate_embedded_control_message, - grpclib.const.Cardinality.UNARY_UNARY, - PropagateEmbeddedControlMessageRequest, - PropagateEmbeddedControlMessageResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint": grpclib.const.Handler( - self.__rpc_take_global_checkpoint, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing": grpclib.const.Handler( + self.__rpc_send_ping, grpclib.const.Cardinality.UNARY_UNARY, - TakeGlobalCheckpointRequest, - TakeGlobalCheckpointResponse, + Ping, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand": grpclib.const.Handler( - self.__rpc_debug_command, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong": grpclib.const.Handler( + self.__rpc_send_pong, grpclib.const.Cardinality.UNARY_UNARY, - DebugCommandRequest, - EmptyReturn, + Pong, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression": grpclib.const.Handler( - self.__rpc_evaluate_python_expression, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested": grpclib.const.Handler( + self.__rpc_send_nested, grpclib.const.Cardinality.UNARY_UNARY, - EvaluatePythonExpressionRequest, - EvaluatePythonExpressionResponse, + Nested, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered": grpclib.const.Handler( - self.__rpc_console_message_triggered, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass": grpclib.const.Handler( + self.__rpc_send_pass, grpclib.const.Cardinality.UNARY_UNARY, - ConsoleMessageTriggeredRequest, - EmptyReturn, + Pass, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted": grpclib.const.Handler( - self.__rpc_port_completed, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand": grpclib.const.Handler( + self.__rpc_send_error_command, grpclib.const.Cardinality.UNARY_UNARY, - PortCompletedRequest, - EmptyReturn, + ErrorCommand, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow": grpclib.const.Handler( - self.__rpc_start_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion": grpclib.const.Handler( + self.__rpc_send_recursion, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - StartWorkflowResponse, + Recursion, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow": grpclib.const.Handler( - self.__rpc_resume_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect": grpclib.const.Handler( + self.__rpc_send_collect, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + Collect, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow": grpclib.const.Handler( - self.__rpc_pause_workflow, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber": grpclib.const.Handler( + self.__rpc_send_generate_number, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, + GenerateNumber, + IntResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated": grpclib.const.Handler( - self.__rpc_worker_state_updated, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall": grpclib.const.Handler( + self.__rpc_send_multi_call, grpclib.const.Cardinality.UNARY_UNARY, - WorkerStateUpdatedRequest, - EmptyReturn, + MultiCall, + StringResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted": grpclib.const.Handler( - self.__rpc_worker_execution_completed, + "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain": grpclib.const.Handler( + self.__rpc_send_chain, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler( - self.__rpc_link_workers, - grpclib.const.Cardinality.UNARY_UNARY, - LinkWorkersRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics": grpclib.const.Handler( - self.__rpc_controller_initiate_query_statistics, - grpclib.const.Cardinality.UNARY_UNARY, - QueryStatisticsRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow": grpclib.const.Handler( - self.__rpc_retry_workflow, - grpclib.const.Cardinality.UNARY_UNARY, - RetryWorkflowRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ReconfigureWorkflow": grpclib.const.Handler( - self.__rpc_reconfigure_workflow, - grpclib.const.Cardinality.UNARY_UNARY, - WorkflowReconfigureRequest, - EmptyReturn, + Chain, + StringResponse, ), } -class RpcTesterBase(ServiceBase): +class WorkerServiceBase(ServiceBase): - async def send_ping(self, ping: "Ping") -> "IntResponse": + async def add_input_channel( + self, add_input_channel_request: "AddInputChannelRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_pong(self, pong: "Pong") -> "IntResponse": + async def add_partitioning( + self, add_partitioning_request: "AddPartitioningRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_nested(self, nested: "Nested") -> "StringResponse": + async def assign_port( + self, assign_port_request: "AssignPortRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_pass(self, pass_: "Pass") -> "StringResponse": + async def finalize_checkpoint( + self, finalize_checkpoint_request: "FinalizeCheckpointRequest" + ) -> "FinalizeCheckpointResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_error_command( - self, error_command: "ErrorCommand" - ) -> "StringResponse": + async def flush_network_buffer( + self, empty_request: "EmptyRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_recursion(self, recursion: "Recursion") -> "StringResponse": + async def initialize_executor( + self, initialize_executor_request: "InitializeExecutorRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_collect(self, collect: "Collect") -> "StringResponse": + async def open_executor(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_generate_number( - self, generate_number: "GenerateNumber" - ) -> "IntResponse": + async def pause_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_multi_call(self, multi_call: "MultiCall") -> "StringResponse": + async def prepare_checkpoint( + self, prepare_checkpoint_request: "PrepareCheckpointRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def send_chain(self, chain: "Chain") -> "StringResponse": + async def query_statistics( + self, empty_request: "EmptyRequest" + ) -> "WorkerMetricsResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_send_ping( - self, stream: "grpclib.server.Stream[Ping, IntResponse]" + async def resume_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def retrieve_state(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def retry_current_tuple(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def start_worker( + self, empty_request: "EmptyRequest" + ) -> "WorkerStateResponse": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def end_worker(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def start_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def end_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def debug_command( + self, debug_command_request: "DebugCommandRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def evaluate_python_expression( + self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" + ) -> "EvaluatedValue": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def no_operation(self, empty_request: "EmptyRequest") -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def update_executor( + self, update_executor_request: "UpdateExecutorRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def __rpc_add_input_channel( + self, stream: "grpclib.server.Stream[AddInputChannelRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_ping(request) + response = await self.add_input_channel(request) await stream.send_message(response) - async def __rpc_send_pong( - self, stream: "grpclib.server.Stream[Pong, IntResponse]" + async def __rpc_add_partitioning( + self, stream: "grpclib.server.Stream[AddPartitioningRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_pong(request) + response = await self.add_partitioning(request) await stream.send_message(response) - async def __rpc_send_nested( - self, stream: "grpclib.server.Stream[Nested, StringResponse]" + async def __rpc_assign_port( + self, stream: "grpclib.server.Stream[AssignPortRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_nested(request) + response = await self.assign_port(request) await stream.send_message(response) - async def __rpc_send_pass( - self, stream: "grpclib.server.Stream[Pass, StringResponse]" + async def __rpc_finalize_checkpoint( + self, + stream: "grpclib.server.Stream[FinalizeCheckpointRequest, FinalizeCheckpointResponse]", ) -> None: request = await stream.recv_message() - response = await self.send_pass(request) + response = await self.finalize_checkpoint(request) await stream.send_message(response) - async def __rpc_send_error_command( - self, stream: "grpclib.server.Stream[ErrorCommand, StringResponse]" + async def __rpc_flush_network_buffer( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_error_command(request) + response = await self.flush_network_buffer(request) await stream.send_message(response) - async def __rpc_send_recursion( - self, stream: "grpclib.server.Stream[Recursion, StringResponse]" + async def __rpc_initialize_executor( + self, stream: "grpclib.server.Stream[InitializeExecutorRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_recursion(request) + response = await self.initialize_executor(request) await stream.send_message(response) - async def __rpc_send_collect( - self, stream: "grpclib.server.Stream[Collect, StringResponse]" + async def __rpc_open_executor( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_collect(request) + response = await self.open_executor(request) await stream.send_message(response) - async def __rpc_send_generate_number( - self, stream: "grpclib.server.Stream[GenerateNumber, IntResponse]" + async def __rpc_pause_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" ) -> None: request = await stream.recv_message() - response = await self.send_generate_number(request) + response = await self.pause_worker(request) await stream.send_message(response) - async def __rpc_send_multi_call( - self, stream: "grpclib.server.Stream[MultiCall, StringResponse]" + async def __rpc_prepare_checkpoint( + self, stream: "grpclib.server.Stream[PrepareCheckpointRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.send_multi_call(request) + response = await self.prepare_checkpoint(request) await stream.send_message(response) - async def __rpc_send_chain( - self, stream: "grpclib.server.Stream[Chain, StringResponse]" + async def __rpc_query_statistics( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerMetricsResponse]" ) -> None: request = await stream.recv_message() - response = await self.send_chain(request) + response = await self.query_statistics(request) + await stream.send_message(response) + + async def __rpc_resume_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + ) -> None: + request = await stream.recv_message() + response = await self.resume_worker(request) + await stream.send_message(response) + + async def __rpc_retrieve_state( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.retrieve_state(request) + await stream.send_message(response) + + async def __rpc_retry_current_tuple( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.retry_current_tuple(request) + await stream.send_message(response) + + async def __rpc_start_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + ) -> None: + request = await stream.recv_message() + response = await self.start_worker(request) + await stream.send_message(response) + + async def __rpc_end_worker( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.end_worker(request) + await stream.send_message(response) + + async def __rpc_start_channel( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.start_channel(request) + await stream.send_message(response) + + async def __rpc_end_channel( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.end_channel(request) + await stream.send_message(response) + + async def __rpc_debug_command( + self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.debug_command(request) + await stream.send_message(response) + + async def __rpc_evaluate_python_expression( + self, + stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatedValue]", + ) -> None: + request = await stream.recv_message() + response = await self.evaluate_python_expression(request) + await stream.send_message(response) + + async def __rpc_no_operation( + self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.no_operation(request) + await stream.send_message(response) + + async def __rpc_update_executor( + self, stream: "grpclib.server.Stream[UpdateExecutorRequest, EmptyReturn]" + ) -> None: + request = await stream.recv_message() + response = await self.update_executor(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPing": grpclib.const.Handler( - self.__rpc_send_ping, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel": grpclib.const.Handler( + self.__rpc_add_input_channel, grpclib.const.Cardinality.UNARY_UNARY, - Ping, - IntResponse, + AddInputChannelRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPong": grpclib.const.Handler( - self.__rpc_send_pong, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning": grpclib.const.Handler( + self.__rpc_add_partitioning, grpclib.const.Cardinality.UNARY_UNARY, - Pong, - IntResponse, + AddPartitioningRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendNested": grpclib.const.Handler( - self.__rpc_send_nested, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort": grpclib.const.Handler( + self.__rpc_assign_port, grpclib.const.Cardinality.UNARY_UNARY, - Nested, - StringResponse, + AssignPortRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendPass": grpclib.const.Handler( - self.__rpc_send_pass, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint": grpclib.const.Handler( + self.__rpc_finalize_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - Pass, - StringResponse, + FinalizeCheckpointRequest, + FinalizeCheckpointResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendErrorCommand": grpclib.const.Handler( - self.__rpc_send_error_command, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer": grpclib.const.Handler( + self.__rpc_flush_network_buffer, grpclib.const.Cardinality.UNARY_UNARY, - ErrorCommand, - StringResponse, + EmptyRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendRecursion": grpclib.const.Handler( - self.__rpc_send_recursion, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor": grpclib.const.Handler( + self.__rpc_initialize_executor, grpclib.const.Cardinality.UNARY_UNARY, - Recursion, - StringResponse, + InitializeExecutorRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendCollect": grpclib.const.Handler( - self.__rpc_send_collect, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor": grpclib.const.Handler( + self.__rpc_open_executor, grpclib.const.Cardinality.UNARY_UNARY, - Collect, - StringResponse, + EmptyRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendGenerateNumber": grpclib.const.Handler( - self.__rpc_send_generate_number, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker": grpclib.const.Handler( + self.__rpc_pause_worker, grpclib.const.Cardinality.UNARY_UNARY, - GenerateNumber, - IntResponse, + EmptyRequest, + WorkerStateResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendMultiCall": grpclib.const.Handler( - self.__rpc_send_multi_call, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint": grpclib.const.Handler( + self.__rpc_prepare_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - MultiCall, - StringResponse, + PrepareCheckpointRequest, + EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.RPCTester/SendChain": grpclib.const.Handler( - self.__rpc_send_chain, + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics": grpclib.const.Handler( + self.__rpc_query_statistics, grpclib.const.Cardinality.UNARY_UNARY, - Chain, - StringResponse, + EmptyRequest, + WorkerMetricsResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker": grpclib.const.Handler( + self.__rpc_resume_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState": grpclib.const.Handler( + self.__rpc_retrieve_state, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple": grpclib.const.Handler( + self.__rpc_retry_current_tuple, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker": grpclib.const.Handler( + self.__rpc_start_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + WorkerStateResponse, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker": grpclib.const.Handler( + self.__rpc_end_worker, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel": grpclib.const.Handler( + self.__rpc_start_channel, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel": grpclib.const.Handler( + self.__rpc_end_channel, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand": grpclib.const.Handler( + self.__rpc_debug_command, + grpclib.const.Cardinality.UNARY_UNARY, + DebugCommandRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression": grpclib.const.Handler( + self.__rpc_evaluate_python_expression, + grpclib.const.Cardinality.UNARY_UNARY, + EvaluatePythonExpressionRequest, + EvaluatedValue, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation": grpclib.const.Handler( + self.__rpc_no_operation, + grpclib.const.Cardinality.UNARY_UNARY, + EmptyRequest, + EmptyReturn, + ), + "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/UpdateExecutor": grpclib.const.Handler( + self.__rpc_update_executor, + grpclib.const.Cardinality.UNARY_UNARY, + UpdateExecutorRequest, + EmptyReturn, ), } -class WorkerServiceBase(ServiceBase): - - async def add_input_channel( - self, add_input_channel_request: "AddInputChannelRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def add_partitioning( - self, add_partitioning_request: "AddPartitioningRequest" - ) -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) +class ControllerServiceBase(ServiceBase): - async def assign_port( - self, assign_port_request: "AssignPortRequest" - ) -> "EmptyReturn": + async def retrieve_workflow_state( + self, empty_request: "EmptyRequest" + ) -> "RetrieveWorkflowStateResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def finalize_checkpoint( - self, finalize_checkpoint_request: "FinalizeCheckpointRequest" - ) -> "FinalizeCheckpointResponse": + async def propagate_embedded_control_message( + self, + propagate_embedded_control_message_request: "PropagateEmbeddedControlMessageRequest", + ) -> "PropagateEmbeddedControlMessageResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def flush_network_buffer( - self, empty_request: "EmptyRequest" - ) -> "EmptyReturn": + async def take_global_checkpoint( + self, take_global_checkpoint_request: "TakeGlobalCheckpointRequest" + ) -> "TakeGlobalCheckpointResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def initialize_executor( - self, initialize_executor_request: "InitializeExecutorRequest" + async def debug_command( + self, debug_command_request: "DebugCommandRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def open_executor(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def evaluate_python_expression( + self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" + ) -> "EvaluatePythonExpressionResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def pause_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": + async def console_message_triggered( + self, console_message_triggered_request: "ConsoleMessageTriggeredRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def prepare_checkpoint( - self, prepare_checkpoint_request: "PrepareCheckpointRequest" + async def port_completed( + self, port_completed_request: "PortCompletedRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def query_statistics( + async def start_workflow( self, empty_request: "EmptyRequest" - ) -> "WorkerMetricsResponse": + ) -> "StartWorkflowResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def resume_worker( - self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": + async def resume_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retrieve_state(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def pause_workflow(self, empty_request: "EmptyRequest") -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def retry_current_tuple(self, empty_request: "EmptyRequest") -> "EmptyReturn": + async def worker_state_updated( + self, worker_state_updated_request: "WorkerStateUpdatedRequest" + ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def start_worker( + async def worker_execution_completed( self, empty_request: "EmptyRequest" - ) -> "WorkerStateResponse": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def end_worker(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def start_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def end_channel(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def debug_command( - self, debug_command_request: "DebugCommandRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def evaluate_python_expression( - self, evaluate_python_expression_request: "EvaluatePythonExpressionRequest" - ) -> "EvaluatedValue": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def no_operation(self, empty_request: "EmptyRequest") -> "EmptyReturn": - raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - - async def update_executor( - self, update_executor_request: "UpdateExecutorRequest" + async def link_workers( + self, link_workers_request: "LinkWorkersRequest" ) -> "EmptyReturn": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_add_input_channel( - self, stream: "grpclib.server.Stream[AddInputChannelRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.add_input_channel(request) - await stream.send_message(response) - - async def __rpc_add_partitioning( - self, stream: "grpclib.server.Stream[AddPartitioningRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.add_partitioning(request) - await stream.send_message(response) + async def controller_initiate_query_statistics( + self, query_statistics_request: "QueryStatisticsRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_assign_port( - self, stream: "grpclib.server.Stream[AssignPortRequest, EmptyReturn]" - ) -> None: - request = await stream.recv_message() - response = await self.assign_port(request) - await stream.send_message(response) + async def retry_workflow( + self, retry_workflow_request: "RetryWorkflowRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_finalize_checkpoint( - self, - stream: "grpclib.server.Stream[FinalizeCheckpointRequest, FinalizeCheckpointResponse]", - ) -> None: - request = await stream.recv_message() - response = await self.finalize_checkpoint(request) - await stream.send_message(response) + async def reconfigure_workflow( + self, workflow_reconfigure_request: "WorkflowReconfigureRequest" + ) -> "EmptyReturn": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) - async def __rpc_flush_network_buffer( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_retrieve_workflow_state( + self, + stream: "grpclib.server.Stream[EmptyRequest, RetrieveWorkflowStateResponse]", ) -> None: request = await stream.recv_message() - response = await self.flush_network_buffer(request) + response = await self.retrieve_workflow_state(request) await stream.send_message(response) - async def __rpc_initialize_executor( - self, stream: "grpclib.server.Stream[InitializeExecutorRequest, EmptyReturn]" + async def __rpc_propagate_embedded_control_message( + self, + stream: "grpclib.server.Stream[PropagateEmbeddedControlMessageRequest, PropagateEmbeddedControlMessageResponse]", ) -> None: request = await stream.recv_message() - response = await self.initialize_executor(request) + response = await self.propagate_embedded_control_message(request) await stream.send_message(response) - async def __rpc_open_executor( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_take_global_checkpoint( + self, + stream: "grpclib.server.Stream[TakeGlobalCheckpointRequest, TakeGlobalCheckpointResponse]", ) -> None: request = await stream.recv_message() - response = await self.open_executor(request) + response = await self.take_global_checkpoint(request) await stream.send_message(response) - async def __rpc_pause_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + async def __rpc_debug_command( + self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.pause_worker(request) + response = await self.debug_command(request) await stream.send_message(response) - async def __rpc_prepare_checkpoint( - self, stream: "grpclib.server.Stream[PrepareCheckpointRequest, EmptyReturn]" + async def __rpc_evaluate_python_expression( + self, + stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatePythonExpressionResponse]", ) -> None: request = await stream.recv_message() - response = await self.prepare_checkpoint(request) + response = await self.evaluate_python_expression(request) await stream.send_message(response) - async def __rpc_query_statistics( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerMetricsResponse]" + async def __rpc_console_message_triggered( + self, + stream: "grpclib.server.Stream[ConsoleMessageTriggeredRequest, EmptyReturn]", ) -> None: request = await stream.recv_message() - response = await self.query_statistics(request) + response = await self.console_message_triggered(request) await stream.send_message(response) - async def __rpc_resume_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" + async def __rpc_port_completed( + self, stream: "grpclib.server.Stream[PortCompletedRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.resume_worker(request) + response = await self.port_completed(request) await stream.send_message(response) - async def __rpc_retrieve_state( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_start_workflow( + self, stream: "grpclib.server.Stream[EmptyRequest, StartWorkflowResponse]" ) -> None: request = await stream.recv_message() - response = await self.retrieve_state(request) + response = await self.start_workflow(request) await stream.send_message(response) - async def __rpc_retry_current_tuple( + async def __rpc_resume_workflow( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.retry_current_tuple(request) - await stream.send_message(response) - - async def __rpc_start_worker( - self, stream: "grpclib.server.Stream[EmptyRequest, WorkerStateResponse]" - ) -> None: - request = await stream.recv_message() - response = await self.start_worker(request) + response = await self.resume_workflow(request) await stream.send_message(response) - async def __rpc_end_worker( + async def __rpc_pause_workflow( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.end_worker(request) + response = await self.pause_workflow(request) await stream.send_message(response) - async def __rpc_start_channel( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_worker_state_updated( + self, stream: "grpclib.server.Stream[WorkerStateUpdatedRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.start_channel(request) + response = await self.worker_state_updated(request) await stream.send_message(response) - async def __rpc_end_channel( + async def __rpc_worker_execution_completed( self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.end_channel(request) + response = await self.worker_execution_completed(request) await stream.send_message(response) - async def __rpc_debug_command( - self, stream: "grpclib.server.Stream[DebugCommandRequest, EmptyReturn]" + async def __rpc_link_workers( + self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.debug_command(request) + response = await self.link_workers(request) await stream.send_message(response) - async def __rpc_evaluate_python_expression( - self, - stream: "grpclib.server.Stream[EvaluatePythonExpressionRequest, EvaluatedValue]", + async def __rpc_controller_initiate_query_statistics( + self, stream: "grpclib.server.Stream[QueryStatisticsRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.evaluate_python_expression(request) + response = await self.controller_initiate_query_statistics(request) await stream.send_message(response) - async def __rpc_no_operation( - self, stream: "grpclib.server.Stream[EmptyRequest, EmptyReturn]" + async def __rpc_retry_workflow( + self, stream: "grpclib.server.Stream[RetryWorkflowRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.no_operation(request) + response = await self.retry_workflow(request) await stream.send_message(response) - async def __rpc_update_executor( - self, stream: "grpclib.server.Stream[UpdateExecutorRequest, EmptyReturn]" + async def __rpc_reconfigure_workflow( + self, stream: "grpclib.server.Stream[WorkflowReconfigureRequest, EmptyReturn]" ) -> None: request = await stream.recv_message() - response = await self.update_executor(request) + response = await self.reconfigure_workflow(request) await stream.send_message(response) def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddInputChannel": grpclib.const.Handler( - self.__rpc_add_input_channel, - grpclib.const.Cardinality.UNARY_UNARY, - AddInputChannelRequest, - EmptyReturn, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AddPartitioning": grpclib.const.Handler( - self.__rpc_add_partitioning, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetrieveWorkflowState": grpclib.const.Handler( + self.__rpc_retrieve_workflow_state, grpclib.const.Cardinality.UNARY_UNARY, - AddPartitioningRequest, - EmptyReturn, + EmptyRequest, + RetrieveWorkflowStateResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/AssignPort": grpclib.const.Handler( - self.__rpc_assign_port, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PropagateEmbeddedControlMessage": grpclib.const.Handler( + self.__rpc_propagate_embedded_control_message, grpclib.const.Cardinality.UNARY_UNARY, - AssignPortRequest, - EmptyReturn, + PropagateEmbeddedControlMessageRequest, + PropagateEmbeddedControlMessageResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FinalizeCheckpoint": grpclib.const.Handler( - self.__rpc_finalize_checkpoint, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/TakeGlobalCheckpoint": grpclib.const.Handler( + self.__rpc_take_global_checkpoint, grpclib.const.Cardinality.UNARY_UNARY, - FinalizeCheckpointRequest, - FinalizeCheckpointResponse, + TakeGlobalCheckpointRequest, + TakeGlobalCheckpointResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/FlushNetworkBuffer": grpclib.const.Handler( - self.__rpc_flush_network_buffer, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/DebugCommand": grpclib.const.Handler( + self.__rpc_debug_command, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, + DebugCommandRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/InitializeExecutor": grpclib.const.Handler( - self.__rpc_initialize_executor, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/EvaluatePythonExpression": grpclib.const.Handler( + self.__rpc_evaluate_python_expression, grpclib.const.Cardinality.UNARY_UNARY, - InitializeExecutorRequest, - EmptyReturn, + EvaluatePythonExpressionRequest, + EvaluatePythonExpressionResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/OpenExecutor": grpclib.const.Handler( - self.__rpc_open_executor, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ConsoleMessageTriggered": grpclib.const.Handler( + self.__rpc_console_message_triggered, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, + ConsoleMessageTriggeredRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PauseWorker": grpclib.const.Handler( - self.__rpc_pause_worker, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/PrepareCheckpoint": grpclib.const.Handler( - self.__rpc_prepare_checkpoint, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PortCompleted": grpclib.const.Handler( + self.__rpc_port_completed, grpclib.const.Cardinality.UNARY_UNARY, - PrepareCheckpointRequest, + PortCompletedRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/QueryStatistics": grpclib.const.Handler( - self.__rpc_query_statistics, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerMetricsResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/ResumeWorker": grpclib.const.Handler( - self.__rpc_resume_worker, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/StartWorkflow": grpclib.const.Handler( + self.__rpc_start_workflow, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, - WorkerStateResponse, + StartWorkflowResponse, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetrieveState": grpclib.const.Handler( - self.__rpc_retrieve_state, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ResumeWorkflow": grpclib.const.Handler( + self.__rpc_resume_workflow, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/RetryCurrentTuple": grpclib.const.Handler( - self.__rpc_retry_current_tuple, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/PauseWorkflow": grpclib.const.Handler( + self.__rpc_pause_workflow, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartWorker": grpclib.const.Handler( - self.__rpc_start_worker, - grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, - WorkerStateResponse, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndWorker": grpclib.const.Handler( - self.__rpc_end_worker, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerStateUpdated": grpclib.const.Handler( + self.__rpc_worker_state_updated, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, + WorkerStateUpdatedRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/StartChannel": grpclib.const.Handler( - self.__rpc_start_channel, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/WorkerExecutionCompleted": grpclib.const.Handler( + self.__rpc_worker_execution_completed, grpclib.const.Cardinality.UNARY_UNARY, EmptyRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EndChannel": grpclib.const.Handler( - self.__rpc_end_channel, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler( + self.__rpc_link_workers, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, + LinkWorkersRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/DebugCommand": grpclib.const.Handler( - self.__rpc_debug_command, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ControllerInitiateQueryStatistics": grpclib.const.Handler( + self.__rpc_controller_initiate_query_statistics, grpclib.const.Cardinality.UNARY_UNARY, - DebugCommandRequest, + QueryStatisticsRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/EvaluatePythonExpression": grpclib.const.Handler( - self.__rpc_evaluate_python_expression, - grpclib.const.Cardinality.UNARY_UNARY, - EvaluatePythonExpressionRequest, - EvaluatedValue, - ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/NoOperation": grpclib.const.Handler( - self.__rpc_no_operation, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/RetryWorkflow": grpclib.const.Handler( + self.__rpc_retry_workflow, grpclib.const.Cardinality.UNARY_UNARY, - EmptyRequest, + RetryWorkflowRequest, EmptyReturn, ), - "/org.apache.texera.amber.engine.architecture.rpc.WorkerService/UpdateExecutor": grpclib.const.Handler( - self.__rpc_update_executor, + "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/ReconfigureWorkflow": grpclib.const.Handler( + self.__rpc_reconfigure_workflow, grpclib.const.Cardinality.UNARY_UNARY, - UpdateExecutorRequest, + WorkflowReconfigureRequest, EmptyReturn, ), } diff --git a/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py b/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py index 8c1464cc76c..55c789aa395 100644 --- a/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py +++ b/amber/src/main/python/proto/org/apache/texera/amber/engine/common/__init__.py @@ -18,27 +18,6 @@ ) -@dataclass(eq=False, repr=False) -class Backpressure(betterproto.Message): - enable_backpressure: bool = betterproto.bool_field(1) - - -@dataclass(eq=False, repr=False) -class CreditUpdate(betterproto.Message): - pass - - -@dataclass(eq=False, repr=False) -class ActorCommand(betterproto.Message): - backpressure: "Backpressure" = betterproto.message_field(1, group="sealed_value") - credit_update: "CreditUpdate" = betterproto.message_field(2, group="sealed_value") - - -@dataclass(eq=False, repr=False) -class PythonActorMessage(betterproto.Message): - payload: "ActorCommand" = betterproto.message_field(1) - - @dataclass(eq=False, repr=False) class DirectControlMessagePayloadV2(betterproto.Message): control_invocation: "_architecture_rpc__.ControlInvocation" = ( @@ -154,3 +133,24 @@ class ExecutionMetadataStore(betterproto.Message): fatal_errors: List["__core__.WorkflowFatalError"] = betterproto.message_field(2) execution_id: "__core__.ExecutionIdentity" = betterproto.message_field(3) is_recovering: bool = betterproto.bool_field(4) + + +@dataclass(eq=False, repr=False) +class Backpressure(betterproto.Message): + enable_backpressure: bool = betterproto.bool_field(1) + + +@dataclass(eq=False, repr=False) +class CreditUpdate(betterproto.Message): + pass + + +@dataclass(eq=False, repr=False) +class ActorCommand(betterproto.Message): + backpressure: "Backpressure" = betterproto.message_field(1, group="sealed_value") + credit_update: "CreditUpdate" = betterproto.message_field(2, group="sealed_value") + + +@dataclass(eq=False, repr=False) +class PythonActorMessage(betterproto.Message): + payload: "ActorCommand" = betterproto.message_field(1) From 94a302e38a5d9b80f4291cae2417ca7bdf1b5827 Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sat, 14 Feb 2026 21:23:48 -0800 Subject: [PATCH 05/15] make java end working --- .../control/update_executor_handler.py | 2 +- .../python/core/runnables/network_receiver.py | 1 + .../DataProcessorRPCHandlerInitializer.scala | 2 +- .../InitializeExecutorHandler.scala | 2 +- .../UpdateExecutorHandler.scala | 2 +- .../amber/engine/e2e/ModifyLogicSpec.scala | 22 ++++++++++++++++--- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py index a5bc1a6c0fc..7449cbb4b59 100644 --- a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py +++ b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py @@ -1 +1 @@ -# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from core.architecture.handlers.control.control_handler_base import ControlHandler from proto.org.apache.texera.amber.engine.architecture.rpc import ( EmptyReturn, UpdateExecutorRequest, ) class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: self.context.executor_manager.update_executor(req.code, req.is_source) return EmptyReturn() \ No newline at end of file +# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from core.architecture.handlers.control.control_handler_base import ControlHandler from proto.org.apache.texera.amber.engine.architecture.rpc import ( EmptyReturn, UpdateExecutorRequest, ) from loguru import logger class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: self.context.executor_manager.update_executor(req.code, req.is_source) return EmptyReturn() \ No newline at end of file diff --git a/amber/src/main/python/core/runnables/network_receiver.py b/amber/src/main/python/core/runnables/network_receiver.py index fd42a8f589b..879b623b765 100644 --- a/amber/src/main/python/core/runnables/network_receiver.py +++ b/amber/src/main/python/core/runnables/network_receiver.py @@ -126,6 +126,7 @@ def control_handler(message: bytes) -> int: payload=python_control_message.payload, ) ) + logger.info(python_control_message.payload) return shared_queue.in_mem_size() self._proxy_server.register_control_handler(control_handler) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala index d2837860f5a..cfbc40b5879 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala @@ -72,7 +72,7 @@ class DataProcessorRPCHandlerInitializer(val dp: DataProcessor) override def noOperation(request: EmptyRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = ??? - def initializeExecutor(execInitInfo: OpExecInitInfo, workerIdx: Int, workerCount: Int): Unit = { + def setupExecutor(execInitInfo: OpExecInitInfo, workerIdx: Int, workerCount: Int): Unit = { dp.executor = execInitInfo match { case OpExecWithClassName(className, descString) => ExecFactory.newExecFromJavaClassName(className, descString, workerIdx, workerCount) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index f6e7f3d8b21..a7fd26c4342 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -42,7 +42,7 @@ trait InitializeExecutorHandler { dp.serializationManager.setOpInitialization(req) val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) cachedTotalWorkerCount = req.totalWorkerCount - initializeExecutor(req.opExecInitInfo, workerIdx, cachedTotalWorkerCount) + setupExecutor(req.opExecInitInfo, workerIdx, cachedTotalWorkerCount) EmptyReturn() } } diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala index 8cb0e30be21..5c2f65c257b 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala @@ -35,7 +35,7 @@ trait UpdateExecutorHandler { ctx: AsyncRPCContext ): Future[EmptyReturn] = { val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) - initializeExecutor(request.newExecInitInfo, workerIdx, cachedTotalWorkerCount) + setupExecutor(request.newExecInitInfo, workerIdx, cachedTotalWorkerCount) dp.executor.open() EmptyReturn() } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala index 2572a78137d..699329bfae5 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -25,7 +25,7 @@ import org.apache.pekko.actor.{ActorSystem, Props} import org.apache.pekko.testkit.{ImplicitSender, TestKit} import org.apache.pekko.util.Timeout import org.apache.texera.amber.clustering.SingleNodeListener -import org.apache.texera.amber.core.executor.{OpExecInitInfo, OpExecWithCode} +import org.apache.texera.amber.core.executor.{OpExecInitInfo, OpExecWithClassName, OpExecWithCode} import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.storage.model.VirtualDocument import org.apache.texera.amber.core.tuple.Tuple @@ -38,7 +38,7 @@ import org.apache.texera.amber.engine.common.AmberRuntime import org.apache.texera.amber.engine.common.client.AmberClient import org.apache.texera.amber.engine.e2e.TestUtils.{cleanupWorkflowExecutionData, initiateTexeraDBForTestCases, setUpWorkflowExecutionData} import org.apache.texera.amber.operator.{LogicalOp, TestOperators} -import org.apache.texera.amber.operator.TestOperators.{pythonOpDesc, pythonSourceOpDesc} +import org.apache.texera.amber.operator.TestOperators.{mediumCsvScanOpDesc, pythonOpDesc, pythonSourceOpDesc} import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource.getResultUriByLogicalPortId import org.apache.texera.workflow.LogicalLink import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome, Retries} @@ -64,6 +64,7 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim implicit val timeout: Timeout = Timeout(5.seconds) val logger = Logger("ModifyLogicSpecLogger") + val ctx = new WorkflowContext() override protected def beforeEach(): Unit = { setUpWorkflowExecutionData() @@ -94,7 +95,7 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim checkResultLambda: (Map[OperatorIdentity, List[Tuple]]) => Boolean ): Unit = { val workflow = - TestUtils.buildWorkflow(operators, links, new WorkflowContext()) + TestUtils.buildWorkflow(operators, links, ctx) val client = new AmberClient( system, @@ -177,4 +178,19 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim ) } + "Engine" should "be able to modify a java operator in workflow" in { + val sourceOpDesc = mediumCsvScanOpDesc() + val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") + val keywordMatchManyOpDesc = TestOperators.keywordSearchOpDesc("Region", "Asia") + shouldReconfigure(List(sourceOpDesc, keywordMatchNoneOpDesc), List(LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + keywordMatchNoneOpDesc.operatorIdentifier, + PortIdentity() + )), + keywordMatchNoneOpDesc, keywordMatchManyOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo, + results => results(keywordMatchNoneOpDesc.operatorIdentifier).nonEmpty + ) + } + } From 7d0342ef70418fe36bae11510dcf07afc1d2b70f Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 15 Feb 2026 01:37:46 -0800 Subject: [PATCH 06/15] fix python end --- .../control/update_executor_handler.py | 2 +- .../rpc/async_rpc_handler_initializer.py | 2 + .../main/python/core/models/internal_queue.py | 3 + .../python/core/runnables/network_receiver.py | 1 - .../ReconfigurationHandler.scala | 9 +- .../amber/engine/e2e/ModifyLogicSpec.scala | 130 ++++++++++++------ .../texera/amber/operator/TestOperators.scala | 5 +- 7 files changed, 104 insertions(+), 48 deletions(-) diff --git a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py index 7449cbb4b59..a79d31c180a 100644 --- a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py +++ b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py @@ -1 +1 @@ -# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from core.architecture.handlers.control.control_handler_base import ControlHandler from proto.org.apache.texera.amber.engine.architecture.rpc import ( EmptyReturn, UpdateExecutorRequest, ) from loguru import logger class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: self.context.executor_manager.update_executor(req.code, req.is_source) return EmptyReturn() \ No newline at end of file +# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from core.architecture.handlers.control.control_handler_base import ControlHandler from proto.org.apache.texera.amber.engine.architecture.rpc import ( EmptyReturn, UpdateExecutorRequest, ) from core.util import get_one_of from proto.org.apache.texera.amber.core import OpExecWithCode class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: op_exec_with_code: OpExecWithCode = get_one_of(req.new_exec_init_info) self.context.executor_manager.update_executor(op_exec_with_code.code, False) return EmptyReturn() \ No newline at end of file diff --git a/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py b/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py index c2574028a13..7b60bc27fb2 100644 --- a/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py +++ b/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py @@ -45,6 +45,7 @@ from core.architecture.handlers.control.resume_worker_handler import ResumeWorkerHandler from core.architecture.handlers.control.start_channel_handler import StartChannelHandler from core.architecture.handlers.control.start_worker_handler import StartWorkerHandler +from core.architecture.handlers.control.update_executor_handler import UpdateExecutorHandler class AsyncRPCHandlerInitializer( @@ -64,5 +65,6 @@ class AsyncRPCHandlerInitializer( StartChannelHandler, EndChannelHandler, NoOperationHandler, + UpdateExecutorHandler, ): pass diff --git a/amber/src/main/python/core/models/internal_queue.py b/amber/src/main/python/core/models/internal_queue.py index abc1793ff6c..ae0247aa2db 100644 --- a/amber/src/main/python/core/models/internal_queue.py +++ b/amber/src/main/python/core/models/internal_queue.py @@ -79,6 +79,9 @@ def put(self, item: T) -> None: if item.tag not in self._queue_ids: self._queue.add_sub_queue(item.tag, 1 if item.tag.is_control else 2) self._queue_ids.add(item.tag) + if len(self._queue_state) > 0 and not item.tag.is_control: + # if paused, then the newly added queue will also be paused. + self._queue.disable(item.tag) if isinstance(item, (DataElement, InternalMarker, ECMElement)): self._queue.put(item.tag, item) elif isinstance(item, DCMElement): diff --git a/amber/src/main/python/core/runnables/network_receiver.py b/amber/src/main/python/core/runnables/network_receiver.py index 879b623b765..fd42a8f589b 100644 --- a/amber/src/main/python/core/runnables/network_receiver.py +++ b/amber/src/main/python/core/runnables/network_receiver.py @@ -126,7 +126,6 @@ def control_handler(message: bytes) -> int: payload=python_control_message.payload, ) ) - logger.info(python_control_message.payload) return shared_queue.in_mem_size() self._proxy_server.register_control_handler(control_handler) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index 573306bc065..dd31aae4207 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -37,7 +37,14 @@ trait ReconfigurationHandler { override def reconfigureWorkflow(msg: WorkflowReconfigureRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { val futures = mutable.ArrayBuffer[Future[_]]() - FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg).foreach{ + val friesComponents = FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg) + val needToSendECMToSources = friesComponents.exists(comp => comp.sources.exists(sourceOp => cp.workflowScheduler.physicalPlan.getOperator(sourceOp).isSourceOperator)) + if(needToSendECMToSources){ + throw new IllegalStateException( + "Reconfiguration cannot be propagated through source operators" + ) + } + friesComponents.foreach{ friesComponent => if(friesComponent.scope.size == 1){ val updateExecutorRequest = friesComponent.reconfigurations.head diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala index 699329bfae5..41016a81444 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -90,10 +90,9 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim def shouldReconfigure( operators: List[LogicalOp], links: List[LogicalLink], - targetOp: LogicalOp, - newOpExecInitInfo: OpExecInitInfo, - checkResultLambda: (Map[OperatorIdentity, List[Tuple]]) => Boolean - ): Unit = { + targetOps: Seq[LogicalOp], + newOpExecInitInfo: OpExecInitInfo + ): Map[OperatorIdentity, List[Tuple]] = { val workflow = TestUtils.buildWorkflow(operators, links, ctx) val client = @@ -105,43 +104,42 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim error => {} ) val completion = Promise[Unit]() + var result: Map[OperatorIdentity, List[Tuple]] = null client .registerCallback[ExecutionStateUpdate](evt => { if (evt.state == COMPLETED) { -// checkResultLambda(workflow.logicalPlan.getTerminalOperatorIds -// .filter(terminalOpId => { -// val uri = getResultUriByLogicalPortId( -// workflow.context.executionId, -// terminalOpId, -// PortIdentity() -// ) -// uri.nonEmpty -// }) -// .map(terminalOpId => { -// //TODO: remove the delay after fixing the issue of reporting "completed" status too early. -// Thread.sleep(1000) -// val uri = getResultUriByLogicalPortId( -// workflow.context.executionId, -// terminalOpId, -// PortIdentity() -// ).get -// terminalOpId -> DocumentFactory -// .openDocument(uri) -// ._1 -// .asInstanceOf[VirtualDocument[Tuple]] -// .get() -// .toList -// }) -// .toMap) + result = workflow.logicalPlan.getTerminalOperatorIds + .filter(terminalOpId => { + val uri = getResultUriByLogicalPortId( + workflow.context.executionId, + terminalOpId, + PortIdentity() + ) + uri.nonEmpty + }) + .map(terminalOpId => { + //TODO: remove the delay after fixing the issue of reporting "completed" status too early. + Thread.sleep(1000) + val uri = getResultUriByLogicalPortId( + workflow.context.executionId, + terminalOpId, + PortIdentity() + ).get + terminalOpId -> DocumentFactory + .openDocument(uri) + ._1 + .asInstanceOf[VirtualDocument[Tuple]] + .get() + .toList + }) + .toMap completion.setDone() } }) Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ())) Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ())) Thread.sleep(4000) - val physicalOps = workflow.physicalPlan.getPhysicalOpsOfLogicalOp(targetOp.operatorIdentifier) - assert(physicalOps.nonEmpty && physicalOps.length == 1, - "cannot reconfigure more than one physical operator in this test") + val physicalOps = targetOps.flatMap(op => workflow.physicalPlan.getPhysicalOpsOfLogicalOp(op.operatorIdentifier)) Await.result(client.controllerInterface.reconfigureWorkflow(WorkflowReconfigureRequest( reconfiguration = physicalOps.map(op => UpdateExecutorRequest(op.id, newOpExecInitInfo) @@ -149,48 +147,94 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ())) Thread.sleep(400) Await.result(completion, Duration.fromMinutes(1)) + result } "Engine" should "be able to modify a python UDF worker in workflow" in { + val sourceOpDesc = TestOperators.smallCsvScanOpDesc() val udfOpDesc = pythonOpDesc() - val sourceOpDesc = pythonSourceOpDesc(5000) val code = """ |from pytexera import * | |class ProcessTupleOperator(UDFOperatorV2): | @overrides | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: - | tuple_['field_2'] = tuple_['field_2'] + '_reconfigured' + | tuple_['Region'] = tuple_['Region'] + '_reconfigured' | yield tuple_ |""".stripMargin - shouldReconfigure(List(sourceOpDesc, udfOpDesc), List(LogicalLink( + val result = shouldReconfigure(List(sourceOpDesc, udfOpDesc), List(LogicalLink( sourceOpDesc.operatorIdentifier, PortIdentity(), udfOpDesc.operatorIdentifier, PortIdentity() )), - udfOpDesc, OpExecWithCode(code, "python"), - results => results(udfOpDesc.operatorIdentifier).exists { - t => t.getField("field_2").asInstanceOf[String].contains("_reconfigured") - } - ) + Seq(udfOpDesc), OpExecWithCode(code, "python")) + assert(result(udfOpDesc.operatorIdentifier).exists { + t => t.getField("Region").asInstanceOf[String].contains("_reconfigured") + }) } "Engine" should "be able to modify a java operator in workflow" in { val sourceOpDesc = mediumCsvScanOpDesc() val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") val keywordMatchManyOpDesc = TestOperators.keywordSearchOpDesc("Region", "Asia") - shouldReconfigure(List(sourceOpDesc, keywordMatchNoneOpDesc), List(LogicalLink( + val result = shouldReconfigure(List(sourceOpDesc, keywordMatchNoneOpDesc), List(LogicalLink( sourceOpDesc.operatorIdentifier, PortIdentity(), keywordMatchNoneOpDesc.operatorIdentifier, PortIdentity() )), - keywordMatchNoneOpDesc, keywordMatchManyOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo, - results => results(keywordMatchNoneOpDesc.operatorIdentifier).nonEmpty + Seq(keywordMatchNoneOpDesc), keywordMatchManyOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo ) + assert(result(keywordMatchNoneOpDesc.operatorIdentifier).nonEmpty) + } + + "Engine" should "not be able to modify a source operator in workflow" in { + val sourceOpDesc = mediumCsvScanOpDesc() + val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") + val ex = intercept[Throwable] { + shouldReconfigure(List(sourceOpDesc, keywordMatchNoneOpDesc), List(LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + keywordMatchNoneOpDesc.operatorIdentifier, + PortIdentity() + )), + Seq(sourceOpDesc), sourceOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo + )} + assert(ex.getMessage == "java.lang.IllegalStateException: Reconfiguration cannot be propagated through source operators") + } + + "Engine" should "be able to modify two python UDFs in workflow" in { + val sourceOpDesc = TestOperators.smallCsvScanOpDesc() + val udfOpDesc1 = pythonOpDesc() + val udfOpDesc2 = pythonOpDesc() + val code = """ + |from pytexera import * + | + |class ProcessTupleOperator(UDFOperatorV2): + | @overrides + | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: + | tuple_['Region'] = tuple_['Region'] + '_reconfigured' + | yield tuple_ + |""".stripMargin + + val result = shouldReconfigure(List(sourceOpDesc, udfOpDesc1, udfOpDesc2), List(LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + udfOpDesc1.operatorIdentifier, + PortIdentity() + ), LogicalLink( + udfOpDesc1.operatorIdentifier, + PortIdentity(), + udfOpDesc2.operatorIdentifier, + PortIdentity() + )), + Seq(udfOpDesc1, udfOpDesc2), OpExecWithCode(code, "python")) + assert(result(udfOpDesc2.operatorIdentifier).exists { + t => t.getField("Region").asInstanceOf[String].contains("_reconfigured_reconfigured") + }) } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index 0f6fcd669a3..0dedd09e1fe 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -182,18 +182,19 @@ object TestOperators { udf } - def pythonSourceOpDesc(num_tuple: Int): PythonUDFSourceOpDescV2 = { + def pythonSourceOpDesc(num_tuple: Int, delay_in_sec: Double): PythonUDFSourceOpDescV2 = { val udf = new PythonUDFSourceOpDescV2() udf.workers = 1 udf.columns = List(new Attribute("field_1", AttributeType.INTEGER), new Attribute("field_2", AttributeType.STRING)) udf.code = s""" |from pytexera import * - | + |from time import sleep |class ProcessTupleOperator(UDFSourceOperator): | @overrides | def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]: | for i in range($num_tuple): + | sleep($delay_in_sec) | yield {'field_1': i, 'field_2': str(i)} |""".stripMargin udf From 9cc880267625580f179d1c04b80de18588cdd257 Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 15 Feb 2026 01:44:13 -0800 Subject: [PATCH 07/15] format fixes --- .../control/update_executor_handler.py | 32 ++++++++++++++++++- common/config/src/main/resources/storage.conf | 4 +-- common/config/src/main/resources/udf.conf | 2 +- .../texera/amber/operator/TestOperators.scala | 19 ----------- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py index a79d31c180a..07353337aa7 100644 --- a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py +++ b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py @@ -1 +1,31 @@ -# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from core.architecture.handlers.control.control_handler_base import ControlHandler from proto.org.apache.texera.amber.engine.architecture.rpc import ( EmptyReturn, UpdateExecutorRequest, ) from core.util import get_one_of from proto.org.apache.texera.amber.core import OpExecWithCode class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: op_exec_with_code: OpExecWithCode = get_one_of(req.new_exec_init_info) self.context.executor_manager.update_executor(op_exec_with_code.code, False) return EmptyReturn() \ No newline at end of file +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from core.architecture.handlers.control.control_handler_base import ControlHandler +from proto.org.apache.texera.amber.engine.architecture.rpc import ( + EmptyReturn, + UpdateExecutorRequest, +) +from core.util import get_one_of +from proto.org.apache.texera.amber.core import OpExecWithCode + + +class UpdateExecutorHandler(ControlHandler): + async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: + op_exec_with_code: OpExecWithCode = get_one_of(req.new_exec_init_info) + self.context.executor_manager.update_executor(op_exec_with_code.code, False) + return EmptyReturn() diff --git a/common/config/src/main/resources/storage.conf b/common/config/src/main/resources/storage.conf index f8b18689417..6ac5905765d 100644 --- a/common/config/src/main/resources/storage.conf +++ b/common/config/src/main/resources/storage.conf @@ -37,7 +37,7 @@ storage { username = "texera" username = ${?STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME} - password = "123456" + password = "" password = ${?STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD} } } @@ -134,7 +134,7 @@ storage { username = "postgres" username = ${?STORAGE_JDBC_USERNAME} - password = "123456" + password = "" password = ${?STORAGE_JDBC_PASSWORD} } } diff --git a/common/config/src/main/resources/udf.conf b/common/config/src/main/resources/udf.conf index c009aed0ded..7212d1720e5 100644 --- a/common/config/src/main/resources/udf.conf +++ b/common/config/src/main/resources/udf.conf @@ -17,7 +17,7 @@ python { # python3 executable path - path = "C:\\Users\\12198\\IdeaProjects\\texera\\.venv\\Scripts\\python.exe" + path = "" path = ${?UDF_PYTHON_PATH} log { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index 0dedd09e1fe..c602ccbdd38 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -176,27 +176,8 @@ object TestOperators { |class ProcessTupleOperator(UDFOperatorV2): | @overrides | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: - | print(tuple_) | yield tuple_ |""".stripMargin udf } - - def pythonSourceOpDesc(num_tuple: Int, delay_in_sec: Double): PythonUDFSourceOpDescV2 = { - val udf = new PythonUDFSourceOpDescV2() - udf.workers = 1 - udf.columns = List(new Attribute("field_1", AttributeType.INTEGER), new Attribute("field_2", AttributeType.STRING)) - udf.code = - s""" - |from pytexera import * - |from time import sleep - |class ProcessTupleOperator(UDFSourceOperator): - | @overrides - | def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]: - | for i in range($num_tuple): - | sleep($delay_in_sec) - | yield {'field_1': i, 'field_2': str(i)} - |""".stripMargin - udf - } } From b32587091d14921c335a1158148622981a2c29ef Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 15 Feb 2026 02:20:58 -0800 Subject: [PATCH 08/15] reformat --- .../core/architecture/rpc/async_rpc_handler_initializer.py | 4 +++- .../controller/promisehandlers/ReconfigurationHandler.scala | 4 ++-- .../worker/promisehandlers/InitializeExecutorHandler.scala | 3 --- .../worker/promisehandlers/UpdateExecutorHandler.scala | 1 - .../org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala | 4 ++-- .../org/apache/texera/amber/operator/TestOperators.scala | 2 -- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py b/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py index 7b60bc27fb2..146cf91b0d3 100644 --- a/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py +++ b/amber/src/main/python/core/architecture/rpc/async_rpc_handler_initializer.py @@ -45,7 +45,9 @@ from core.architecture.handlers.control.resume_worker_handler import ResumeWorkerHandler from core.architecture.handlers.control.start_channel_handler import StartChannelHandler from core.architecture.handlers.control.start_worker_handler import StartWorkerHandler -from core.architecture.handlers.control.update_executor_handler import UpdateExecutorHandler +from core.architecture.handlers.control.update_executor_handler import ( + UpdateExecutorHandler, +) class AsyncRPCHandlerInitializer( diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index dd31aae4207..beb82981211 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -21,10 +21,10 @@ package org.apache.texera.amber.engine.architecture.controller.promisehandlers import com.twitter.util.Future import org.apache.texera.amber.core.virtualidentity.{ChannelIdentity, EmbeddedControlMessageIdentity} -import org.apache.texera.amber.engine.architecture.controller.{ControllerAsyncRPCHandlerInitializer, ExecutionStatsUpdate} +import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.ALL_ALIGNMENT import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, WorkflowReconfigureRequest} -import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{EmptyReturn, PropagateEmbeddedControlMessageResponse} +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.common.FriesReconfigurationAlgorithm import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER import org.apache.texera.amber.util.VirtualIdentityUtils diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index a7fd26c4342..dacccafd62e 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -20,17 +20,14 @@ package org.apache.texera.amber.engine.architecture.worker.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.core.executor._ import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, InitializeExecutorRequest } import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer -import org.apache.texera.amber.operator.source.cache.CacheSourceOpExec import org.apache.texera.amber.util.VirtualIdentityUtils -import java.net.URI trait InitializeExecutorHandler { this: DataProcessorRPCHandlerInitializer => diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala index 5c2f65c257b..50fed1a22b8 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala @@ -25,7 +25,6 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyRetur import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer import org.apache.texera.amber.util.VirtualIdentityUtils -import scala.reflect.runtime.universe._ trait UpdateExecutorHandler { this: DataProcessorRPCHandlerInitializer => diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala index 41016a81444..da5e3c6d3d8 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -25,7 +25,7 @@ import org.apache.pekko.actor.{ActorSystem, Props} import org.apache.pekko.testkit.{ImplicitSender, TestKit} import org.apache.pekko.util.Timeout import org.apache.texera.amber.clustering.SingleNodeListener -import org.apache.texera.amber.core.executor.{OpExecInitInfo, OpExecWithClassName, OpExecWithCode} +import org.apache.texera.amber.core.executor.{OpExecInitInfo, OpExecWithCode} import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.storage.model.VirtualDocument import org.apache.texera.amber.core.tuple.Tuple @@ -38,7 +38,7 @@ import org.apache.texera.amber.engine.common.AmberRuntime import org.apache.texera.amber.engine.common.client.AmberClient import org.apache.texera.amber.engine.e2e.TestUtils.{cleanupWorkflowExecutionData, initiateTexeraDBForTestCases, setUpWorkflowExecutionData} import org.apache.texera.amber.operator.{LogicalOp, TestOperators} -import org.apache.texera.amber.operator.TestOperators.{mediumCsvScanOpDesc, pythonOpDesc, pythonSourceOpDesc} +import org.apache.texera.amber.operator.TestOperators.{mediumCsvScanOpDesc, pythonOpDesc} import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource.getResultUriByLogicalPortId import org.apache.texera.workflow.LogicalLink import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome, Retries} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index c602ccbdd38..f89204c6121 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -20,7 +20,6 @@ package org.apache.texera.amber.operator import org.apache.texera.amber.core.storage.FileResolver -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType} import org.apache.texera.amber.operator.aggregate.{AggregateOpDesc, AggregationFunction, AggregationOperation} import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc @@ -29,7 +28,6 @@ import org.apache.texera.amber.operator.source.scan.json.JSONLScanSourceOpDesc import org.apache.texera.amber.operator.source.sql.asterixdb.AsterixDBSourceOpDesc import org.apache.texera.amber.operator.source.sql.mysql.MySQLSourceOpDesc import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 -import org.apache.texera.amber.operator.udf.python.source.PythonUDFSourceOpDescV2 import java.nio.file.Path From 7002130a370c88131286f7d9054c5b94d34eb7ee Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 15 Feb 2026 02:23:42 -0800 Subject: [PATCH 09/15] reformat --- .../ReconfigurationHandler.scala | 119 +++++++----- .../DataProcessorRPCHandlerInitializer.scala | 15 +- .../InitializeExecutorHandler.scala | 1 - .../UpdateExecutorHandler.scala | 12 +- .../FriesReconfigurationAlgorithm.scala | 19 +- .../amber/engine/e2e/ModifyLogicSpec.scala | 173 +++++++++++------- .../texera/amber/operator/TestOperators.scala | 6 +- 7 files changed, 214 insertions(+), 131 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index beb82981211..3baa4d5069d 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -20,10 +20,16 @@ package org.apache.texera.amber.engine.architecture.controller.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.core.virtualidentity.{ChannelIdentity, EmbeddedControlMessageIdentity} +import org.apache.texera.amber.core.virtualidentity.{ + ChannelIdentity, + EmbeddedControlMessageIdentity +} import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer import org.apache.texera.amber.engine.architecture.rpc.controlcommands.EmbeddedControlMessageType.ALL_ALIGNMENT -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, WorkflowReconfigureRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ + AsyncRPCContext, + WorkflowReconfigureRequest +} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.common.FriesReconfigurationAlgorithm import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER @@ -35,62 +41,79 @@ import scala.collection.mutable trait ReconfigurationHandler { this: ControllerAsyncRPCHandlerInitializer => - override def reconfigureWorkflow(msg: WorkflowReconfigureRequest, ctx: AsyncRPCContext): Future[EmptyReturn] = { + override def reconfigureWorkflow( + msg: WorkflowReconfigureRequest, + ctx: AsyncRPCContext + ): Future[EmptyReturn] = { val futures = mutable.ArrayBuffer[Future[_]]() - val friesComponents = FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg) - val needToSendECMToSources = friesComponents.exists(comp => comp.sources.exists(sourceOp => cp.workflowScheduler.physicalPlan.getOperator(sourceOp).isSourceOperator)) - if(needToSendECMToSources){ + val friesComponents = + FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg) + val needToSendECMToSources = friesComponents.exists(comp => + comp.sources.exists(sourceOp => + cp.workflowScheduler.physicalPlan.getOperator(sourceOp).isSourceOperator + ) + ) + if (needToSendECMToSources) { throw new IllegalStateException( "Reconfiguration cannot be propagated through source operators" ) } - friesComponents.foreach{ - friesComponent => - if(friesComponent.scope.size == 1){ - val updateExecutorRequest = friesComponent.reconfigurations.head - val workerIds = cp.workflowExecution.getLatestOperatorExecution(updateExecutorRequest.targetOpId).getWorkerIds - workerIds.foreach{ - worker => - futures.append(workerInterface.updateExecutor(updateExecutorRequest, mkContext(worker))) - } - }else{ - val channelScope = cp.workflowExecution.getRunningRegionExecutions - .flatMap(regionExecution => - regionExecution.getAllLinkExecutions - .map(_._2) - .flatMap(linkExecution => linkExecution.getAllChannelExecutions.map(_._1)) - ).filter(channelId => { + friesComponents.foreach { friesComponent => + if (friesComponent.scope.size == 1) { + val updateExecutorRequest = friesComponent.reconfigurations.head + val workerIds = cp.workflowExecution + .getLatestOperatorExecution(updateExecutorRequest.targetOpId) + .getWorkerIds + workerIds.foreach { worker => + futures.append(workerInterface.updateExecutor(updateExecutorRequest, mkContext(worker))) + } + } else { + val channelScope = cp.workflowExecution.getRunningRegionExecutions + .flatMap(regionExecution => + regionExecution.getAllLinkExecutions + .map(_._2) + .flatMap(linkExecution => linkExecution.getAllChannelExecutions.map(_._1)) + ) + .filter(channelId => { + friesComponent.scope + .contains(VirtualIdentityUtils.getPhysicalOpId(channelId.fromWorkerId)) && friesComponent.scope - .contains(VirtualIdentityUtils.getPhysicalOpId(channelId.fromWorkerId)) && - friesComponent.scope - .contains(VirtualIdentityUtils.getPhysicalOpId(channelId.toWorkerId)) - }) - val controlChannels = friesComponent.sources.flatMap { source => - cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.flatMap { worker => - Seq( - ChannelIdentity(CONTROLLER, worker, isControl = true), - ChannelIdentity(worker, CONTROLLER, isControl = true) - ) - } + .contains(VirtualIdentityUtils.getPhysicalOpId(channelId.toWorkerId)) + }) + val controlChannels = friesComponent.sources.flatMap { source => + cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.flatMap { worker => + Seq( + ChannelIdentity(CONTROLLER, worker, isControl = true), + ChannelIdentity(worker, CONTROLLER, isControl = true) + ) } - val finalScope = channelScope ++ controlChannels - val cmdMapping = - friesComponent.reconfigurations .flatMap { updateReq => - val workers = cp.workflowExecution .getLatestOperatorExecution(updateReq.targetOpId) .getWorkerIds - workers.map(worker => worker.name -> createInvocation(METHOD_UPDATE_EXECUTOR.getBareMethodName, updateReq, worker)) } .toMap - futures += cmdMapping.map(_._2._2) - friesComponent.sources.foreach { source => - cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.foreach { worker => - sendECM( - EmbeddedControlMessageIdentity(msg.reconfigurationId), - ALL_ALIGNMENT, - finalScope.toSet, - cmdMapping.map(x => (x._1, x._2._1)), - ChannelIdentity(actorId, worker, isControl = true) + } + val finalScope = channelScope ++ controlChannels + val cmdMapping = + friesComponent.reconfigurations.flatMap { updateReq => + val workers = + cp.workflowExecution.getLatestOperatorExecution(updateReq.targetOpId).getWorkerIds + workers.map(worker => + worker.name -> createInvocation( + METHOD_UPDATE_EXECUTOR.getBareMethodName, + updateReq, + worker ) - } + ) + }.toMap + futures += cmdMapping.map(_._2._2) + friesComponent.sources.foreach { source => + cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.foreach { worker => + sendECM( + EmbeddedControlMessageIdentity(msg.reconfigurationId), + ALL_ALIGNMENT, + finalScope.toSet, + cmdMapping.map(x => (x._1, x._2._1)), + ChannelIdentity(actorId, worker, isControl = true) + ) } } + } } Future.collect(futures.toList).map { _ => EmptyReturn() diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala index cfbc40b5879..6b0c62ac3f2 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/DataProcessorRPCHandlerInitializer.scala @@ -20,9 +20,20 @@ package org.apache.texera.amber.engine.architecture.worker import com.twitter.util.Future -import org.apache.texera.amber.core.executor.{ExecFactory, OpExecInitInfo, OpExecSource, OpExecWithClassName, OpExecWithCode} +import org.apache.texera.amber.core.executor.{ + ExecFactory, + OpExecInitInfo, + OpExecSource, + OpExecWithClassName, + OpExecWithCode +} import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, DebugCommandRequest, EmptyRequest, EvaluatePythonExpressionRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ + AsyncRPCContext, + DebugCommandRequest, + EmptyRequest, + EvaluatePythonExpressionRequest +} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{EmptyReturn, EvaluatedValue} import org.apache.texera.amber.engine.architecture.rpc.workerservice.WorkerServiceFs2Grpc import org.apache.texera.amber.engine.architecture.worker.promisehandlers._ diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala index dacccafd62e..212a980e5ed 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala @@ -28,7 +28,6 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyRetur import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer import org.apache.texera.amber.util.VirtualIdentityUtils - trait InitializeExecutorHandler { this: DataProcessorRPCHandlerInitializer => diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala index 50fed1a22b8..ddb43bfea58 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala @@ -20,19 +20,21 @@ package org.apache.texera.amber.engine.architecture.worker.promisehandlers import com.twitter.util.Future -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{AsyncRPCContext, UpdateExecutorRequest} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ + AsyncRPCContext, + UpdateExecutorRequest +} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn import org.apache.texera.amber.engine.architecture.worker.DataProcessorRPCHandlerInitializer import org.apache.texera.amber.util.VirtualIdentityUtils - trait UpdateExecutorHandler { this: DataProcessorRPCHandlerInitializer => override def updateExecutor( - request: UpdateExecutorRequest, - ctx: AsyncRPCContext - ): Future[EmptyReturn] = { + request: UpdateExecutorRequest, + ctx: AsyncRPCContext + ): Future[EmptyReturn] = { val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) setupExecutor(request.newExecInitInfo, workerIdx, cachedTotalWorkerCount) dp.executor.open() diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala index f1c82e6343b..c13e7801190 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/FriesReconfigurationAlgorithm.scala @@ -37,15 +37,16 @@ object FriesReconfigurationAlgorithm { case class FriesComponent( sources: Set[PhysicalOpIdentity], scope: Set[PhysicalOpIdentity], - reconfigurations: Set[UpdateExecutorRequest]) + reconfigurations: Set[UpdateExecutorRequest] + ) private def getOneToManyOperators(region: Region): Set[PhysicalOpIdentity] = { region.getOperators.filter(op => op.isOneToManyOp).map(op => op.id) } def getReconfigurations( - workflowExecutionCoordinator: WorkflowExecutionCoordinator, - reconfiguration: WorkflowReconfigureRequest + workflowExecutionCoordinator: WorkflowExecutionCoordinator, + reconfiguration: WorkflowReconfigureRequest ): Set[FriesComponent] = { // independently schedule reconfigurations for each region: workflowExecutionCoordinator.getExecutingRegions @@ -112,13 +113,13 @@ object FriesReconfigurationAlgorithm { val componentSet = component.asScala.toSet val componentPlan = mcsPlan.getSubPlan(componentSet) val reconfigCommands = - reconfiguration.reconfiguration - .filter(req => component.contains(req.targetOpId)) - .toSet + reconfiguration.reconfiguration + .filter(req => component.contains(req.targetOpId)) + .toSet - // find the source operators of the component - val sources = componentSet.intersect(mcsPlan.getSourceOperatorIds) - epochMarkers += FriesComponent(sources, componentPlan.operators.map(_.id), reconfigCommands) + // find the source operators of the component + val sources = componentSet.intersect(mcsPlan.getSourceOperatorIds) + epochMarkers += FriesComponent(sources, componentPlan.operators.map(_.id), reconfigCommands) }) epochMarkers.toList } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala index da5e3c6d3d8..cc5b6a55fa6 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -31,12 +31,23 @@ import org.apache.texera.amber.core.storage.model.VirtualDocument import org.apache.texera.amber.core.tuple.Tuple import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} -import org.apache.texera.amber.engine.architecture.controller.{ControllerConfig, ExecutionStateUpdate} -import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{EmptyRequest, UpdateExecutorRequest, WorkflowReconfigureRequest} +import org.apache.texera.amber.engine.architecture.controller.{ + ControllerConfig, + ExecutionStateUpdate +} +import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ + EmptyRequest, + UpdateExecutorRequest, + WorkflowReconfigureRequest +} import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.COMPLETED import org.apache.texera.amber.engine.common.AmberRuntime import org.apache.texera.amber.engine.common.client.AmberClient -import org.apache.texera.amber.engine.e2e.TestUtils.{cleanupWorkflowExecutionData, initiateTexeraDBForTestCases, setUpWorkflowExecutionData} +import org.apache.texera.amber.engine.e2e.TestUtils.{ + cleanupWorkflowExecutionData, + initiateTexeraDBForTestCases, + setUpWorkflowExecutionData +} import org.apache.texera.amber.operator.{LogicalOp, TestOperators} import org.apache.texera.amber.operator.TestOperators.{mediumCsvScanOpDesc, pythonOpDesc} import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource.getResultUriByLogicalPortId @@ -46,18 +57,19 @@ import org.scalatest.flatspec.AnyFlatSpecLike import scala.concurrent.duration._ -class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntime.akkaConfig)) - with ImplicitSender - with AnyFlatSpecLike - with BeforeAndAfterAll - with BeforeAndAfterEach - with Retries { +class ModifyLogicSpec + extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntime.akkaConfig)) + with ImplicitSender + with AnyFlatSpecLike + with BeforeAndAfterAll + with BeforeAndAfterEach + with Retries { /** - * This block retries each test once if it fails. - * In the CI environment, there is a chance that executeWorkflow does not receive "COMPLETED" status. - * Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs. - */ + * This block retries each test once if it fails. + * In the CI environment, there is a chance that executeWorkflow does not receive "COMPLETED" status. + * Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs. + */ override def withFixture(test: NoArgTest): Outcome = withRetry { super.withFixture(test) } @@ -86,13 +98,12 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim TestKit.shutdownActorSystem(system) } - def shouldReconfigure( - operators: List[LogicalOp], - links: List[LogicalLink], - targetOps: Seq[LogicalOp], - newOpExecInitInfo: OpExecInitInfo - ): Map[OperatorIdentity, List[Tuple]] = { + operators: List[LogicalOp], + links: List[LogicalLink], + targetOps: Seq[LogicalOp], + newOpExecInitInfo: OpExecInitInfo + ): Map[OperatorIdentity, List[Tuple]] = { val workflow = TestUtils.buildWorkflow(operators, links, ctx) val client = @@ -139,18 +150,24 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim Await.result(client.controllerInterface.startWorkflow(EmptyRequest(), ())) Await.result(client.controllerInterface.pauseWorkflow(EmptyRequest(), ())) Thread.sleep(4000) - val physicalOps = targetOps.flatMap(op => workflow.physicalPlan.getPhysicalOpsOfLogicalOp(op.operatorIdentifier)) - Await.result(client.controllerInterface.reconfigureWorkflow(WorkflowReconfigureRequest( - reconfiguration = - physicalOps.map(op => UpdateExecutorRequest(op.id, newOpExecInitInfo) - ), reconfigurationId = "test-reconfigure-1"), ())) + val physicalOps = targetOps.flatMap(op => + workflow.physicalPlan.getPhysicalOpsOfLogicalOp(op.operatorIdentifier) + ) + Await.result( + client.controllerInterface.reconfigureWorkflow( + WorkflowReconfigureRequest( + reconfiguration = physicalOps.map(op => UpdateExecutorRequest(op.id, newOpExecInitInfo)), + reconfigurationId = "test-reconfigure-1" + ), + () + ) + ) Await.result(client.controllerInterface.resumeWorkflow(EmptyRequest(), ())) Thread.sleep(400) Await.result(completion, Duration.fromMinutes(1)) result } - "Engine" should "be able to modify a python UDF worker in workflow" in { val sourceOpDesc = TestOperators.smallCsvScanOpDesc() val udfOpDesc = pythonOpDesc() @@ -164,29 +181,40 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim | yield tuple_ |""".stripMargin - val result = shouldReconfigure(List(sourceOpDesc, udfOpDesc), List(LogicalLink( - sourceOpDesc.operatorIdentifier, - PortIdentity(), - udfOpDesc.operatorIdentifier, - PortIdentity() - )), - Seq(udfOpDesc), OpExecWithCode(code, "python")) - assert(result(udfOpDesc.operatorIdentifier).exists { - t => t.getField("Region").asInstanceOf[String].contains("_reconfigured") - }) + val result = shouldReconfigure( + List(sourceOpDesc, udfOpDesc), + List( + LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + udfOpDesc.operatorIdentifier, + PortIdentity() + ) + ), + Seq(udfOpDesc), + OpExecWithCode(code, "python") + ) + assert(result(udfOpDesc.operatorIdentifier).exists { t => + t.getField("Region").asInstanceOf[String].contains("_reconfigured") + }) } "Engine" should "be able to modify a java operator in workflow" in { val sourceOpDesc = mediumCsvScanOpDesc() val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") val keywordMatchManyOpDesc = TestOperators.keywordSearchOpDesc("Region", "Asia") - val result = shouldReconfigure(List(sourceOpDesc, keywordMatchNoneOpDesc), List(LogicalLink( - sourceOpDesc.operatorIdentifier, - PortIdentity(), - keywordMatchNoneOpDesc.operatorIdentifier, - PortIdentity() - )), - Seq(keywordMatchNoneOpDesc), keywordMatchManyOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo + val result = shouldReconfigure( + List(sourceOpDesc, keywordMatchNoneOpDesc), + List( + LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + keywordMatchNoneOpDesc.operatorIdentifier, + PortIdentity() + ) + ), + Seq(keywordMatchNoneOpDesc), + keywordMatchManyOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo ) assert(result(keywordMatchNoneOpDesc.operatorIdentifier).nonEmpty) } @@ -195,15 +223,23 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim val sourceOpDesc = mediumCsvScanOpDesc() val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") val ex = intercept[Throwable] { - shouldReconfigure(List(sourceOpDesc, keywordMatchNoneOpDesc), List(LogicalLink( - sourceOpDesc.operatorIdentifier, - PortIdentity(), - keywordMatchNoneOpDesc.operatorIdentifier, - PortIdentity() - )), - Seq(sourceOpDesc), sourceOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo - )} - assert(ex.getMessage == "java.lang.IllegalStateException: Reconfiguration cannot be propagated through source operators") + shouldReconfigure( + List(sourceOpDesc, keywordMatchNoneOpDesc), + List( + LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + keywordMatchNoneOpDesc.operatorIdentifier, + PortIdentity() + ) + ), + Seq(sourceOpDesc), + sourceOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo + ) + } + assert( + ex.getMessage == "java.lang.IllegalStateException: Reconfiguration cannot be propagated through source operators" + ) } "Engine" should "be able to modify two python UDFs in workflow" in { @@ -220,20 +256,27 @@ class ModifyLogicSpec extends TestKit(ActorSystem("ModifyLogicSpec", AmberRuntim | yield tuple_ |""".stripMargin - val result = shouldReconfigure(List(sourceOpDesc, udfOpDesc1, udfOpDesc2), List(LogicalLink( - sourceOpDesc.operatorIdentifier, - PortIdentity(), - udfOpDesc1.operatorIdentifier, - PortIdentity() - ), LogicalLink( - udfOpDesc1.operatorIdentifier, - PortIdentity(), - udfOpDesc2.operatorIdentifier, - PortIdentity() - )), - Seq(udfOpDesc1, udfOpDesc2), OpExecWithCode(code, "python")) - assert(result(udfOpDesc2.operatorIdentifier).exists { - t => t.getField("Region").asInstanceOf[String].contains("_reconfigured_reconfigured") + val result = shouldReconfigure( + List(sourceOpDesc, udfOpDesc1, udfOpDesc2), + List( + LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + udfOpDesc1.operatorIdentifier, + PortIdentity() + ), + LogicalLink( + udfOpDesc1.operatorIdentifier, + PortIdentity(), + udfOpDesc2.operatorIdentifier, + PortIdentity() + ) + ), + Seq(udfOpDesc1, udfOpDesc2), + OpExecWithCode(code, "python") + ) + assert(result(udfOpDesc2.operatorIdentifier).exists { t => + t.getField("Region").asInstanceOf[String].contains("_reconfigured_reconfigured") }) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index f89204c6121..bfa2d64ce6d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -20,7 +20,11 @@ package org.apache.texera.amber.operator import org.apache.texera.amber.core.storage.FileResolver -import org.apache.texera.amber.operator.aggregate.{AggregateOpDesc, AggregationFunction, AggregationOperation} +import org.apache.texera.amber.operator.aggregate.{ + AggregateOpDesc, + AggregationFunction, + AggregationOperation +} import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc import org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc From 053aa3172243b60ca6b78c75f7b058c3d765981b Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 15 Feb 2026 02:28:48 -0800 Subject: [PATCH 10/15] Update storage.conf --- common/config/src/main/resources/storage.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/config/src/main/resources/storage.conf b/common/config/src/main/resources/storage.conf index 6ac5905765d..85a62b77a3b 100644 --- a/common/config/src/main/resources/storage.conf +++ b/common/config/src/main/resources/storage.conf @@ -37,7 +37,7 @@ storage { username = "texera" username = ${?STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME} - password = "" + password = "password" password = ${?STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD} } } @@ -134,7 +134,7 @@ storage { username = "postgres" username = ${?STORAGE_JDBC_USERNAME} - password = "" + password = "postgres" password = ${?STORAGE_JDBC_PASSWORD} } } From 5a98a473bef18d464f2f5686c1a87bc81bfad916 Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sun, 15 Feb 2026 12:43:23 -0800 Subject: [PATCH 11/15] allow propagation through sources --- .github/workflows/github-action-build.yml | 5 ++ .../control/update_executor_handler.py | 4 +- .../main/python/core/models/internal_queue.py | 3 -- .../main/python/core/runnables/main_loop.py | 8 +++- .../python/core/runnables/network_receiver.py | 3 +- .../ReconfigurationHandler.scala | 17 ++++--- .../engine/common/rpc/AsyncRPCServer.scala | 4 +- .../amber/engine/e2e/ModifyLogicSpec.scala | 46 +++++++++++++++---- .../texera/amber/operator/TestOperators.scala | 19 ++++++++ 9 files changed, 83 insertions(+), 26 deletions(-) diff --git a/.github/workflows/github-action-build.yml b/.github/workflows/github-action-build.yml index 2bffd133eca..fd8b495a1ec 100644 --- a/.github/workflows/github-action-build.yml +++ b/.github/workflows/github-action-build.yml @@ -115,6 +115,11 @@ jobs: python-version: '3.11' - name: Show Python run: python --version || python3 --version + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f amber/requirements.txt ]; then pip install -r amber/requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then pip install -r amber/operator-requirements.txt; fi - name: Setup sbt launcher uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 diff --git a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py index 07353337aa7..f2b6d16d46a 100644 --- a/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py +++ b/amber/src/main/python/core/architecture/handlers/control/update_executor_handler.py @@ -27,5 +27,7 @@ class UpdateExecutorHandler(ControlHandler): async def update_executor(self, req: UpdateExecutorRequest) -> EmptyReturn: op_exec_with_code: OpExecWithCode = get_one_of(req.new_exec_init_info) - self.context.executor_manager.update_executor(op_exec_with_code.code, False) + self.context.executor_manager.update_executor( + op_exec_with_code.code, self.context.executor_manager.executor.is_source + ) return EmptyReturn() diff --git a/amber/src/main/python/core/models/internal_queue.py b/amber/src/main/python/core/models/internal_queue.py index ae0247aa2db..abc1793ff6c 100644 --- a/amber/src/main/python/core/models/internal_queue.py +++ b/amber/src/main/python/core/models/internal_queue.py @@ -79,9 +79,6 @@ def put(self, item: T) -> None: if item.tag not in self._queue_ids: self._queue.add_sub_queue(item.tag, 1 if item.tag.is_control else 2) self._queue_ids.add(item.tag) - if len(self._queue_state) > 0 and not item.tag.is_control: - # if paused, then the newly added queue will also be paused. - self._queue.disable(item.tag) if isinstance(item, (DataElement, InternalMarker, ECMElement)): self._queue.put(item.tag, item) elif isinstance(item, DCMElement): diff --git a/amber/src/main/python/core/runnables/main_loop.py b/amber/src/main/python/core/runnables/main_loop.py index d73c655734f..794224c97f3 100644 --- a/amber/src/main/python/core/runnables/main_loop.py +++ b/amber/src/main/python/core/runnables/main_loop.py @@ -118,7 +118,13 @@ def _check_and_process_control(self) -> None: or not self._input_queue.is_data_enabled() ): next_entry = self.interruptible_get() - self._process_dcm(next_entry) + match( + next_entry, + DCMElement, + self._process_dcm, + ECMElement, + self._process_ecm, + ) @overrides def pre_start(self) -> None: diff --git a/amber/src/main/python/core/runnables/network_receiver.py b/amber/src/main/python/core/runnables/network_receiver.py index fd42a8f589b..d5a765c5739 100644 --- a/amber/src/main/python/core/runnables/network_receiver.py +++ b/amber/src/main/python/core/runnables/network_receiver.py @@ -90,7 +90,8 @@ def data_handler(command: bytes, table: Table) -> int: # Explicitly set is_control to trigger lazy computation. # If not set, it may be computed at different times, # causing hash inconsistencies. - data_header.tag.is_control = False + if not data_header.tag.is_control: + data_header.tag.is_control = False payload = match( data_header.payload_type, "Data", diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index 3baa4d5069d..2bc501c38dc 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -45,19 +45,18 @@ trait ReconfigurationHandler { msg: WorkflowReconfigureRequest, ctx: AsyncRPCContext ): Future[EmptyReturn] = { - val futures = mutable.ArrayBuffer[Future[_]]() - val friesComponents = - FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg) - val needToSendECMToSources = friesComponents.exists(comp => - comp.sources.exists(sourceOp => - cp.workflowScheduler.physicalPlan.getOperator(sourceOp).isSourceOperator + if ( + msg.reconfiguration.exists(req => + cp.workflowScheduler.physicalPlan.getOperator(req.targetOpId).isSourceOperator ) - ) - if (needToSendECMToSources) { + ) { throw new IllegalStateException( - "Reconfiguration cannot be propagated through source operators" + "Reconfiguration cannot be applied to source operators" ) } + val futures = mutable.ArrayBuffer[Future[_]]() + val friesComponents = + FriesReconfigurationAlgorithm.getReconfigurations(cp.workflowExecutionCoordinator, msg) friesComponents.foreach { friesComponent => if (friesComponent.scope.size == 1) { val updateExecutorRequest = friesComponent.reconfigurations.head diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala index 78c1ed206e7..e9a3e2cc455 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/common/rpc/AsyncRPCServer.scala @@ -86,9 +86,7 @@ class AsyncRPCServer( } catch { case e: java.lang.reflect.InvocationTargetException => throw Option(e.getCause).getOrElse(e) - case e: Throwable => - logger.info(s"error when invoking ${method.getName} + ${requestArg} with ${contextArg}") - throw e + case e: Throwable => throw e } result .asInstanceOf[Future[ControlReturn]] diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala index cc5b6a55fa6..83e60bc6519 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -49,7 +49,6 @@ import org.apache.texera.amber.engine.e2e.TestUtils.{ setUpWorkflowExecutionData } import org.apache.texera.amber.operator.{LogicalOp, TestOperators} -import org.apache.texera.amber.operator.TestOperators.{mediumCsvScanOpDesc, pythonOpDesc} import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource.getResultUriByLogicalPortId import org.apache.texera.workflow.LogicalLink import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome, Retries} @@ -170,7 +169,7 @@ class ModifyLogicSpec "Engine" should "be able to modify a python UDF worker in workflow" in { val sourceOpDesc = TestOperators.smallCsvScanOpDesc() - val udfOpDesc = pythonOpDesc() + val udfOpDesc = TestOperators.pythonOpDesc() val code = """ |from pytexera import * | @@ -200,7 +199,7 @@ class ModifyLogicSpec } "Engine" should "be able to modify a java operator in workflow" in { - val sourceOpDesc = mediumCsvScanOpDesc() + val sourceOpDesc = TestOperators.mediumCsvScanOpDesc() val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") val keywordMatchManyOpDesc = TestOperators.keywordSearchOpDesc("Region", "Asia") val result = shouldReconfigure( @@ -220,7 +219,8 @@ class ModifyLogicSpec } "Engine" should "not be able to modify a source operator in workflow" in { - val sourceOpDesc = mediumCsvScanOpDesc() + val sourceOpDesc = TestOperators.mediumCsvScanOpDesc() + val sourceOpDesc2 = TestOperators.mediumCsvScanOpDesc() val keywordMatchNoneOpDesc = TestOperators.keywordSearchOpDesc("Region", "ShouldMatchNone") val ex = intercept[Throwable] { shouldReconfigure( @@ -234,18 +234,48 @@ class ModifyLogicSpec ) ), Seq(sourceOpDesc), - sourceOpDesc.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo + sourceOpDesc2.getPhysicalOp(ctx.workflowId, ctx.executionId).opExecInitInfo ) } assert( - ex.getMessage == "java.lang.IllegalStateException: Reconfiguration cannot be propagated through source operators" + ex.getMessage == "java.lang.IllegalStateException: Reconfiguration cannot be applied to source operators" ) } + "Engine" should "propagate reconfiguration through a source operator in workflow" in { + val sourceOpDesc = TestOperators.pythonSourceOpDesc(10000) + val udfOpDesc = TestOperators.pythonOpDesc() + val code = """ + |from pytexera import * + | + |class ProcessTupleOperator(UDFOperatorV2): + | @overrides + | def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: + | tuple_['field_1'] = tuple_['field_1'] + '_reconfigured' + | yield tuple_ + |""".stripMargin + val result = shouldReconfigure( + List(sourceOpDesc, udfOpDesc), + List( + LogicalLink( + sourceOpDesc.operatorIdentifier, + PortIdentity(), + udfOpDesc.operatorIdentifier, + PortIdentity() + ) + ), + Seq(udfOpDesc), + OpExecWithCode(code, "python") + ) + assert(result(udfOpDesc.operatorIdentifier).exists { t => + t.getField("field_1").asInstanceOf[String].contains("_reconfigured") + }) + } + "Engine" should "be able to modify two python UDFs in workflow" in { val sourceOpDesc = TestOperators.smallCsvScanOpDesc() - val udfOpDesc1 = pythonOpDesc() - val udfOpDesc2 = pythonOpDesc() + val udfOpDesc1 = TestOperators.pythonOpDesc() + val udfOpDesc2 = TestOperators.pythonOpDesc() val code = """ |from pytexera import * | diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala index bfa2d64ce6d..268b06ff8be 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/TestOperators.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator import org.apache.texera.amber.core.storage.FileResolver +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType} import org.apache.texera.amber.operator.aggregate.{ AggregateOpDesc, AggregationFunction, @@ -32,6 +33,7 @@ import org.apache.texera.amber.operator.source.scan.json.JSONLScanSourceOpDesc import org.apache.texera.amber.operator.source.sql.asterixdb.AsterixDBSourceOpDesc import org.apache.texera.amber.operator.source.sql.mysql.MySQLSourceOpDesc import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 +import org.apache.texera.amber.operator.udf.python.source.PythonUDFSourceOpDescV2 import java.nio.file.Path @@ -182,4 +184,21 @@ object TestOperators { |""".stripMargin udf } + + def pythonSourceOpDesc(numTuple: Int): PythonUDFSourceOpDescV2 = { + val udf = new PythonUDFSourceOpDescV2() + udf.workers = 1 + udf.columns = List(new Attribute("field_1", AttributeType.STRING)) + udf.code = s""" + |from pytexera import * + | + |class UDFSourceOperator(UDFSourceOperator): + | @overrides + | def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]: + | for i in range($numTuple): + | yield {'field_1': str(i) } + |""".stripMargin + udf + } + } From 5b09f08fead0ca27b06bfd203b1e1197b97ae34b Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Fri, 27 Feb 2026 19:07:03 -0800 Subject: [PATCH 12/15] Update amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> --- .../org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala index 83e60bc6519..718478d0e72 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/ModifyLogicSpec.scala @@ -67,7 +67,7 @@ class ModifyLogicSpec /** * This block retries each test once if it fails. * In the CI environment, there is a chance that executeWorkflow does not receive "COMPLETED" status. - * Until we find the root cause of this issue, we use a retry mechanism here to stablize CI runs. + * Until we find the root cause of this issue, we use a retry mechanism here to stabilize CI runs. */ override def withFixture(test: NoArgTest): Outcome = withRetry { super.withFixture(test) } From 0c0f43df04feb5fbe854da271636d007ec43720a Mon Sep 17 00:00:00 2001 From: Shengquan Ni <13672781+shengquan-ni@users.noreply.github.com> Date: Sat, 28 Feb 2026 18:21:04 -0800 Subject: [PATCH 13/15] Update amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Shengquan Ni <13672781+shengquan-ni@users.noreply.github.com> --- .../worker/promisehandlers/UpdateExecutorHandler.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala index ddb43bfea58..8ed9ebdc595 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/worker/promisehandlers/UpdateExecutorHandler.scala @@ -36,6 +36,11 @@ trait UpdateExecutorHandler { ctx: AsyncRPCContext ): Future[EmptyReturn] = { val workerIdx = VirtualIdentityUtils.getWorkerIndex(actorId) + // Close the existing executor (if any) before replacing it to avoid resource leaks. + val oldExecutor = dp.executor + if (oldExecutor != null) { + oldExecutor.close() + } setupExecutor(request.newExecInitInfo, workerIdx, cachedTotalWorkerCount) dp.executor.open() EmptyReturn() From cfdce3c8c59857e100b76042c5f59e7adfe532c2 Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sat, 28 Feb 2026 18:41:29 -0800 Subject: [PATCH 14/15] Update ReconfigurationHandler.scala --- .../controller/promisehandlers/ReconfigurationHandler.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index 2bc501c38dc..1fc04c998c4 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -100,7 +100,9 @@ trait ReconfigurationHandler { ) ) }.toMap - futures += cmdMapping.map(_._2._2) + futures += cmdMapping.map{ + case (_, (_, singleWorkerUpdateFuture)) => singleWorkerUpdateFuture + } friesComponent.sources.foreach { source => cp.workflowExecution.getLatestOperatorExecution(source).getWorkerIds.foreach { worker => sendECM( From b80f9d3754f87ac28878b62fc6ef4bf85b2ecfde Mon Sep 17 00:00:00 2001 From: Shengquan Ni Date: Sat, 28 Feb 2026 19:02:01 -0800 Subject: [PATCH 15/15] Update ReconfigurationHandler.scala --- .../controller/promisehandlers/ReconfigurationHandler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala index 1fc04c998c4..210d7c5b98c 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/ReconfigurationHandler.scala @@ -100,7 +100,7 @@ trait ReconfigurationHandler { ) ) }.toMap - futures += cmdMapping.map{ + futures += cmdMapping.map { case (_, (_, singleWorkerUpdateFuture)) => singleWorkerUpdateFuture } friesComponent.sources.foreach { source =>