From 7b6fe0a989caa1b3ae6ad5d7dbe05539aaae41e2 Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Thu, 16 Apr 2026 23:14:10 +0530 Subject: [PATCH] TEZ-4715: Migrate Tez PB Protocols from ProtobufRPCEngine to ProtobufRPCEngine2 --- .codespellrc | 1 - pom.xml | 52 +++++++++++++++++++ tez-api/pom.xml | 8 ++- .../apache/tez/client/FrameworkClient.java | 3 +- .../java/org/apache/tez/client/TezClient.java | 2 +- .../org/apache/tez/client/TezClientUtils.java | 4 +- .../java/org/apache/tez/common/RPCUtil.java | 3 +- .../org/apache/tez/common/TezCommonUtils.java | 3 +- .../java/org/apache/tez/common/TezUtils.java | 5 +- .../apache/tez/dag/api/DagTypeConverters.java | 4 +- .../dag/api/client/rpc/DAGClientRPCImpl.java | 2 +- .../org/apache/tez/client/TestTezClient.java | 4 +- .../org/apache/tez/common/TestRPCUtil.java | 4 +- .../tez/dag/api/client/rpc/TestDAGClient.java | 5 +- tez-common/pom.xml | 4 +- .../apache/tez/common/TezUtilsInternal.java | 7 ++- .../org/apache/tez/common/TestTezUtils.java | 3 +- tez-dag/pom.xml | 8 ++- .../org/apache/tez/client/LocalClient.java | 2 +- .../tez/dag/api/client/DAGClientServer.java | 7 ++- ...GClientAMProtocolBlockingPBServerImpl.java | 7 ++- .../apache/tez/dag/app/RecoveryParser.java | 2 +- .../tez/dag/app/dag/impl/VertexImpl.java | 4 +- .../apache/tez/dag/history/HistoryEvent.java | 4 +- .../dag/history/events/AMLaunchedEvent.java | 7 ++- .../dag/history/events/AMStartedEvent.java | 7 ++- .../dag/history/events/AppLaunchedEvent.java | 5 +- .../events/ContainerLaunchedEvent.java | 7 ++- .../history/events/ContainerStoppedEvent.java | 7 ++- .../history/events/DAGCommitStartedEvent.java | 7 ++- .../dag/history/events/DAGFinishedEvent.java | 8 +-- .../history/events/DAGInitializedEvent.java | 7 ++- .../history/events/DAGKillRequestEvent.java | 7 ++- .../dag/history/events/DAGRecoveredEvent.java | 5 +- .../dag/history/events/DAGStartedEvent.java | 7 ++- .../dag/history/events/DAGSubmittedEvent.java | 7 ++- .../events/TaskAttemptFinishedEvent.java | 6 +-- .../events/TaskAttemptStartedEvent.java | 7 ++- .../dag/history/events/TaskFinishedEvent.java | 7 ++- .../dag/history/events/TaskStartedEvent.java | 7 ++- .../events/VertexCommitStartedEvent.java | 9 ++-- .../events/VertexConfigurationDoneEvent.java | 6 +-- .../history/events/VertexFinishedEvent.java | 7 ++- .../VertexGroupCommitFinishedEvent.java | 6 +-- .../events/VertexGroupCommitStartedEvent.java | 6 +-- .../events/VertexInitializedEvent.java | 6 +-- .../history/events/VertexStartedEvent.java | 7 ++- .../dag/history/recovery/RecoveryService.java | 2 +- .../org/apache/tez/dag/utils/ProtoUtils.java | 3 +- .../apache/tez/dag/app/TestDAGAppMaster.java | 2 +- .../tez/dag/app/dag/impl/TestDAGImpl.java | 2 +- .../tez/dag/app/dag/impl/TestVertexImpl.java | 2 +- .../TestHistoryEventsProtoConversion.java | 4 +- .../GraceShuffleVertexManagerForTest.java | 3 +- tez-ext-service-tests/pom.xml | 8 +++ .../dag/app/TezTestServiceCommunicator.java | 2 +- .../TezTestServiceContainerLauncher.java | 3 +- .../TezTestServiceTaskCommunicatorImpl.java | 5 +- .../TezTestServiceProtocolClientImpl.java | 9 ++-- .../TezTestServiceProtocolServerImpl.java | 10 ++-- tez-mapreduce/pom.xml | 8 ++- .../tez/mapreduce/hadoop/MRInputHelpers.java | 2 +- .../tez/mapreduce/input/MultiMRInput.java | 2 +- .../apache/tez/mapreduce/output/MROutput.java | 2 +- .../common/TestMRInputAMSplitGenerator.java | 4 +- .../common/TestMRInputSplitDistributor.java | 4 +- .../mapreduce/hadoop/TestMRInputHelpers.java | 4 +- tez-plugins/tez-aux-services/pom.xml | 5 -- .../tez-protobuf-history-plugin/pom.xml | 8 ++- .../logging/proto/DatePartitionedLogger.java | 5 +- .../logging/proto/ProtoMessageReader.java | 5 +- .../logging/proto/ProtoMessageWritable.java | 11 ++-- .../logging/proto/ProtoMessageWriter.java | 5 +- .../proto/TestProtoHistoryLoggingService.java | 4 +- .../history/ats/acls/TestATSHistoryV15.java | 6 +-- tez-runtime-internals/pom.xml | 8 ++- .../apache/tez/common/ProtoConverters.java | 3 +- .../apache/tez/runtime/api/impl/TezEvent.java | 7 ++- tez-runtime-library/pom.xml | 8 ++- .../vertexmanager/FairEdgeConfiguration.java | 5 +- .../vertexmanager/FairShuffleEdgeManager.java | 3 +- .../vertexmanager/ShuffleVertexManager.java | 4 +- .../ShuffleVertexManagerBase.java | 4 +- .../CartesianProductConfig.java | 4 +- .../CartesianProductEdgeManager.java | 2 +- .../CartesianProductVertexManager.java | 2 +- .../FairCartesianProductVertexManager.java | 2 +- .../library/common/shuffle/ShuffleUtils.java | 2 +- .../impl/ShuffleInputEventHandlerImpl.java | 5 +- ...huffleInputEventHandlerOrderedGrouped.java | 5 +- .../writers/UnorderedPartitionedKVWriter.java | 2 +- .../TestShuffleVertexManagerUtils.java | 3 +- ...artesianProductEdgeManagerPartitioned.java | 4 +- ...TestFairCartesianProductVertexManager.java | 4 +- .../common/shuffle/TestShuffleUtils.java | 2 +- .../TestShuffleInputEventHandlerImpl.java | 4 +- ...huffleInputEventHandlerOrderedGrouped.java | 4 +- .../library/common/sort/impl/TestIFile.java | 4 +- .../sort/impl/dflt/TestDefaultSorter.java | 4 +- .../TestUnorderedPartitionedKVWriter.java | 2 +- .../output/TestOnFileSortedOutput.java | 4 +- .../output/TestOnFileUnorderedKVOutput.java | 4 +- .../TestOrderedPartitionedKVOutput2.java | 4 +- .../output/TestUnorderedKVOutput2.java | 4 +- .../TestUnorderedPartitionedKVOutput2.java | 4 +- .../RecoveryServiceWithEventHandlingHook.java | 5 +- 106 files changed, 279 insertions(+), 276 deletions(-) diff --git a/.codespellrc b/.codespellrc index fee7e92ef2..689484b8b2 100644 --- a/.codespellrc +++ b/.codespellrc @@ -16,6 +16,5 @@ # specific language governing permissions and limitations # under the License. # - [codespell] ignore-words-list = thirdparty,afterall,AfterAll diff --git a/pom.xml b/pom.xml index fec11d4d7c..a6950ce5d0 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,7 @@ 1.15.0 33.4.8-jre 3.5.0 + 1.5.0 2.18.6 2.46 1.5.4 @@ -109,6 +110,7 @@ ${project.build.directory}/logs 1.7.9 true + 1.5.3 ${scm.url} @@ -900,6 +902,11 @@ protobuf-java ${protobuf.version} + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 + ${hadoop-thirdparty.version} + com.google.guava guava @@ -1243,6 +1250,51 @@ + + com.google.code.maven-replacer-plugin + replacer + ${maven-replacer-plugin.version} + + + replace-generated-sources + generate-sources + + replace + + + ${project.basedir} + + target/generated-sources/**/*.java + + + + ([^\.])com.google.protobuf + $1org.apache.hadoop.thirdparty.protobuf + + + + + + replace-generated-test-sources + generate-sources + + replace + + + ${project.basedir} + + target/generated-test-sources/**/*.java + + + + ([^\.])com.google.protobuf + $1org.apache.hadoop.thirdparty.protobuf + + + + + + diff --git a/tez-api/pom.xml b/tez-api/pom.xml index e54e15226c..4da32ab62a 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -86,8 +86,8 @@ commons-collections4 - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.junit.jupiter @@ -196,6 +196,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java index 8361594310..3b22ca4932 100644 --- a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -49,8 +50,6 @@ import org.apache.tez.frameworkplugins.FrameworkUtils; import org.apache.tez.frameworkplugins.yarn.YarnClientFrameworkService; -import com.google.protobuf.ServiceException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index 474c181a33..2aed57f0bb 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -80,7 +81,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index 4c42bdf34f..845647f4f1 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -56,7 +56,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; @@ -1009,7 +1009,7 @@ public static DAGClientAMProtocolBlockingPB getAMProxy(final Configuration conf, proxy = userUgi.doAs(new PrivilegedExceptionAction() { @Override public DAGClientAMProtocolBlockingPB run() throws IOException { - RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine2.class); return (DAGClientAMProtocolBlockingPB) RPC.getProxy(DAGClientAMProtocolBlockingPB.class, 0, serviceAddr, conf); } diff --git a/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java b/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java index 8cc1ec55fb..cbcd5a6bab 100644 --- a/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java +++ b/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java @@ -23,13 +23,12 @@ import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.dag.api.DAGNotRunningException; import org.apache.tez.dag.api.NoCurrentDAGException; import org.apache.tez.dag.api.SessionNotRunning; import org.apache.tez.dag.api.TezException; -import com.google.protobuf.ServiceException; - public final class RPCUtil { private RPCUtil() {} diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java index d3980c7c31..2a9ab48320 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java @@ -44,6 +44,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.client.TezClient; import org.apache.tez.common.security.JobTokenIdentifier; @@ -51,8 +52,6 @@ import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.TezUncheckedException; -import com.google.protobuf.ByteString; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java index cf8b4ece0a..8274bc439b 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java @@ -28,15 +28,14 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.tez.client.TezClientUtils; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.records.DAGProtos; import org.apache.tez.runtime.api.TaskContext; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; - import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java index be42552e1b..24c99e19eb 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java @@ -39,6 +39,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.ByteString.Output; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -87,8 +89,6 @@ import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import com.google.protobuf.ByteString.Output; @Private public final class DagTypeConverters { diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java index f208121183..f0011a4eec 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -50,7 +51,6 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TryKillDAGRequestProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index 402c7a545e..ff2b1c00e7 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -57,6 +57,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -99,8 +101,6 @@ import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import com.google.common.collect.Maps; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; diff --git a/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java b/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java index f5766bd0df..2af91435f0 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java @@ -25,14 +25,12 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.dag.api.SessionNotRunning; import org.apache.tez.dag.api.TezException; -import com.google.protobuf.ServiceException; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - public class TestRPCUtil { @Test diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java index 788c1fbd6e..a137c51548 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java @@ -47,6 +47,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -83,9 +85,6 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto; import org.apache.tez.dag.api.records.DAGProtos.VertexStatusStateProto; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; diff --git a/tez-common/pom.xml b/tez-common/pom.xml index bce5be4cf3..e4f77b92aa 100644 --- a/tez-common/pom.xml +++ b/tez-common/pom.xml @@ -38,8 +38,8 @@ guava - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.apache.hadoop diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java index f6404cd980..f7d7019ef8 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java @@ -43,6 +43,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.Descriptors; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.log4j.Appender; @@ -65,10 +68,6 @@ import org.apache.tez.serviceplugins.api.TaskAttemptEndReason; import org.apache.tez.util.StopWatch; -import com.google.protobuf.ByteString; -import com.google.protobuf.Descriptors; -import com.google.protobuf.TextFormat; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java index de982840be..2f3ba0f448 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.client.TezClientUtils; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezConstants; @@ -43,8 +44,6 @@ import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor; import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor; -import com.google.protobuf.ByteString; - import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.jupiter.api.Test; diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index f4c9ed8867..e85a77be8a 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -104,8 +104,8 @@ slf4j-api - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.apache.commons @@ -238,6 +238,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java index ac9c26d30e..348ca7f0b9 100644 --- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java +++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -72,7 +73,6 @@ import org.apache.tez.dag.app.dag.DAG; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java index 785e08fa1a..7bbd6c363d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java @@ -25,12 +25,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.tez.dag.api.TezConfiguration; @@ -40,8 +41,6 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol; import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider; -import com.google.protobuf.BlockingService; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,7 +127,7 @@ public void setClientAMSecretKey(ByteBuffer key) { private Server createServer(Class pbProtocol, InetSocketAddress addr, Configuration conf, int numHandlers, BlockingService blockingService, String portRangeConfig) throws IOException { - RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine2.class); RPC.Server server = new RPC.Builder(conf).setProtocol(pbProtocol) .setInstance(blockingService).setBindAddress(addr.getHostString()) .setPort(addr.getPort()).setNumHandlers(numHandlers).setVerbose(false) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java index c5da19477a..a61233e43b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java @@ -27,6 +27,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.client.TezAppMasterStatus; import org.apache.tez.dag.api.DagTypeConverters; @@ -54,10 +57,6 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TryKillDAGResponseProto; import org.apache.tez.dag.api.records.DAGProtos.DAGPlan; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java index 573bc06b78..7c70fee43b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.util.functional.FutureIO; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -76,7 +77,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.CodedInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 90e04f160f..f8d73e5c49 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -51,6 +51,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.service.ServiceStateException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; @@ -205,8 +207,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java index 965528bd1b..db79b67972 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java @@ -20,8 +20,8 @@ import java.io.IOException; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; public interface HistoryEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java index 26e64ab3c4..c18165b91b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java @@ -20,16 +20,15 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.AMLaunchedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class AMLaunchedEvent implements HistoryEvent { private ApplicationAttemptId applicationAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java index 1d004ae0b9..ed68943952 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java @@ -20,16 +20,15 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.AMStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class AMStartedEvent implements HistoryEvent { private ApplicationAttemptId applicationAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java index 9396c701bb..b24fe6fe8f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java @@ -21,14 +21,13 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.VersionInfo; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - public class AppLaunchedEvent implements HistoryEvent { private ApplicationId applicationId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java index 3fa628caf9..f0e4236f32 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -27,10 +30,6 @@ import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.ContainerLaunchedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class ContainerLaunchedEvent implements HistoryEvent { private ContainerId containerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java index 376c7dd232..4ed022c222 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -27,10 +30,6 @@ import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.ContainerStoppedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class ContainerStoppedEvent implements HistoryEvent { private ContainerId containerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java index 76d5171fe6..2653e9ba87 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.io.OutputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -29,10 +32,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGCommitStartedEvent implements HistoryEvent, SummaryEvent { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java index e97d6fa042..a029caac0c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java @@ -22,6 +22,10 @@ import java.io.OutputStream; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.DagTypeConverters; @@ -37,10 +41,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class DAGFinishedEvent implements HistoryEvent, SummaryEvent, DAGIDAware { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java index 16294cf71a..25c6c5942b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.records.DAGIDAware; @@ -29,10 +32,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos; import org.apache.tez.dag.recovery.records.RecoveryProtos.DAGInitializedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGInitializedEvent implements HistoryEvent, DAGIDAware { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java index 54c765164f..c3d85e9ac0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.io.OutputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -30,10 +33,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGKillRequestEvent implements HistoryEvent, SummaryEvent { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java index f4caef91f6..486e75abe5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java @@ -20,15 +20,14 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.tez.dag.app.dag.DAGState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.records.TezDAGID; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - public class DAGRecoveredEvent implements HistoryEvent { private final ApplicationAttemptId applicationAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java index 3bb3ed893d..1a182da5c2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.app.dag.DAGState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; @@ -27,10 +30,6 @@ import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.recovery.records.RecoveryProtos.DAGStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGStartedEvent implements HistoryEvent, DAGIDAware { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java index 8ba51ca40c..93eb2da9bb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java @@ -23,6 +23,9 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -38,10 +41,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java index db5fb7a9ea..1c259604b8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java @@ -23,6 +23,9 @@ import javax.annotation.Nullable; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -46,9 +49,6 @@ import org.apache.tez.runtime.api.impl.TezEvent; import com.google.common.collect.Lists; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java index abbcf3386f..89b5d64af6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -29,10 +32,6 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.recovery.records.RecoveryProtos.TaskAttemptStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class TaskAttemptStartedEvent implements HistoryEvent, TaskAttemptIDAware { private TezTaskAttemptID taskAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java index 0debc09188..7dbe623224 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.common.counters.CounterGroup; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.common.counters.TezCounters; @@ -31,10 +34,6 @@ import org.apache.tez.dag.records.TezTaskID; import org.apache.tez.dag.recovery.records.RecoveryProtos.TaskFinishedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java index ad38b0bc16..3bbe723ca5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.api.oldrecords.TaskState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; @@ -27,10 +30,6 @@ import org.apache.tez.dag.records.TezTaskID; import org.apache.tez.dag.recovery.records.RecoveryProtos.TaskStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class TaskStartedEvent implements HistoryEvent, TaskIDAware { private TezTaskID taskID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java index 2b3f1613fd..d8f5d6acdf 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java @@ -22,6 +22,10 @@ import java.io.OutputStream; import java.nio.charset.Charset; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -30,11 +34,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexCommitStartedProto; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class VertexCommitStartedEvent implements HistoryEvent, SummaryEvent { private TezVertexID vertexID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java index 4d5551ac71..7a8a7ff80c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java @@ -23,6 +23,9 @@ import java.util.Map; import java.util.Map.Entry; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.VertexLocationHint; @@ -36,9 +39,6 @@ import org.apache.tez.runtime.api.InputSpecUpdate; import com.google.common.collect.Maps; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexConfigurationDoneEvent implements HistoryEvent, VertexIDAware { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java index 986ec09bb0..af00213517 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java @@ -22,6 +22,9 @@ import java.io.OutputStream; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.app.dag.VertexState; import org.apache.tez.dag.app.dag.impl.ServicePluginInfo; @@ -36,10 +39,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexFinishStateProto; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexFinishedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java index 45741571cf..e7a491d680 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java @@ -22,6 +22,9 @@ import java.io.OutputStream; import java.util.Collection; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -33,9 +36,6 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexGroupCommitFinishedEvent implements HistoryEvent, SummaryEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java index 4a6e346e29..4c69f90597 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java @@ -22,6 +22,9 @@ import java.io.OutputStream; import java.util.Collection; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -33,9 +36,6 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexGroupCommitStartedEvent implements HistoryEvent, SummaryEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java index 28b66663a9..71a124fde5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java @@ -23,6 +23,9 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.InputDescriptor; import org.apache.tez.dag.api.InputInitializerDescriptor; @@ -40,9 +43,6 @@ import org.apache.tez.runtime.api.impl.TezEvent; import com.google.common.collect.Lists; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexInitializedEvent implements HistoryEvent, VertexIDAware { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java index 498bc088d2..b4916235dd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.app.dag.VertexState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; @@ -27,10 +30,6 @@ import org.apache.tez.dag.records.VertexIDAware; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class VertexStartedEvent implements HistoryEvent, VertexIDAware { private TezVertexID vertexID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java index b4aaa9d3c1..29da02d442 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezConfiguration; @@ -45,7 +46,6 @@ import org.apache.tez.dag.records.TezDAGID; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.CodedOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java index a7c7fc4659..a7c9db03a2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java @@ -18,12 +18,11 @@ */ package org.apache.tez.dag.utils; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.recovery.records.RecoveryProtos; -import com.google.protobuf.ByteString; - public final class ProtoUtils { private ProtoUtils() {} diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java index e98d68f644..d03dbdd741 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java @@ -59,6 +59,7 @@ import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -93,7 +94,6 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java index a36f4127a2..81010c6dfd 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java @@ -51,6 +51,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -163,7 +164,6 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java index 4794a90703..38ac00f5f9 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java @@ -69,6 +69,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -237,7 +238,6 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java index 60db6821ea..0c71460451 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java @@ -36,6 +36,8 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; @@ -77,8 +79,6 @@ import org.apache.tez.runtime.api.impl.TezEvent; import com.google.common.collect.Lists; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; import org.junit.jupiter.api.Test; import org.slf4j.Logger; diff --git a/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java b/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java index 09e9a30279..b75c0b9a9c 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java @@ -23,6 +23,7 @@ import java.util.Objects; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.TezUncheckedException; @@ -31,8 +32,6 @@ import org.apache.tez.dag.api.event.VertexStateUpdate; import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager; -import com.google.protobuf.ByteString; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index 9ef694afca..5aa813f61e 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -42,6 +42,10 @@ com.google.guava guava + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 + org.apache.hadoop hadoop-common @@ -188,6 +192,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java index 3d8812d212..cab3f033e6 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java @@ -25,6 +25,7 @@ import java.util.concurrent.Executors; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.Message; import org.apache.tez.common.GuavaShim; import org.apache.tez.service.TezTestServiceProtocolBlockingPB; import org.apache.tez.service.impl.TezTestServiceProtocolClientImpl; @@ -39,7 +40,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.Message; public class TezTestServiceCommunicator extends AbstractService { diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java index 6c1aa6b5d5..2db2b3b82b 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -38,8 +39,6 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto; -import com.google.protobuf.ByteString; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java index 8803e0c787..1f76b8e2e1 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java @@ -28,6 +28,8 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.dag.app.TezTaskCommunicatorImpl; @@ -42,9 +44,6 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto; import org.apache.tez.util.ProtoConverters; -import com.google.protobuf.ByteString; -import com.google.protobuf.ServiceException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java index dd35a51cab..7d49af6e6d 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java @@ -22,17 +22,16 @@ import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.service.TezTestServiceProtocolBlockingPB; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; - public class TezTestServiceProtocolClientImpl implements TezTestServiceProtocolBlockingPB { private final Configuration conf; @@ -78,7 +77,7 @@ public TezTestServiceProtocolBlockingPB getProxy() throws IOException { public TezTestServiceProtocolBlockingPB createProxy() throws IOException { TezTestServiceProtocolBlockingPB p; // TODO Fix security - RPC.setProtocolEngine(conf, TezTestServiceProtocolBlockingPB.class, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, TezTestServiceProtocolBlockingPB.class, ProtobufRpcEngine2.class); p = (TezTestServiceProtocolBlockingPB) RPC .getProxy(TezTestServiceProtocolBlockingPB.class, 0, serverAddr, conf); return p; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java index 051f011531..1515daa2ad 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java @@ -24,10 +24,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.BlockingService; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.dag.api.TezException; import org.apache.tez.service.ContainerRunner; import org.apache.tez.service.TezTestServiceProtocolBlockingPB; @@ -38,9 +41,6 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.BlockingService; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,7 +127,7 @@ InetSocketAddress getBindAddress() { private RPC.Server createServer(Class pbProtocol, InetSocketAddress addr, Configuration conf, int numHandlers, BlockingService blockingService) throws IOException { - RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine2.class); RPC.Server server = new RPC.Builder(conf) .setProtocol(pbProtocol) .setInstance(blockingService) diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml index 74e2fd71a1..f72a1d6b0c 100644 --- a/tez-mapreduce/pom.xml +++ b/tez-mapreduce/pom.xml @@ -132,8 +132,8 @@ commons-collections4 - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 @@ -168,6 +168,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java index 689be23253..0b38276360 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java @@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.JobSubmissionFiles; import org.apache.hadoop.mapreduce.split.JobSplitWriter; import org.apache.hadoop.mapreduce.split.TezGroupedSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -74,7 +75,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java index f2edb05842..1c550d8c2f 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.mapreduce.input.base.MRInputBase; @@ -51,7 +52,6 @@ import com.google.common.base.Function; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java index 829fa47a6b..b48162529e 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java @@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TaskCounter; @@ -70,7 +71,6 @@ import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java index b04ce5f963..6198acdb0c 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java @@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.split.TezGroupedSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.api.DataSourceDescriptor; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.mapreduce.TezTestUtils; @@ -52,11 +53,8 @@ import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent; import org.apache.tez.runtime.api.events.InputDataInformationEvent; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - public class TestMRInputAMSplitGenerator { private static String SPLITS_LENGTHS = "splits.length"; diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java index f945fe4dac..3ec23add49 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java @@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.mapreduce.TezTestUtils; @@ -46,11 +47,8 @@ import org.apache.tez.runtime.api.events.InputDataInformationEvent; import org.apache.tez.runtime.api.events.InputUpdatePayloadEvent; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - public class TestMRInputSplitDistributor { @Test diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java index 62eb0c9f33..00f78551fe 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java @@ -47,20 +47,18 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.split.JobSplit; import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.dag.api.DataSourceDescriptor; import org.apache.tez.dag.api.TaskLocationHint; import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto; import org.apache.tez.runtime.api.events.InputDataInformationEvent; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - public class TestMRInputHelpers { protected static MiniDFSCluster dfsCluster; diff --git a/tez-plugins/tez-aux-services/pom.xml b/tez-plugins/tez-aux-services/pom.xml index 2b51e5da5b..c75e01394d 100644 --- a/tez-plugins/tez-aux-services/pom.xml +++ b/tez-plugins/tez-aux-services/pom.xml @@ -120,11 +120,6 @@ org.slf4j slf4j-api - - com.google.protobuf - protobuf-java - provided - ${leveldbjni.group} leveldbjni-all diff --git a/tez-plugins/tez-protobuf-history-plugin/pom.xml b/tez-plugins/tez-protobuf-history-plugin/pom.xml index 492e909a9b..79a07bfba1 100644 --- a/tez-plugins/tez-protobuf-history-plugin/pom.xml +++ b/tez-plugins/tez-protobuf-history-plugin/pom.xml @@ -42,8 +42,8 @@ hadoop-common - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.junit.jupiter @@ -87,6 +87,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java index 902dba796f..0a7fca6ad8 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java @@ -34,11 +34,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; import org.apache.hadoop.yarn.util.Clock; -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java index dd4b644191..8aa8290e27 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java @@ -25,9 +25,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile.Reader; - -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; public class ProtoMessageReader implements Closeable { private final Path filePath; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java index 2be16654dc..331bec1a45 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java @@ -26,12 +26,11 @@ import java.io.OutputStream; import org.apache.hadoop.io.Writable; - -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; public class ProtoMessageWritable implements Writable { private T message; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java index 75b5e48374..7cc90ee805 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java @@ -27,11 +27,10 @@ import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.SequenceFile.Writer; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; import org.apache.tez.common.StreamHelper; -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; - public class ProtoMessageWriter implements Closeable { private final Path filePath; private final Writer writer; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java index 596a62916c..0dab556cb8 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -70,11 +71,8 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.hadoop.shim.HadoopShim; -import com.google.protobuf.CodedInputStream; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; - public class TestProtoHistoryLoggingService { private static ApplicationId appId = ApplicationId.newInstance(1000l, 1); private static ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java index 53983ac284..192ab412cc 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java @@ -34,6 +34,8 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; @@ -58,14 +60,10 @@ import org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorConfig; import org.apache.tez.tests.MiniTezClusterWithTimeline; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TestATSHistoryV15 { diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml index 53d4595259..b753a0ed53 100644 --- a/tez-runtime-internals/pom.xml +++ b/tez-runtime-internals/pom.xml @@ -79,8 +79,8 @@ hadoop-yarn-common - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 com.google.guava @@ -132,6 +132,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java index 4c25a12847..798370860d 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.runtime.api.events.CompositeDataMovementEvent; import org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent; import org.apache.tez.runtime.api.events.CustomProcessorEvent; @@ -31,8 +32,6 @@ import org.apache.tez.runtime.api.events.InputInitializerEvent; import org.apache.tez.runtime.api.events.VertexManagerEvent; -import com.google.protobuf.ByteString; - public final class ProtoConverters { public static EventProtos.CustomProcessorEventProto convertCustomProcessorEventToProto( diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java index 2ffe11d722..b69218d223 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java @@ -27,6 +27,9 @@ import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.thirdparty.protobuf.AbstractMessage; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.tez.common.ProtoConverters; import org.apache.tez.common.TezConverterUtils; import org.apache.tez.dag.api.TezUncheckedException; @@ -56,10 +59,6 @@ import org.apache.tez.runtime.internals.api.events.SystemEventProtos.TaskAttemptFailedEventProto; import org.apache.tez.runtime.internals.api.events.SystemEventProtos.TaskAttemptKilledEventProto; -import com.google.protobuf.AbstractMessage; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - public class TezEvent implements Writable { private EventType eventType; diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml index cba4c2f4f3..a63e2e64ec 100644 --- a/tez-runtime-library/pom.xml +++ b/tez-runtime-library/pom.xml @@ -54,8 +54,8 @@ tez-common - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.junit.jupiter @@ -146,6 +146,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java index ae6687d005..e352a4dd20 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java @@ -22,14 +22,13 @@ import java.util.HashMap; import java.util.Map.Entry; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.library.vertexmanager.FairShuffleUserPayloads.FairShuffleEdgeManagerConfigPayloadProto; import org.apache.tez.dag.library.vertexmanager.FairShuffleUserPayloads.FairShuffleEdgeManagerDestinationTaskPropProto; import org.apache.tez.dag.library.vertexmanager.FairShuffleUserPayloads.RangeProto; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - /** * Handles edge configuration serialization and de-serialization between diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java index cd71c4b3d2..638e888fd2 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java @@ -22,12 +22,11 @@ import javax.annotation.Nullable; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.dag.api.EdgeManagerPluginContext; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand; import org.apache.tez.dag.api.UserPayload; -import com.google.protobuf.InvalidProtocolBufferException; - /** * Edge manager for fair routing. Each destination task has its * DestinationTaskInputsProperty used to decide how to do event routing diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index 1616b15dd1..ee03832ff5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -32,6 +32,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.EdgeManagerPluginContext; @@ -48,8 +50,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.ShuffleEdgeManagerConfigPayloadProto; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java index 6f5eaa513e..43983687d2 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java @@ -34,6 +34,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; @@ -62,8 +64,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java index 992d47b418..880bc9d15b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java @@ -28,14 +28,14 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.Preconditions; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.UserPayload; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; /** * CartesianProductConfig is used to configure both diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java index 1d8f806385..0a58cad148 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java @@ -22,12 +22,12 @@ import javax.annotation.Nullable; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.dag.api.EdgeManagerPluginContext; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; /** * This EM wrap a real edge manager implementation object. It choose whether it's partitioned or diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java index f20c59d374..b65296717e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.dag.api.EdgeManagerPluginDescriptor; import org.apache.tez.dag.api.EdgeProperty; @@ -42,7 +43,6 @@ import org.apache.tez.runtime.api.events.VertexManagerEvent; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; /** * This VM wrap a real vertex manager implementation object. It choose whether it's partitioned or diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java index c21a4f1aaa..3260c3be18 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Queue; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.VertexManagerPluginContext; @@ -45,7 +46,6 @@ import com.google.common.math.LongMath; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java index b2135b0566..ca571011ba 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java @@ -42,6 +42,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; @@ -68,7 +69,6 @@ import org.apache.tez.util.FastNumberFormat; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index b832e97720..0a7f8c3bb4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -25,6 +25,8 @@ import java.util.zip.Inflater; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezUncheckedException; @@ -44,9 +46,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataProto; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java index b2d0041a49..bde8edb9db 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java @@ -24,6 +24,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.Inflater; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezUncheckedException; @@ -39,9 +41,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.util.StringInterner; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index e644d8c12e..d19c4dbb65 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -52,6 +52,7 @@ import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.CallableWithNdc; import org.apache.tez.common.GuavaShim; import org.apache.tez.common.Preconditions; @@ -85,7 +86,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java index b6e049da0b..cc5ab4d709 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; @@ -59,8 +60,6 @@ import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto; -import com.google.protobuf.ByteString; - import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.roaringbitmap.RoaringBitmap; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java index 2ab8436959..639864d18b 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java @@ -30,17 +30,15 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.api.EdgeManagerPluginContext; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata; import org.apache.tez.dag.api.UserPayload; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - public class TestCartesianProductEdgeManagerPartitioned { private EdgeManagerPluginContext mockContext; private CartesianProductEdgeManagerPartitioned edgeManager; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java index 17f127e558..080dcb75e1 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java @@ -39,6 +39,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.dag.api.EdgeManagerPluginDescriptor; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.VertexManagerPluginContext; @@ -56,8 +58,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java index 1891cf4609..b3a149db9e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java @@ -56,6 +56,7 @@ import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; @@ -77,7 +78,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java index 0eed8c65a3..6b6aee13c5 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezExecutors; import org.apache.tez.common.TezRuntimeFrameworkConfigs; @@ -70,13 +71,10 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.mockito.MockedStatic; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java index fa6abcce24..09d3f1e970 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java @@ -44,6 +44,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezExecutors; @@ -65,13 +66,10 @@ import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; /** diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java index 29704e27fa..4c77623402 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java @@ -57,6 +57,7 @@ import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.io.serializer.WritableSerialization; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.runtime.library.common.InputAttemptIdentifier; @@ -70,13 +71,10 @@ import org.apache.tez.runtime.library.utils.BufferUtils; import org.apache.tez.runtime.library.utils.CodecUtils; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TestIFile { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java index 3441fd079b..72134d4a05 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java @@ -50,6 +50,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; @@ -79,15 +80,12 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; import org.apache.tez.util.StringInterner; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.mockito.ArgumentCaptor; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java index 82fef1a886..e1616759d6 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java @@ -68,6 +68,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; @@ -100,7 +101,6 @@ import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import com.google.protobuf.ByteString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java index 88a8b4aecd..725a087535 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TezCounters; @@ -63,14 +64,11 @@ import org.apache.tez.runtime.library.partitioner.HashPartitioner; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @SuppressWarnings({ "rawtypes", "unchecked" }) diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java index 38989254d9..cc907330cc 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; import org.apache.tez.common.TezSharedExecutor; import org.apache.tez.common.TezUtils; @@ -74,13 +75,10 @@ import org.apache.tez.runtime.library.testutils.KVDataGen; import org.apache.tez.runtime.library.testutils.KVDataGen.KVPair; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.mockito.ArgumentCaptor; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java index 421de863a0..032a911be0 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtilsInternal; @@ -43,13 +44,10 @@ import org.apache.tez.runtime.library.partitioner.HashPartitioner; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - // Tests which don't require parameterization public class TestOrderedPartitionedKVOutput2 { private Configuration conf; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java index a036250619..e57da9efa1 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtilsInternal; @@ -44,13 +45,10 @@ import org.apache.tez.runtime.library.partitioner.HashPartitioner; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - // Tests which don't require parameterization public class TestUnorderedKVOutput2 { private Configuration conf; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java index 7c93b45358..144f6b7924 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.runtime.api.Event; @@ -36,11 +37,8 @@ import org.apache.tez.runtime.api.events.CompositeDataMovementEvent; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; - // Tests which don't require parameterization public class TestUnorderedPartitionedKVOutput2 { diff --git a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java index 61a88dc0a5..fb3133f75e 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java +++ b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java @@ -25,6 +25,8 @@ import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.tez.common.Preconditions; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.dag.api.TezReflectionException; @@ -46,9 +48,6 @@ import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.test.RecoveryServiceWithEventHandlingHook.SimpleShutdownCondition.TIMING; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - import org.slf4j.Logger; import org.slf4j.LoggerFactory;