HDDS-14949 — Migrate gRPC transport to the Ratis-shaded gRPC/Netty/Protobuf stack

hadoop-hdds/client/pom.xml

@@ -52,6 +52,10 @@
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-config</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-datanode-grpc-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-erasurecode</artifactId>
@@ -101,6 +105,21 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUsedUndeclaredDependencies combine.children="append">
+                <!-- protobuf-java is used via hdds-interface-client transitively -->
+                <ignoredUsedUndeclaredDependency>com.google.protobuf:protobuf-java</ignoredUsedUndeclaredDependency>
+              </ignoredUsedUndeclaredDependencies>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>com.github.spotbugs</groupId>
         <artifactId>spotbugs-maven-plugin</artifactId>

hadoop-hdds/common/pom.xml

@@ -112,6 +112,10 @@
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-config</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-datanode-grpc-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-interface-client</artifactId>
@@ -248,6 +252,7 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
+          <useIncrementalCompilation>false</useIncrementalCompilation>
           <annotationProcessorPaths>
             <path>
               <groupId>org.apache.ozone</groupId>
@@ -263,6 +268,26 @@
           </compilerArgs>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUnusedDeclaredDependencies combine.children="append">
+                <!--
+                  hdds-config is used only as an annotationProcessorPath (ConfigFileGenerator).
+                  Its annotation types are @Retention(SOURCE) so they don't appear in compiled
+                  bytecode; the dependency analyzer therefore flags it as unused.
+                -->
+                <ignoredUnusedDeclaredDependency>org.apache.ozone:hdds-config</ignoredUnusedDeclaredDependency>
+                <ignoredUnusedDeclaredDependency>org.apache.ratis:ratis-thirdparty-misc</ignoredUnusedDeclaredDependency>
+              </ignoredUnusedDeclaredDependencies>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -34,8 +34,9 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.net.HostAndPort;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.ServiceException;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.ratis.thirdparty.com.google.protobuf.ServiceException;
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
 import java.io.File;
@@ -86,7 +87,6 @@
 import org.apache.hadoop.ozone.ha.ConfUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.SecretManager;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
 import org.apache.ratis.util.SizeInBytes;
 import org.slf4j.Logger;

hadoop-hdds/container-service/pom.xml

@@ -336,6 +336,24 @@
               <overWriteSnapshots>true</overWriteSnapshots>
             </configuration>
           </execution>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUnusedDeclaredDependencies combine.children="append">
+                <!--
+                  hadoop-auth is used by Hadoop delegation tokens at runtime; its types
+                  are accessed through hadoop-common's service-loader mechanism and not
+                  directly imported in compiled bytecode.
+                -->
+                <ignoredUnusedDeclaredDependency>org.apache.hadoop:hadoop-auth</ignoredUnusedDeclaredDependency>
+                <!--
+                  ratis-netty is a Ratis transport module loaded at runtime via service
+                  discovery. Its classes are not referenced directly in compiled bytecode.
+                -->
+                <ignoredUnusedDeclaredDependency>org.apache.ratis:ratis-netty</ignoredUnusedDeclaredDependency>
+              </ignoredUnusedDeclaredDependencies>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
     </plugins>

hadoop-hdds/datanode-grpc-client/pom.xml

@@ -0,0 +1,164 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.ozone</groupId>
+    <artifactId>hdds</artifactId>
+    <version>2.2.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>hdds-datanode-grpc-client</artifactId>
+  <version>2.2.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+  <name>Apache Ozone HDDS DataNode gRPC Client</name>
+  <description>DataNode XCeiver and IntraDatanode gRPC stubs generated from DatanodeClientProtocol.proto.
+    Package names in generated sources are rewritten by maven-antrun-plugin (Ant replace task, pure Java)
+    from vanilla com.google.protobuf / io.grpc / com.google.common to their ratis-thirdparty counterparts
+    before compilation, so target/classes/ already contains the shaded namespace and downstream modules
+    in the same Maven reactor can compile against the correct types.</description>
+
+  <properties>
+    <!-- used in temp dir names; must be filesystem/protoc-safe (no ':') -->
+    <maven.build.timestamp.format>yyyyMMddHHmmss</maven.build.timestamp.format>
+    <!-- no testable code in this module -->
+    <maven.test.skip>true</maven.test.skip>
+    <mdep.analyze.skip>true</mdep.analyze.skip>
+    <spotbugs.skip>true</spotbugs.skip>
+  </properties>
+
+  <dependencies>
+    <!--
+      Shaded gRPC / Netty / protobuf runtime. The generated sources are rewritten to reference
+      org.apache.ratis.thirdparty.* directly, so ratis-thirdparty-misc must be on the compile
+      classpath and is exposed transitively to all consumers of this module.
+    -->
+    <dependency>
+      <groupId>org.apache.ratis</groupId>
+      <artifactId>ratis-thirdparty-misc</artifactId>
+    </dependency>
+    <dependency>
+      <!-- for @Generated annotation in generated sources (Java 11+) -->
+      <groupId>javax.annotation</groupId>
+      <artifactId>javax.annotation-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>com.gradle</groupId>
+          <artifactId>develocity-maven-extension</artifactId>
+          <configuration>
+            <develocity>
+              <plugins>
+                <plugin>
+                  <artifactId>maven-compiler-plugin</artifactId>
+                  <outputs>
+                    <notCacheableBecause>compiles generated + rewritten sources</notCacheableBecause>
+                  </outputs>
+                </plugin>
+                <plugin>
+                  <artifactId>maven-antrun-plugin</artifactId>
+                  <outputs>
+                    <notCacheableBecause>rewrites generated source files</notCacheableBecause>
+                  </outputs>
+                </plugin>
+                <plugin>
+                  <artifactId>protobuf-maven-plugin</artifactId>
+                  <outputs>
+                    <notCacheableBecause>generates sources</notCacheableBecause>
+                  </outputs>
+                </plugin>
+              </plugins>
+            </develocity>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>com.salesforce.servicelibs</groupId>
+        <artifactId>proto-backwards-compatibility</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <proc>none</proc>
+          <useIncrementalCompilation>false</useIncrementalCompilation>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>compile-proto-datanode</id>
+            <goals>
+              <goal>compile</goal>
+              <goal>compile-custom</goal>
+            </goals>
+            <phase>generate-sources</phase>
+            <configuration>
+              <!--
+                Use the same protoc/grpc-java versions as ratis-thirdparty so that the generated
+                API surface is compatible with the shaded runtime in ratis-thirdparty-misc.
+                Output directly to proto-java-ratis so that both Maven and the IDE/LSP compile the
+                same ratis-shaded source tree, eliminating any race between Maven's javac and the
+                LSP writing vanilla class files to target/classes/.
+              -->
+              <protocArtifact>com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+              <outputDirectory>${project.build.directory}/generated-sources/proto-java-ratis</outputDirectory>
+              <temporaryProtoFileDirectory>${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/${maven.build.timestamp}</temporaryProtoFileDirectory>
+              <clearOutputDirectory>false</clearOutputDirectory>
+              <pluginId>grpc-java</pluginId>
+              <pluginArtifact>io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <!--
+              Rewrite vanilla package names produced by protoc/grpc-java in-place inside
+              proto-java-ratis/ (the protobuf-maven-plugin outputDirectory).  Because the
+              IDE/LSP also uses proto-java-ratis/ as its Java source root, it will compile
+              the same shaded sources and write shaded class files to target/classes/, which
+              means both Maven javac and the LSP produce identical bytecode – no more race
+              where the LSP overwrites Maven's correctly-shaded class files.
+            -->
+            <id>rewrite-generated-sources-to-ratis-ns</id>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <phase>process-sources</phase>
+            <configuration>
+              <target>
+                <replace dir="${project.build.directory}/generated-sources/proto-java-ratis" token="com.google.common" value="org.apache.ratis.thirdparty.com.google.common" />
+                <replace dir="${project.build.directory}/generated-sources/proto-java-ratis" token="com.google.protobuf" value="org.apache.ratis.thirdparty.com.google.protobuf" />
+                <replace dir="${project.build.directory}/generated-sources/proto-java-ratis" token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc" />
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>