From 1d0f89a0cea3e985f00126c96bc818b39df2722b Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 3 Sep 2026 16:09:10 +0000 Subject: [PATCH 1/2] fix: stop publishing a bigtable-beam-import pom with no dependencies bigtable-beam-import attaches its shaded jar under a classifier, so the main artifact is the thin, unshaded jar and its pom must keep every compile dependency. maven-shade-plugin still strips them from the generated dependency-reduced pom in that configuration (MSHADE-419), which is not fixed in 3.6.0 despite what the comment in the root pom claimed. The bump to 3.6.0 in #4626 therefore published 2.20.0 and 2.20.1 poms declaring zero compile dependencies. Consumers resolving the artifact from Maven Central get no transitive deps at all. GoogleCloudPlatform/DataflowTemplates hits this staging the Cloud_Bigtable_to_GCS_SequenceFile template, which fails with ClassNotFoundException: org.apache.hadoop.hbase.io.ImmutableBytesWritable. Nothing caught it in-repo because the reactor resolves sibling modules from source poms and never reads dependency-reduced-pom.xml. - set createDependencyReducedPom=false explicitly on the module rather than relying on a plugin default that changed between 3.2.4 and 3.6.0 - add a verify-published-pom-deps goal to bigtable-build-helper that inspects the pom that would actually be deployed and fails the build if a required compile dependency is missing, wired into the module's verify phase; covered by integration tests for both the healthy and the MSHADE-419-stripped case - correct the root pom comment and drop the renovate rule that pinned maven-shade-plugin away from 3.3.0, which was both an incomplete description of the bug and moot now that the flag is explicit Change-Id: I2d269c60175e10da7e5b361f410dd06f7861ba48 --- .../bigtable-beam-import/pom.xml | 39 +++++ .../it/verify-published-pom-deps-ok/pom.xml | 91 ++++++++++++ .../invoker.properties | 1 + .../pom.xml | 93 ++++++++++++ .../verify.bsh | 36 +++++ .../plugins/VerifyPublishedPomDepsMojo.java | 133 ++++++++++++++++++ pom.xml | 10 +- renovate.json5 | 7 - 8 files changed, 400 insertions(+), 10 deletions(-) create mode 100644 bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-ok/pom.xml create mode 100644 bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/invoker.properties create mode 100644 bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/pom.xml create mode 100644 bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/verify.bsh create mode 100644 bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java diff --git a/bigtable-dataflow-parent/bigtable-beam-import/pom.xml b/bigtable-dataflow-parent/bigtable-beam-import/pom.xml index 226650b487..5de323222d 100644 --- a/bigtable-dataflow-parent/bigtable-beam-import/pom.xml +++ b/bigtable-dataflow-parent/bigtable-beam-import/pom.xml @@ -320,6 +320,14 @@ limitations under the License. true + + false @@ -378,6 +386,37 @@ limitations under the License. true + + + com.google.cloud.bigtable.test + bigtable-build-helper + 2.20.2-SNAPSHOT + + + verify-published-pom-deps + verify + + verify-published-pom-deps + + + + + com.google.cloud.bigtable:bigtable-hbase-beam + org.apache.beam:beam-sdks-java-core + org.apache.beam:beam-sdks-java-io-hadoop-format + org.apache.beam:beam-runners-google-cloud-dataflow-java + org.apache.hadoop:hadoop-client-api + org.apache.hadoop:hadoop-client-runtime + org.apache.hbase:hbase-shaded-mapreduce + com.google.cloud.bigdataoss:gcs-connector + + + + + diff --git a/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-ok/pom.xml b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-ok/pom.xml new file mode 100644 index 0000000000..f162d8eb7f --- /dev/null +++ b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-ok/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + + com.google.guava + verify-published-pom-deps-ok + 1.0-SNAPSHOT + + Test project where the attached shaded jar leaves the published pom's compile deps intact + + + UTF-8 + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + true + false + + + *:* + + META-INF/*.MF + + + + + + + + + + com.google.cloud.bigtable.test + bigtable-build-helper + 2.20.2-SNAPSHOT + + + test + + verify-published-pom-deps + + + + com.google.guava:guava + + + + + + + + + + + + ${project.groupId} + guava + 30.1-android + + + diff --git a/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/invoker.properties b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/invoker.properties new file mode 100644 index 0000000000..c21e972fc6 --- /dev/null +++ b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/invoker.properties @@ -0,0 +1 @@ +invoker.buildResult = failure diff --git a/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/pom.xml b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/pom.xml new file mode 100644 index 0000000000..eebc4cce40 --- /dev/null +++ b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/pom.xml @@ -0,0 +1,93 @@ + + + + 4.0.0 + + com.google.cloud.bigtable.hbase.enforcer.it + verify-published-pom-deps-stripped + 1.0-SNAPSHOT + + + Reproduces MSHADE-419: the shaded jar is attached under a classifier, so the main artifact is + the thin unshaded jar, yet the generated dependency-reduced pom still strips its compile deps. + The check must fail rather than let such a pom be published. + + + + UTF-8 + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + true + true + + + *:* + + META-INF/*.MF + + + + + + + + + + com.google.cloud.bigtable.test + bigtable-build-helper + 2.20.2-SNAPSHOT + + + test + + verify-published-pom-deps + + + + com.google.guava:guava + + + + + + + + + + + + com.google.guava + guava + 30.1-android + + + diff --git a/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/verify.bsh b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/verify.bsh new file mode 100644 index 0000000000..96d9eafc74 --- /dev/null +++ b/bigtable-test/bigtable-build-helper/src/it/verify-published-pom-deps-stripped/verify.bsh @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.nio.file.Files; + +File buildLog = new File(basedir, "build.log"); +String content = new String(Files.readAllBytes(buildLog.toPath())); + +boolean success = true; + +assertContains(s) { + if (!content.contains(s)) { + print( "FAILED: Expected build log to contain: " + s); + success = false; + } +} + +assertContains("[ERROR] com.google.guava:guava is missing from the pom that will be published"); +assertContains("createDependencyReducedPom"); + +if (!success) { + throw new IllegalStateException("Build output was missing expected details, " + + "see target/it/verify-published-pom-deps-stripped/build.log for more details"); +} diff --git a/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java b/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java new file mode 100644 index 0000000000..b4b4adc0cf --- /dev/null +++ b/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java @@ -0,0 +1,133 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.test.plugins; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import org.apache.maven.model.Dependency; +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Verifies that the pom Maven is about to deploy still declares the dependencies that consumers + * need to resolve. + * + *

This exists because maven-shade-plugin can silently swap the project's pom for a generated + * dependency-reduced one. That is correct when the shaded jar *is* the main artifact, but wrong + * when {@code shadedArtifactAttached} is set: there the main artifact is the thin, unshaded jar, + * and stripping its dependencies leaves downstream consumers with nothing to resolve. MSHADE-419 + * does precisely that in shade >= 3.3.0. + * + *

Nothing inside the reactor can catch this, because Maven resolves sibling modules from their + * source poms and never from the generated one, so the damage is only visible after deploy. Hence + * this check reads {@link MavenProject#getFile()} — whichever pom is actually slated for deploy at + * {@code verify} time — rather than the source pom on disk. + */ +@Mojo(name = "verify-published-pom-deps", defaultPhase = LifecyclePhase.VERIFY) +public class VerifyPublishedPomDepsMojo extends AbstractMojo { + private static final Logger LOGGER = LoggerFactory.getLogger(VerifyPublishedPomDepsMojo.class); + + @Parameter(defaultValue = "${project}", readonly = true, required = true) + private MavenProject project; + + /** + * Coordinates, as {@code groupId:artifactId}, that must be declared in the published pom with a + * scope that propagates to consumers (compile or runtime). + */ + @Parameter(required = true) + private List requiredDependencies; + + @Override + public void execute() throws MojoExecutionException, MojoFailureException { + File publishedPom = project.getFile(); + Set declared = readPropagatingDependencies(publishedPom); + + List missing = new ArrayList<>(); + for (String required : requiredDependencies) { + if (!declared.contains(required)) { + missing.add(required); + } + } + + if (missing.isEmpty()) { + return; + } + + Collections.sort(missing); + for (String coordinate : missing) { + LOGGER.error( + "{} is missing from the pom that will be published ({}); consumers resolving this" + + " artifact from a repository would not get it", + coordinate, + publishedPom); + } + LOGGER.error( + "If {} is a generated dependency-reduced pom, set" + + " false on maven-shade-plugin" + + " (see MSHADE-419).", + publishedPom.getName()); + + throw new MojoFailureException("Published pom is missing required dependencies"); + } + + /** Reads the raw pom and returns the {@code groupId:artifactId} of its compile/runtime deps. */ + private Set readPropagatingDependencies(File pom) throws MojoFailureException { + Model model; + try (FileInputStream fin = new FileInputStream(pom)) { + model = new MavenXpp3Reader().read(fin); + } catch (XmlPullParserException | IOException e) { + throw new MojoFailureException("Failed to read " + pom, e); + } + + Set declared = new LinkedHashSet<>(); + for (Dependency dependency : model.getDependencies()) { + String scope = dependency.getScope(); + // An absent scope means compile. + if (scope != null && !"compile".equals(scope) && !"runtime".equals(scope)) { + continue; + } + declared.add(resolveGroupId(dependency.getGroupId()) + ":" + dependency.getArtifactId()); + } + return declared; + } + + /** + * Maven deploys the raw pom, so a dependency on a sibling module is still written as {@code + * ${project.groupId}} rather than the resolved value. + */ + private String resolveGroupId(String groupId) { + if ("${project.groupId}".equals(groupId) || "${pom.groupId}".equals(groupId)) { + return project.getGroupId(); + } + return groupId; + } +} diff --git a/pom.xml b/pom.xml index dffa8f1860..ae0875def0 100644 --- a/pom.xml +++ b/pom.xml @@ -193,9 +193,13 @@ limitations under the License. maven-shade-plugin + older ASM fails on those with "Unsupported class file major version 61". + NOTE: MSHADE-419 (reduced pom dropping compile deps when shadedArtifactAttached + is set) is NOT fixed in 3.6.0. An earlier version of this comment claimed it was, + and 2.20.0/2.20.1 shipped bigtable-beam-import poms with zero compile deps as a + result. Modules in that configuration must set + false explicitly; the + verify-published-pom-deps goal guards against it regressing. --> 3.6.0 diff --git a/renovate.json5 b/renovate.json5 index a0e207c37e..e7ccabacc5 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -81,13 +81,6 @@ "packagePatterns": ["^error_prone_annotations"], "enabled": false }, - { - "packagePatterns": [ - "^org.apache.maven.plugins:maven-shade-plugin" - ], - // Exclude version 3.3.0 due to https://issues.apache.org/jira/projects/MSHADE/issues/MSHADE-419 - "allowedVersions": "(,3.3.0),(3.3.0,)" - }, { "packagePatterns": [ "^com.google.cloud:google-cloud-bigtable", From 23d3938ae8e6c4ed3007f0e7fcc1d17a73bb14bc Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 3 Sep 2026 16:20:05 +0000 Subject: [PATCH 2/2] fix: guard against a missing or half-declared pom in verify-published-pom-deps Change-Id: Ice5b8a44978b4833bc48c5fe037a263bbce68b77 --- .../test/plugins/VerifyPublishedPomDepsMojo.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java b/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java index b4b4adc0cf..43254a4c65 100644 --- a/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java +++ b/bigtable-test/bigtable-build-helper/src/main/java/com/google/cloud/bigtable/test/plugins/VerifyPublishedPomDepsMojo.java @@ -69,6 +69,12 @@ public class VerifyPublishedPomDepsMojo extends AbstractMojo { @Override public void execute() throws MojoExecutionException, MojoFailureException { File publishedPom = project.getFile(); + if (publishedPom == null || !publishedPom.isFile()) { + // Whatever pom is slated for deploy has to be readable; if it isn't, fail loudly rather + // than let a build that publishes an unverified pom look like it passed the check. + throw new MojoFailureException( + "There is no pom to verify, project.getFile() is " + publishedPom); + } Set declared = readPropagatingDependencies(publishedPom); List missing = new ArrayList<>(); @@ -115,7 +121,14 @@ private Set readPropagatingDependencies(File pom) throws MojoFailureExce if (scope != null && !"compile".equals(scope) && !"runtime".equals(scope)) { continue; } - declared.add(resolveGroupId(dependency.getGroupId()) + ":" + dependency.getArtifactId()); + String groupId = dependency.getGroupId(); + String artifactId = dependency.getArtifactId(); + // A half-declared coordinate cannot satisfy a requirement anyway, so drop it and let the + // dependency it was meant to be get reported as missing. + if (groupId == null || artifactId == null) { + continue; + } + declared.add(resolveGroupId(groupId) + ":" + artifactId); } return declared; }