From 81c97e93d6805944311200fc32c11dd48bee1a0b Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 4 Aug 2026 10:09:31 +0800 Subject: [PATCH] Revert "[refactor](fe) Modularize external metadata cache (#66382)" This reverts commit 60be0bc346ee09ae9d192de9e3c5f4cb77636400. --- fe/fe-connector/README.md | 11 +- .../pom.xml | 35 +- .../doris/connector/cache}/CacheFactory.java | 9 +- .../doris/connector/cache}/CacheSpec.java | 17 +- .../cache}/ConnectorMetadataCache.java | 4 +- .../connector/cache}/ConnectorTableKey.java | 2 +- .../doris/connector/cache/MetaCacheEntry.java | 345 ++++++++++++++++++ .../connector/cache/MetaCacheEntryStats.java | 201 ++++++++++ .../doris/connector/cache}/package-info.java | 11 +- .../doris/connector/cache}/CacheSpecTest.java | 8 +- .../cache}/ConnectorMetadataCacheTest.java | 2 +- .../connector/cache/MetaCacheEntryTest.java} | 39 +- fe/fe-connector/fe-connector-hive/pom.xml | 18 +- .../doris/connector/hive/HiveConnector.java | 4 +- .../connector/hive/HiveConnectorMetadata.java | 4 +- .../connector/hive/HiveConnectorProvider.java | 2 +- .../connector/hive/HiveFileListingCache.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../HiveConnectorPartitionViewCacheTest.java | 4 +- fe/fe-connector/fe-connector-hms/pom.xml | 6 +- .../doris/connector/hms/CachingHmsClient.java | 4 +- fe/fe-connector/fe-connector-iceberg/pom.xml | 11 +- .../iceberg/IcebergCatalogFactory.java | 2 +- .../iceberg/IcebergCommentCache.java | 4 +- .../connector/iceberg/IcebergConnector.java | 4 +- .../iceberg/IcebergConnectorMetadata.java | 4 +- .../iceberg/IcebergConnectorProvider.java | 2 +- .../connector/iceberg/IcebergFormatCache.java | 4 +- .../iceberg/IcebergLatestSnapshotCache.java | 4 +- .../iceberg/IcebergManifestCache.java | 4 +- .../iceberg/IcebergPartitionCache.java | 4 +- .../iceberg/IcebergScanPlanProvider.java | 2 +- .../connector/iceberg/IcebergTableCache.java | 4 +- .../iceberg/IcebergConnectorCacheTest.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../IcebergLatestSnapshotCacheTest.java | 2 +- .../iceberg/IcebergTableCacheTest.java | 2 +- .../fe-connector-maxcompute/pom.xml | 18 +- .../maxcompute/MaxComputePartitionCache.java | 6 +- .../MaxComputePartitionCacheTest.java | 2 +- .../fe-connector-metacache-spi/pom.xml | 52 --- .../metacache/spi/MetaCacheLifecycle.java | 60 --- .../metacache/AbstractMetaCache.java | 280 -------------- .../metacache/MetaCacheRegistry.java | 98 ----- .../metacache/AbstractMetaCacheTest.java | 136 ------- .../metacache/MetaCacheRegistryTest.java | 81 ---- fe/fe-connector/fe-connector-paimon/pom.xml | 17 +- .../connector/paimon/PaimonConnector.java | 5 +- .../paimon/PaimonConnectorMetadata.java | 4 +- .../paimon/PaimonConnectorProvider.java | 2 +- .../paimon/PaimonLatestSnapshotCache.java | 4 +- .../paimon/PaimonConnectorCacheTest.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../paimon/PaimonLatestSnapshotCacheTest.java | 2 +- fe/fe-connector/pom.xml | 3 +- fe/fe-core/pom.xml | 10 - .../doris/datasource/ExternalCatalog.java | 15 +- .../doris/datasource/ExternalDatabase.java | 16 +- .../datasource/ExternalMetaCacheMgr.java | 6 +- .../doris/DorisExternalMetaCache.java | 12 +- .../metacache/AbstractExternalMetaCache.java | 267 +++++++++++++- .../doris/datasource/metacache/CacheSpec.java | 287 +++++++++++++++ .../metacache/CatalogEntryGroup.java | 7 +- .../metacache/ExternalMetaCache.java | 76 +++- .../metacache/ExternalMetaCacheRegistry.java | 84 ++++- .../datasource}/metacache/IdNameIndex.java | 34 +- .../datasource}/metacache/MetaCacheEntry.java | 165 ++------- .../metacache}/MetaCacheEntryDef.java | 30 +- .../MetaCacheEntryInvalidation.java | 32 +- .../metacache}/MetaCacheEntryStats.java | 4 +- .../datasource/metacache/NameCacheValue.java | 5 +- .../tablefunction/MetadataGenerator.java | 2 +- .../doris/catalog/RefreshManagerTest.java | 6 +- .../doris/datasource/ExternalCatalogTest.java | 4 +- .../datasource/ExternalDatabaseTest.java | 6 +- .../doris/DorisExternalMetaCacheTest.java | 2 +- .../AbstractExternalMetaCacheTest.java | 12 +- .../datasource/metacache/CacheSpecTest.java | 169 +++++++++ .../metacache/IdNameIndexTest.java | 2 +- .../metacache/MetaCacheDeadlockTest.java | 10 +- .../metacache/MetaCacheEntryTest.java | 175 +-------- 81 files changed, 1689 insertions(+), 1316 deletions(-) rename fe/fe-connector/{fe-connector-metacache => fe-connector-cache}/pom.xml (59%) rename fe/fe-connector/{fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache => fe-connector-cache/src/main/java/org/apache/doris/connector/cache}/CacheFactory.java (89%) rename fe/fe-connector/{fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi => fe-connector-cache/src/main/java/org/apache/doris/connector/cache}/CacheSpec.java (90%) rename fe/fe-connector/{fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache => fe-connector-cache/src/main/java/org/apache/doris/connector/cache}/ConnectorMetadataCache.java (98%) rename fe/fe-connector/{fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache => fe-connector-cache/src/main/java/org/apache/doris/connector/cache}/ConnectorTableKey.java (98%) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java rename fe/fe-connector/{fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache => fe-connector-cache/src/main/java/org/apache/doris/connector/cache}/package-info.java (64%) rename fe/fe-connector/{fe-connector-metacache-spi/src/test/java/org/apache/doris/connector/metacache/spi => fe-connector-cache/src/test/java/org/apache/doris/connector/cache}/CacheSpecTest.java (96%) rename fe/fe-connector/{fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache => fe-connector-cache/src/test/java/org/apache/doris/connector/cache}/ConnectorMetadataCacheTest.java (99%) rename fe/fe-connector/{fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache/ConnectorMetaCacheEntryCompatibilityTest.java => fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java} (88%) delete mode 100644 fe/fe-connector/fe-connector-metacache-spi/pom.xml delete mode 100644 fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/MetaCacheLifecycle.java delete mode 100644 fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/AbstractMetaCache.java delete mode 100644 fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/MetaCacheRegistry.java delete mode 100644 fe/fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache/AbstractMetaCacheTest.java delete mode 100644 fe/fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector/metacache/MetaCacheRegistryTest.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java rename fe/{fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector => fe-core/src/main/java/org/apache/doris/datasource}/metacache/CatalogEntryGroup.java (88%) rename fe/{fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector => fe-core/src/main/java/org/apache/doris/datasource}/metacache/IdNameIndex.java (86%) rename fe/{fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector => fe-core/src/main/java/org/apache/doris/datasource}/metacache/MetaCacheEntry.java (84%) rename fe/{fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi => fe-core/src/main/java/org/apache/doris/datasource/metacache}/MetaCacheEntryDef.java (88%) rename fe/{fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi => fe-core/src/main/java/org/apache/doris/datasource/metacache}/MetaCacheEntryInvalidation.java (73%) rename fe/{fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi => fe-core/src/main/java/org/apache/doris/datasource/metacache}/MetaCacheEntryStats.java (98%) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java rename fe/{fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector => fe-core/src/test/java/org/apache/doris/datasource}/metacache/IdNameIndexTest.java (99%) rename fe/{fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector => fe-core/src/test/java/org/apache/doris/datasource}/metacache/MetaCacheDeadlockTest.java (91%) rename fe/{fe-connector/fe-connector-metacache/src/test/java/org/apache/doris/connector => fe-core/src/test/java/org/apache/doris/datasource}/metacache/MetaCacheEntryTest.java (91%) diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index b3019c35955951..5a1dcef5e2d883 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -56,8 +56,7 @@ endpoint properties) | Module | Role | |---|---| -| `fe-connector-metacache-spi` | JDK-only cache policy, entry-definition, invalidation, statistics, and lifecycle contracts. It has no fe-core, cache-library, or data-source SDK dependency. | -| `fe-connector-metacache` | Shared Caffeine-backed cache runtime used by fe-core and connector plugins. It owns entry generations, refresh, scoped invalidation, catalog entry groups, and the engine registry; it never depends on fe-core or a data-source SDK. | +| `fe-connector-cache` | Self-contained caching framework used by several connectors. No fe-core dependency; it is bundled into each consuming plugin, so shared third-party libraries stay at the consumers' lowest common version (see the version notes in consumer poms). | | `fe-connector-hms-hive-shade` | Slim, relocated HMS metastore-client closure for connectors that speak HMS thrift. The pom comments say exactly what relocates where and why. | | `fe-connector-paimon-hive-shade` | Paimon-private relocated HMS-thrift closure; same idea, different owner. | @@ -184,10 +183,10 @@ metastore/shade/cache). For a write path, the richest example is 6. **Property ownership.** Metadata-connection properties are parsed in your connector (or the metastore layer). Storage properties belong to `fe-filesystem`. Do not add parsing to fe-core — rule 2 above. -7. **Caching.** Describe reusable entries with `fe-connector-metacache-spi` - and run them through `fe-connector-metacache` (example: - `PaimonLatestSnapshotCache`). Keep shared third-party versions aligned with - the other consumers (see the version notes in `fe-connector-paimon/pom.xml`). Respect the +7. **Caching.** Reuse `fe-connector-cache` (example: + `PaimonLatestSnapshotCache`). Bundle the caching library into your plugin + zip and keep shared third-party versions aligned with the other consumers + (see the version notes in `fe-connector-paimon/pom.xml`). Respect the authorization invariant in `AGENTS.md`: a cross-query cache must never serve metadata that would bypass per-user, load-time authorization. 8. **Shading.** If your client stack drags a conflicting closure (hive/thrift diff --git a/fe/fe-connector/fe-connector-metacache/pom.xml b/fe/fe-connector/fe-connector-cache/pom.xml similarity index 59% rename from fe/fe-connector/fe-connector-metacache/pom.xml rename to fe/fe-connector/fe-connector-cache/pom.xml index 33ced432a3a733..f9ba332609f91b 100644 --- a/fe/fe-connector/fe-connector-metacache/pom.xml +++ b/fe/fe-connector/fe-connector-cache/pom.xml @@ -29,21 +29,24 @@ under the License. ../pom.xml - fe-connector-metacache + fe-connector-cache jar - Doris FE Connector MetaCache Runtime + Doris FE Connector Cache Framework - Shared external metadata cache runtime used by fe-core and connector plugins. - Contains the Caffeine-backed entry implementation, catalog entry grouping and reusable - connector cache helpers. It depends on fe-connector-metacache-spi and never depends on fe-core. + Connector-side meta-cache framework (CacheSpec + MetaCacheEntry + CacheFactory + MetaCacheEntryStats), + an INDEPENDENT copy of fe-core's `org.apache.doris.datasource.metacache` framework re-homed under the + `org.apache.doris.connector.*` prefix so the connector plugins can reuse it (they cannot import fe-core). + fe-core keeps its own copy untouched; the two live side-by-side until every connector has migrated, then + the fe-core copy is retired. fe-core does NOT depend on this module. + + This module is bundled into each connector plugin zip (child-first), so it uses the plugin's own bundled + Caffeine at runtime; Caffeine is therefore `provided` here (compiled against, never packaged by this + module). The framework's public API (MetaCacheEntry) is Caffeine-free, and fe-core and the connectors + never share a cache object across the classloader boundary, so no Caffeine type crosses and there is no + split-brain. Two knobs fe-core reads from static Config are constructor-injected here. - - ${project.groupId} - fe-connector-metacache-spi - ${project.version} - com.github.ben-manes.caffeine caffeine @@ -55,19 +58,9 @@ under the License. junit-jupiter test - - junit - junit - test - - - com.google.guava - guava - test - - doris-fe-connector-metacache + doris-fe-connector-cache diff --git a/fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/CacheFactory.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java similarity index 89% rename from fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/CacheFactory.java rename to fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java index 5c809753bc821b..03e4126e9911be 100644 --- a/fe/fe-connector/fe-connector-metacache/src/main/java/org/apache/doris/connector/metacache/CacheFactory.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.connector.metacache; +package org.apache.doris.connector.cache; import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; @@ -32,8 +32,11 @@ /** * Factory to create Caffeine cache. * - *

This type is internal to the shared MetaCache runtime. Its public methods return Caffeine - * types; callers outside this module use {@link MetaCacheEntry} instead. + *

Connector-side copy of fe-core {@code org.apache.doris.common.CacheFactory} (independent-copy meta-cache + * migration): connector plugins cannot import fe-core, so the framework is duplicated under + * {@code org.apache.doris.connector.cache}. This type is framework-internal — its public methods RETURN + * Caffeine types, which must never cross to connector (child-first) code; connectors only touch the + * Caffeine-free {@link MetaCacheEntry} API. Keep behaviourally in sync with the fe-core original. * *

This class is used to create Caffeine cache with specified parameters. * It is used to create both sync and async cache. diff --git a/fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java similarity index 90% rename from fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/CacheSpec.java rename to fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java index 0f572ccf7155ca..0524b31d402f48 100644 --- a/fe/fe-connector/fe-connector-metacache-spi/src/main/java/org/apache/doris/connector/metacache/spi/CacheSpec.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.connector.metacache.spi; +package org.apache.doris.connector.cache; import java.util.HashMap; import java.util.Map; @@ -25,9 +25,18 @@ /** * Common cache specification for external metadata caches. * - *

The type is part of the connector-facing MetaCache SPI and therefore depends only on JDK types. - * Property validation reports {@link IllegalArgumentException}; fe-core adapters may translate that - * exception at their own API boundary. + *

Connector-side copy of the meta-cache property model (independent-copy meta-cache migration). fe-core is + * NOT changed: it keeps its own {@code org.apache.doris.datasource.metacache.CacheSpec}; this is a separate + * class under {@code org.apache.doris.connector.*} used only by the connector plugins. Although that prefix is + * parent-first, fe-core does not depend on this module, so the class resolves parent → miss → CHILD and is + * child-loaded per plugin — fe-core and the plugins do NOT share one {@code Class} identity. It carries no + * third-party dependency (JDK only) and never crosses the fe-core↔connector boundary as an object (only its + * {@code IllegalArgumentException}, a JDK type, crosses), so it is safe on both classpaths. + * + *

The {@code check*Property} validators throw {@link IllegalArgumentException} (fe-core's + * {@code PluginDrivenExternalCatalog.checkProperties} re-wraps it into a {@code DdlException} verbatim; the + * legacy fe-core catalogs that still call these validators declare {@code throws DdlException} but no longer + * need it). The user-facing message text is identical to the legacy one ({@code "... is wrong, value is ..."}). * *

Semantics: *