From f019a75137fa6d63c75f123e20cd8b4786c01006 Mon Sep 17 00:00:00 2001 From: SolrBot Date: Tue, 1 Sep 2026 01:02:56 +0000 Subject: [PATCH 01/14] Update dependency org.apache.tika:tika-core to v4 --- gradle/libs.versions.toml | 2 +- solr/licenses/commonmark-0.30.0.jar.sha1 | 1 + ...mark-ext-gfm-strikethrough-0.30.0.jar.sha1 | 1 + .../commonmark-ext-gfm-tables-0.30.0.jar.sha1 | 1 + solr/licenses/tika-core-3.3.1.jar.sha1 | 1 - solr/licenses/tika-core-4.0.0.jar.sha1 | 1 + solr/modules/extraction/gradle.lockfile | 5 ++++- solr/ui/gradle.lockfile | 20 +++++++++---------- 8 files changed, 19 insertions(+), 13 deletions(-) create mode 100644 solr/licenses/commonmark-0.30.0.jar.sha1 create mode 100644 solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 create mode 100644 solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 delete mode 100644 solr/licenses/tika-core-3.3.1.jar.sha1 create mode 100644 solr/licenses/tika-core-4.0.0.jar.sha1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 607ed54ecfe1..abdb17a97a5d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -42,7 +42,7 @@ apache-log4j = "2.26.1" apache-lucene = "10.4.0" apache-opennlp = "2.5.10" apache-rat = "0.18" -apache-tika = "3.3.1" +apache-tika = "4.0.0" apache-tomcat = "6.0.53" apache-zookeeper = "3.9.5" # @keep for version alignment diff --git a/solr/licenses/commonmark-0.30.0.jar.sha1 b/solr/licenses/commonmark-0.30.0.jar.sha1 new file mode 100644 index 000000000000..ef17e4e5f72b --- /dev/null +++ b/solr/licenses/commonmark-0.30.0.jar.sha1 @@ -0,0 +1 @@ +fc611136b3191a08d2b84c5d5a26116e3531afee diff --git a/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 b/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 new file mode 100644 index 000000000000..79f99ad6fa50 --- /dev/null +++ b/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 @@ -0,0 +1 @@ +e9addacba704ffce26d6b8bcd1c50e235ec2c991 diff --git a/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 b/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 new file mode 100644 index 000000000000..8f580d0ef23c --- /dev/null +++ b/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 @@ -0,0 +1 @@ +84560a4af5afe4808543bd9c5587a660ce148b14 diff --git a/solr/licenses/tika-core-3.3.1.jar.sha1 b/solr/licenses/tika-core-3.3.1.jar.sha1 deleted file mode 100644 index 04665fed0998..000000000000 --- a/solr/licenses/tika-core-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -075c89f5a662324e7186da5e315ea9966acb7e8a diff --git a/solr/licenses/tika-core-4.0.0.jar.sha1 b/solr/licenses/tika-core-4.0.0.jar.sha1 new file mode 100644 index 000000000000..bab6900e1458 --- /dev/null +++ b/solr/licenses/tika-core-4.0.0.jar.sha1 @@ -0,0 +1 @@ +f370d8619de9e19329b2cc117943c83468c2ff9f diff --git a/solr/modules/extraction/gradle.lockfile b/solr/modules/extraction/gradle.lockfile index 461a77d6a907..bff47aa8ff7a 100644 --- a/solr/modules/extraction/gradle.lockfile +++ b/solr/modules/extraction/gradle.lockfile @@ -118,11 +118,14 @@ org.apache.lucene:lucene-spatial-extras:10.4.0=jarValidation,runtimeClasspath,ru org.apache.lucene:lucene-spatial3d:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.lucene:lucene-suggest:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.lucene:lucene-test-framework:10.4.0=jarValidation,testCompileClasspath,testRuntimeClasspath -org.apache.tika:tika-core:3.3.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.apache.tika:tika-core:4.0.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.zookeeper:zookeeper-jute:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.zookeeper:zookeeper:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspath org.codehaus.woodstox:stax2-api:4.3.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.commonmark:commonmark-ext-gfm-tables:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.commonmark:commonmark:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.compression:jetty-compression-common:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.compression:jetty-compression-gzip:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.ee10:jetty-ee10-servlet:12.1.10=jarValidation,testRuntimeClasspath diff --git a/solr/ui/gradle.lockfile b/solr/ui/gradle.lockfile index 2a0144af5101..140406144c3c 100644 --- a/solr/ui/gradle.lockfile +++ b/solr/ui/gradle.lockfile @@ -343,26 +343,26 @@ org.jetbrains.compose.ui:ui-wasm-js:1.11.1=wasmJsCompileClasspath,wasmJsNpmAggre org.jetbrains.compose.ui:ui:1.10.0=composeHotReloadDevTools org.jetbrains.compose.ui:ui:1.11.1=allDevSourceSetsCompileDependenciesMetadata,allSourceSetsCompileDependenciesMetadata,allTestSourceSetsCompileDependenciesMetadata,commonMainResolvableDependenciesMetadata,commonTestResolvableDependenciesMetadata,composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,jvmCompileClasspath,jvmDevCompileClasspath,jvmDevResolvableDependenciesMetadata,jvmDevRuntimeClasspath,jvmMainCompileClasspath,jvmMainResolvableDependenciesMetadata,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath,metadataCommonMainCompileClasspath,metadataCompileClasspath,wasmJsCompileClasspath,wasmJsMainResolvableDependenciesMetadata,wasmJsNpmAggregated,wasmJsRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestResolvableDependenciesMetadata,wasmJsTestRuntimeClasspath,webMainResolvableDependenciesMetadata,webTestResolvableDependenciesMetadata org.jetbrains.kotlin:kotlin-build-tools-api:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-build-tools-api:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-build-tools-api:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-build-tools-compat:2.4.0=kotlinAbiValidationCompatClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath org.jetbrains.kotlin:kotlin-build-tools-cri-impl:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-build-tools-cri-impl:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-build-tools-cri-impl:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-build-tools-impl:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-build-tools-impl:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-build-tools-impl:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-compiler-embeddable:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-compiler-embeddable:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-compiler-embeddable:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-compiler-runner:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-compiler-runner:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-compiler-runner:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-compose-compiler-plugin-embeddable:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest org.jetbrains.kotlin:kotlin-daemon-client:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-daemon-client:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-daemon-client:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-daemon-embeddable:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-daemon-embeddable:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-daemon-embeddable:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-klib-commonizer-embeddable:2.4.0=kotlinKlibCommonizerClasspath org.jetbrains.kotlin:kotlin-reflect:1.6.10=kotlinAbiValidationCompatClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath org.jetbrains.kotlin:kotlin-reflect:2.3.21=allDevSourceSetsCompileDependenciesMetadata,allSourceSetsCompileDependenciesMetadata,allTestSourceSetsCompileDependenciesMetadata,composeHotReloadDevDesktopDevRuntimeClasspath,composeHotReloadDevDesktopRuntimeClasspath,composeHotReloadDevDesktopTestRuntimeClasspath,composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,desktopCompileClasspath,desktopDevCompileClasspath,desktopDevResolvableDependenciesMetadata,desktopDevRuntimeClasspath,desktopMainCompileClasspath,desktopMainResolvableDependenciesMetadata,desktopMainRuntimeClasspath,desktopRuntimeClasspath,desktopTestCompileClasspath,desktopTestResolvableDependenciesMetadata,desktopTestRuntimeClasspath,jvmCompileClasspath,jvmDevCompileClasspath,jvmDevResolvableDependenciesMetadata,jvmDevRuntimeClasspath,jvmMainCompileClasspath,jvmMainResolvableDependenciesMetadata,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath org.jetbrains.kotlin:kotlin-script-runtime:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-script-runtime:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-script-runtime:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-scripting-common:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest @@ -375,12 +375,12 @@ org.jetbrains.kotlin:kotlin-stdlib-wasm-js:2.4.0=wasmJsCompileClasspath,wasmJsNp org.jetbrains.kotlin:kotlin-stdlib:2.1.21=composeHotReloadRuntime org.jetbrains.kotlin:kotlin-stdlib:2.2.21=composeHotReloadDevTools org.jetbrains.kotlin:kotlin-stdlib:2.4.0=allDevSourceSetsCompileDependenciesMetadata,allSourceSetsCompileDependenciesMetadata,allTestSourceSetsCompileDependenciesMetadata,commonMainResolvableDependenciesMetadata,commonTestResolvableDependenciesMetadata,composeHotReloadDevDesktopDevRuntimeClasspath,composeHotReloadDevDesktopRuntimeClasspath,composeHotReloadDevDesktopTestRuntimeClasspath,composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,desktopCompileClasspath,desktopDevCompileClasspath,desktopDevResolvableDependenciesMetadata,desktopDevRuntimeClasspath,desktopMainCompileClasspath,desktopMainResolvableDependenciesMetadata,desktopMainRuntimeClasspath,desktopRuntimeClasspath,desktopTestCompileClasspath,desktopTestResolvableDependenciesMetadata,desktopTestRuntimeClasspath,jvmCompileClasspath,jvmDevCompileClasspath,jvmDevResolvableDependenciesMetadata,jvmDevRuntimeClasspath,jvmMainCompileClasspath,jvmMainResolvableDependenciesMetadata,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest,kotlinKlibCommonizerClasspath,metadataCommonMainCompileClasspath,metadataCompileClasspath,wasmJsCompileClasspath,wasmJsMainResolvableDependenciesMetadata,wasmJsNpmAggregated,wasmJsRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestResolvableDependenciesMetadata,wasmJsTestRuntimeClasspath,webMainResolvableDependenciesMetadata,webTestResolvableDependenciesMetadata -org.jetbrains.kotlin:kotlin-stdlib:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-stdlib:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-test-junit:2.4.0=composeHotReloadDevDesktopTestRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,desktopTestCompileClasspath,desktopTestRuntimeClasspath,jvmTestCompileClasspath,jvmTestRuntimeClasspath org.jetbrains.kotlin:kotlin-test-wasm-js:2.4.0=wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestRuntimeClasspath org.jetbrains.kotlin:kotlin-test:2.4.0=allTestSourceSetsCompileDependenciesMetadata,commonTestResolvableDependenciesMetadata,composeHotReloadDevJvmTestRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestResolvableDependenciesMetadata,wasmJsTestRuntimeClasspath,webTestResolvableDependenciesMetadata org.jetbrains.kotlin:kotlin-tooling-core:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-tooling-core:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-tooling-core:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlinx:atomicfu-jvm:0.23.2=composeHotReloadDevTools org.jetbrains.kotlinx:atomicfu-jvm:0.33.0=composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,jvmDevRuntimeClasspath,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestRuntimeClasspath org.jetbrains.kotlinx:atomicfu-wasm-js:0.33.0=wasmJsCompileClasspath,wasmJsNpmAggregated,wasmJsRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestRuntimeClasspath From 17bcad6a21ac15f20fcf775b63c951ddabb412a7 Mon Sep 17 00:00:00 2001 From: SolrBot Date: Tue, 1 Sep 2026 01:03:38 +0000 Subject: [PATCH 02/14] Add changelog entry for PR#4853 --- .../PR#4853-update-org-apache-tika-tika-core.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml diff --git a/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml b/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml new file mode 100644 index 000000000000..bf8bad38c896 --- /dev/null +++ b/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml @@ -0,0 +1,7 @@ +title: Update org.apache.tika:tika-core to v4 +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#4853 + url: https://github.com/apache/solr/pull/4853 From fe734927d3d2f2014ea99c3693e2b5c0f7904a99 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 1 Sep 2026 16:59:07 -0400 Subject: [PATCH 03/14] Add missing license for tika 4 --- solr/licenses/commonmark-LICENSE-BSD.txt | 23 +++++++++++++++++++++++ solr/licenses/commonmark-NOTICE.txt | 0 2 files changed, 23 insertions(+) create mode 100644 solr/licenses/commonmark-LICENSE-BSD.txt create mode 100644 solr/licenses/commonmark-NOTICE.txt diff --git a/solr/licenses/commonmark-LICENSE-BSD.txt b/solr/licenses/commonmark-LICENSE-BSD.txt new file mode 100644 index 000000000000..604b777d32c7 --- /dev/null +++ b/solr/licenses/commonmark-LICENSE-BSD.txt @@ -0,0 +1,23 @@ +Copyright (c) 2015, Robin Stocker +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/solr/licenses/commonmark-NOTICE.txt b/solr/licenses/commonmark-NOTICE.txt new file mode 100644 index 000000000000..e69de29bb2d1 From d9daf06557e9dcd599c12f2c70fded2a4b0e2e80 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 1 Sep 2026 19:56:44 -0400 Subject: [PATCH 04/14] barebones implmentation that doesn't not use markdown, it stays with xml. This is mostly a "hey, look, I got us to tika 4", but it feels like its still trying to push Tika 4 into a tika 3 approach, when Tika wants to be different. --- .../extraction/ExtractingDocumentLoader.java | 1 + .../handler/extraction/ExtractingParams.java | 10 + .../handler/extraction/ExtractionRequest.java | 16 +- .../TikaServerExtractionBackend.java | 195 ++++++++++++++---- .../handler/extraction/TikaServerParser.java | 7 +- .../extraction/tika-server-config.json | 5 + .../ExtractingRequestHandlerTestAbstract.java | 74 ++++--- ...xtractingRequestHandlerTikaServerTest.java | 9 +- .../TikaServerExtractionBackendTest.java | 131 +++++++++++- solr/packaging/test/test_extraction.bats | 2 +- .../getting-started/pages/tutorial-diy.adoc | 2 +- .../pages/indexing-with-tika.adoc | 44 +++- 12 files changed, 406 insertions(+), 90 deletions(-) create mode 100644 solr/modules/extraction/src/test-files/extraction/tika-server-config.json diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java index d4fd1c4cbd1b..d5d343846834 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java @@ -143,6 +143,7 @@ public void load( .tikaServerRecursive(tikaserverRecursive) .tikaServerTimeoutSeconds(tikaTimeoutSecs) .tikaServerRequestHeaders(Map.of()) + .tikaServerConfigJson(params.get(ExtractingParams.TIKASERVER_CONFIG_JSON)) .build(); boolean captureAttr = params.getBool(ExtractingParams.CAPTURE_ATTRIBUTES, false); diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index a99025127b19..5a52f2ac02aa 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -157,4 +157,14 @@ public interface ExtractingParams { /** Default or per-request timeout in seconds for TikaServer HTTP calls. */ String TIKASERVER_TIMEOUT_SECS = "tikaserver.timeoutSeconds"; + + /** + * Optional raw JSON object sent as the "config" part of a per-request TikaServer configuration + * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). Tika Server 4.x removed its + * X-Tika-* configuration headers in favor of this JSON mechanism; the server must additionally + * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Ignored for + * recursive (tikaserver.recursive) requests, since TikaServer has no XML-output variant of + * /rmeta/config. + */ + String TIKASERVER_CONFIG_JSON = "tikaserver.config"; } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java index 79068816271a..d34496772ce3 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java @@ -38,6 +38,7 @@ public class ExtractionRequest { public final boolean tikaServerRecursive; public final Integer tikaServerTimeoutSeconds; // optional per-request override public final Map tikaServerRequestHeaders = new HashMap<>(); + public final String tikaServerConfigJson; // optional raw JSON "config" part; TikaServer 4.x only /** * Constructs an ExtractionRequest object containing metadata and configurations for extraction @@ -59,6 +60,8 @@ public class ExtractionRequest { * only). If null or ≤ 0, the default timeout will be used * @param tikaServerRequestHeaders optional headers to be included in requests to the extraction * service. TikaServer only + * @param tikaServerConfigJson optional raw JSON object sent as the per-request "config" part. + * TikaServer 4.x only; requires allowPerRequestConfig=true on the server */ private ExtractionRequest( String streamType, @@ -73,7 +76,8 @@ private ExtractionRequest( String extractFormat, boolean tikaServerRecursive, Integer tikaServerTimeoutSeconds, - Map tikaServerRequestHeaders) { + Map tikaServerRequestHeaders, + String tikaServerConfigJson) { this.streamType = streamType; this.resourceName = resourceName; this.contentType = contentType; @@ -89,6 +93,7 @@ private ExtractionRequest( if (tikaServerRequestHeaders != null) { this.tikaServerRequestHeaders.putAll(tikaServerRequestHeaders); } + this.tikaServerConfigJson = tikaServerConfigJson; } /** Creates a new Builder for constructing ExtractionRequest instances. */ @@ -111,6 +116,7 @@ public static class Builder { private boolean tikaServerRecursive = false; private Integer tikaServerTimeoutSeconds; private Map tikaServerRequestHeaders; + private String tikaServerConfigJson; private Builder() {} @@ -179,6 +185,11 @@ public Builder tikaServerRequestHeaders(Map tikaServerRequestHea return this; } + public Builder tikaServerConfigJson(String tikaServerConfigJson) { + this.tikaServerConfigJson = tikaServerConfigJson; + return this; + } + public ExtractionRequest build() { return new ExtractionRequest( streamType, @@ -193,7 +204,8 @@ public ExtractionRequest build() { extractFormat, tikaServerRecursive, tikaServerTimeoutSeconds, - tikaServerRequestHeaders); + tikaServerRequestHeaders, + tikaServerConfigJson); } } } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index a6d08bb11f77..395345087c9f 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PushbackInputStream; +import java.lang.invoke.MethodHandles; import java.net.ConnectException; import java.net.MalformedURLException; import java.net.SocketTimeoutException; @@ -37,15 +39,23 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.Utils; import org.apache.solr.util.RefCounted; import org.apache.tika.sax.BodyContentHandler; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.InputStreamRequestContent; import org.eclipse.jetty.client.InputStreamResponseListener; +import org.eclipse.jetty.client.MultiPartRequestContent; import org.eclipse.jetty.client.Request; import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.client.StringRequestContent; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.MultiPart; import org.eclipse.jetty.io.EofException; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.helpers.DefaultHandler; /** @@ -53,6 +63,8 @@ * import of org.apache.tika.sax.BodyContentHandler; */ public class TikaServerExtractionBackend implements ExtractionBackend { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** * Default maximum response size (100MB) to prevent excessive memory usage from large documents */ @@ -173,26 +185,81 @@ public void extractWithSaxHandler( * request.tikaserverRecursive */ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) throws Exception { - String url = baseUrl + (request.tikaServerRecursive ? "/rmeta" : "/tika"); + ExtractionMetadata md = buildMetadataFromRequest(request); + String pwd = resolvePassword(request, md); + String configJson = resolveConfigJson(request, pwd); HttpClient client = acquiredResourcesRef.get().client; - - Request req = client.newRequest(url).method("PUT"); Duration effectiveTimeout = (request.tikaServerTimeoutSeconds != null && request.tikaServerTimeoutSeconds > 0) ? Duration.ofSeconds(request.tikaServerTimeoutSeconds) : defaultTimeout; - req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); - // Also set idle timeout in case of heavy server side work like OCR - req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); - - // Headers - String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); - req.headers(h -> h.add("Accept", accept)); String contentType = (request.streamType != null) ? request.streamType : request.contentType; - if (contentType != null) { - req.headers(h -> h.add("Content-Type", contentType)); + + String url; + Request req; + if (configJson != null && request.tikaServerRecursive) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Per-request TikaServer config (password or " + + ExtractingParams.TIKASERVER_CONFIG_JSON + + ") is not supported together with " + + ExtractingParams.TIKASERVER_RECURSIVE + + "=true: TikaServer 4.x has no XML-output variant of /rmeta/config."); + } + if (configJson != null) { + // Tika 4.x dropped its X-Tika-* configuration headers (including Password) in favor of a + // per-request JSON "config" part on a multipart request; the server must additionally opt + // in with allowPerRequestConfig=true. There is no XML content-handler variant of + // /rmeta/config in Tika 4.x, so this path only covers non-recursive extraction (checked + // above). + url = baseUrl + "/tika/config/xml"; + req = client.newRequest(url).method("POST"); + req.headers(h -> h.add("Accept", "text/xml")); + + HttpFields.Mutable fileFields = HttpFields.build(); + if (contentType != null) { + fileFields.add(HttpHeader.CONTENT_TYPE, contentType); + } + try (MultiPartRequestContent multiPart = new MultiPartRequestContent()) { + multiPart.addPart( + new MultiPart.ContentSourcePart( + "file", + request.resourceName, + fileFields, + new InputStreamRequestContent(inputStream))); + multiPart.addPart( + new MultiPart.ContentSourcePart( + "config", + null, + HttpFields.build().add(HttpHeader.CONTENT_TYPE, "application/json"), + new StringRequestContent(configJson))); + req.body(multiPart); + } + } else { + // Tika 4.x's default content handler is Markdown, not XHTML/XML (TIKA-4663); Solr's SAX-based + // content handling needs the previous XHTML/XML output, requested via the /xml path variants. + url = baseUrl + (request.tikaServerRecursive ? "/rmeta/xml" : "/tika/xml"); + req = client.newRequest(url).method("PUT"); + String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); + req.headers(h -> h.add("Accept", accept)); + if (contentType != null) { + req.headers(h -> h.add("Content-Type", contentType)); + } + if (request.resourceName != null) { + req.headers( + h -> + h.add( + "Content-Disposition", + "attachment; filename=\"" + request.resourceName + "\"")); + } + if (contentType != null) { + req.body(new InputStreamRequestContent(contentType, inputStream)); + } else { + req.body(new InputStreamRequestContent(inputStream)); + } } + if (!request.tikaServerRequestHeaders.isEmpty()) { req.headers( h -> @@ -202,32 +269,9 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t })); } - ExtractionMetadata md = buildMetadataFromRequest(request); - if (request.resourcePassword != null || request.passwordsMap != null) { - RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider(); - if (request.resourcePassword != null) { - passwordProvider.setExplicitPassword(request.resourcePassword); - } - if (request.passwordsMap != null) { - passwordProvider.setPasswordMap(request.passwordsMap); - } - String pwd = passwordProvider.getPassword(md); - if (pwd != null) { - req.headers(h -> h.add("Password", pwd)); // Tika Server expects this header if provided - } - } - if (request.resourceName != null) { - req.headers( - h -> - h.add( - "Content-Disposition", "attachment; filename=\"" + request.resourceName + "\"")); - } - - if (contentType != null) { - req.body(new InputStreamRequestContent(contentType, inputStream)); - } else { - req.body(new InputStreamRequestContent(inputStream)); - } + req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); + // Also set idle timeout in case of heavy server side work like OCR + req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); InputStreamResponseListener listener = new InputStreamResponseListener(); req.send(listener); @@ -273,7 +317,33 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t } int code = response.getStatus(); - if (code < 200 || code >= 300) { + InputStream responseStream = listener.getInputStream(); + // Tika 4.x's raw /tika* endpoints (non-recursive) return 422 whenever a container-level + // exception occurred during parsing -- including a non-aborting one like a writeLimit + // truncation -- but the body still carries whatever content was successfully extracted + // (there's no envelope to carry the exception itself on these endpoints; use /rmeta for + // that). A request that extracted nothing at all (e.g. a wrong password) also gets 422, but + // with an empty body -- treat that case as the failure it is instead of a silent empty + // "success". Peek the first byte to tell the two apart. + if (code == 422 && !request.tikaServerRecursive) { + PushbackInputStream peekable = new PushbackInputStream(responseStream, 1); + int firstByte = peekable.read(); + if (firstByte == -1) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "TikaServer " + + url + + " returned status 422 (Unprocessable Entity) with no content -- the document" + + " could not be parsed at all (check the password, if one was required)."); + } + peekable.unread(firstByte); + log.warn( + "TikaServer {} returned 422 (a container-level exception occurred during parsing); " + + "using the partial content it still returned. Use tikaserver.recursive=true " + + "against /rmeta for the exception detail.", + url); + responseStream = peekable; + } else if (code < 200 || code >= 300) { SolrException.ErrorCode errorCode = SolrException.ErrorCode.getErrorCode(code); String reason = response.getReason(); String msg = @@ -285,11 +355,56 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t throw new SolrException(errorCode, msg); } - InputStream responseStream = listener.getInputStream(); // Bound the amount of data we read from Tika Server to avoid excessive memory/CPU usage return new LimitingInputStream(responseStream, maxCharsLimit); } + /** Resolves the password to use for an encrypted document, or null if none applies. */ + private String resolvePassword(ExtractionRequest request, ExtractionMetadata md) { + if (request.resourcePassword == null && request.passwordsMap == null) { + return null; + } + RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider(); + if (request.resourcePassword != null) { + passwordProvider.setExplicitPassword(request.resourcePassword); + } + if (request.passwordsMap != null) { + passwordProvider.setPasswordMap(request.passwordsMap); + } + return passwordProvider.getPassword(md); + } + + /** + * Builds the per-request TikaServer JSON "config" payload, merging any resolved password with any + * caller-supplied {@link ExtractingParams#TIKASERVER_CONFIG_JSON}. Returns null if neither + * applies, meaning no per-request config is needed. + */ + @SuppressWarnings("unchecked") + private String resolveConfigJson(ExtractionRequest request, String pwd) { + Map config = new LinkedHashMap<>(); + if (request.tikaServerConfigJson != null && !request.tikaServerConfigJson.isBlank()) { + Object parsed; + try { + parsed = Utils.fromJSONString(request.tikaServerConfigJson); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Invalid JSON in " + ExtractingParams.TIKASERVER_CONFIG_JSON + ": " + e.getMessage(), + e); + } + if (!(parsed instanceof Map)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + ExtractingParams.TIKASERVER_CONFIG_JSON + " must be a JSON object"); + } + config.putAll((Map) parsed); + } + if (pwd != null && !config.containsKey("simple-password-provider")) { + config.put("simple-password-provider", Map.of("password", pwd)); + } + return config.isEmpty() ? null : Utils.toJSONString(config); + } + private static class LimitingInputStream extends InputStream { private final InputStream in; private final long max; diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java index e34514040b6e..41f9b83364c1 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java @@ -87,11 +87,14 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe for (Object o : list) { if (!(o instanceof Map)) continue; Map map = (Map) o; + // Tika 4.x renamed X-TIKA:content to tk:content (TIKA-4816); accept either so this also + // works against a Tika Server 3.x still on the old key. + String contentKey = map.containsKey("tk:content") ? "tk:content" : "X-TIKA:content"; // Copy metadata for (Object k : map.keySet()) { String key = String.valueOf(k); Object val = map.get(k); - if ("X-TIKA:content".equalsIgnoreCase(key)) { + if (contentKey.equalsIgnoreCase(key)) { // handled below continue; } @@ -103,7 +106,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe md.add(key, String.valueOf(val)); } } - Object content = map.get("X-TIKA:content"); + Object content = map.get(contentKey); if (content != null) { String xhtml = String.valueOf(content); if (!xhtml.isEmpty() && handler != null) { diff --git a/solr/modules/extraction/src/test-files/extraction/tika-server-config.json b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json new file mode 100644 index 000000000000..63a1e4fcbd77 --- /dev/null +++ b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json @@ -0,0 +1,5 @@ +{ + "server": { + "allowPerRequestConfig": true + } +} diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java index c9c872bc99de..f6f4cef080a1 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java @@ -440,17 +440,20 @@ public void testLiterals() throws Exception { "one", "literal.extractionLiteral", "two", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-parser-override", + "ignored_parser", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.Last-Modified", "extractedDate"); @@ -480,17 +483,20 @@ public void testLiterals() throws Exception { "extractedLanguage", "literal.extractionLiteral", "one", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.Last-Modified", "extractedDate"); @@ -596,21 +602,26 @@ public void testPlainTextSpecifyingMimeType() throws Exception { "one", "fmap.language", "extractedLanguage", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:detected-encoding", + "ignored_parser", + "fmap.tk:encoding-detector", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:encoding-detection-trace", "ignored_parser", - "fmap.X-TIKA:detectedEncoding", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:encodingDetector", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.content", "extractedContent", @@ -644,21 +655,28 @@ public void testPlainTextSpecifyingResourceName() throws Exception { "one", "fmap.language", "extractedLanguage", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:detected-encoding", + "ignored_parser", + "fmap.tk:encoding-detector", + "ignored_parser", + "fmap.tk:encoding-detection-trace", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:detectedEncoding", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:encodingDetector", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-magic-detected", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:resource-name", "ignored_parser", "fmap.content", "extractedContent", diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java index 23470049c985..1c76fe6e668a 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; @ThreadLeakFilters(filters = {SolrIgnoredThreadsFilter.class, QuickPatchThreadsFilter.class}) public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHandlerTestAbstract { @@ -43,9 +44,15 @@ public static void beforeClassTika() { String baseUrl; try { + // allowPerRequestConfig is off by default (it lets a client inject arbitrary parser + // config, e.g. for encrypted-document passwords); enabling it here is test-only. tika = - new GenericContainer<>("apache/tika:3.2.3.0-full") + new GenericContainer<>("apache/tika:4.0.0-full") .withExposedPorts(9998) + .withCopyFileToContainer( + MountableFile.forHostPath(getFile("extraction/tika-server-config.json")), + "/tika-config.json") + .withCommand("-c", "/tika-config.json") .waitingFor(Wait.forListeningPort()); tika.start(); baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998); diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index b7723da94ff9..dc740edd793b 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -34,6 +34,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.MountableFile; /** * Integration tests for TikaServerExtractionBackend using a real Tika Server via Testcontainers. @@ -70,7 +71,16 @@ public static void startTikaServer() { "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); try { - tika = new GenericContainer<>("apache/tika:3.2.3.0-full").withExposedPorts(9998); + // allowPerRequestConfig is off by default (it lets a client inject arbitrary parser + // config, e.g. for encrypted-document passwords or tikaserver.config); enabling it here is + // test-only. + tika = + new GenericContainer<>("apache/tika:4.0.0-full") + .withExposedPorts(9998) + .withCopyFileToContainer( + MountableFile.forHostPath(getFile("extraction/tika-server-config.json")), + "/tika-config.json") + .withCommand("-c", "/tika-config.json"); tika.start(); baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998); } catch (Throwable t) { @@ -154,14 +164,15 @@ public void testPdfWithImageRecursive() throws Exception { Assume.assumeTrue("Tika server container not started", tika != null); try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); - // Enable recursive extraction and set header to extract images from PDF + // Tika 4.x removed the X-Tika-* header family entirely (see resolveConfigJson's javadoc); + // there is no replacement for this combination. Per-request config now requires the + // multipart /config endpoints, but Tika 4.x has no XML-output variant of /rmeta/config, so + // per-request PDF options (e.g. explicit inline-image extraction) cannot be requested + // together with tikaserver.recursive=true. The PDF's embedded image still gets OCR'd into + // the main document's content by default, just not exposed as a separate embedded + // resource entry the way the pre-4.x X-Tika-PDFextractInlineImages header used to. ExtractionRequest request = - newRequest( - "pdf-with-image.pdf", - "application/pdf", - "xml", - true, - Map.of("X-Tika-PDFextractInlineImages", "true")); + newRequest("pdf-with-image.pdf", "application/pdf", "xml", true, Map.of()); try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { ToXMLContentHandler xmlHandler = new ToXMLContentHandler(); ExtractionMetadata md = backend.buildMetadataFromRequest(request); @@ -169,9 +180,8 @@ public void testPdfWithImageRecursive() throws Exception { String c = xmlHandler.toString(); assertNotNull(c); assertTrue(c.contains("Puppet Apply")); - assertTrue(c.contains("embedded:image0.jpg")); - assertEquals( - "org.apache.tika.parser.DefaultParser", md.getFirst("X-TIKA:Parsed-By-Full-Set")); + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + assertEquals("org.apache.tika.parser.DefaultParser", md.getFirst("tk:parsed-by-full-set")); } } } @@ -227,4 +237,103 @@ public void testMaxCharsLimitEnforcedWithSaxHandler() throws Exception { } } } + + private static ExtractionRequest newRequestWithConfig( + String resourceName, String contentType, String extractFormat, String configJson) { + return ExtractionRequest.builder() + .streamType(contentType) + .resourceName(resourceName) + .contentType(contentType) + .streamName(resourceName) + .extractFormat(extractFormat) + .tikaServerConfigJson(configJson) + .build(); + } + + @Test + public void testConfigJsonDisablesOcr() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); + // With no config, the PDF's embedded image gets OCR'd and "Puppet Apply" (from the image) + // appears in the extracted content. Disabling OCR via tikaserver.config should suppress it. + ExtractionRequest request = + newRequestWithConfig( + "pdf-with-image.pdf", + "application/pdf", + "xml", + "{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + ExtractionResult res = backend.extract(in, request); + assertNotNull(res.getContent()); + assertFalse( + "Expected tikaserver.config's NO_OCR strategy to suppress the OCR'd image text", + res.getContent().contains("Puppet Apply")); + } + } + } + + @Test + public void testConfigJsonMergesWithPassword() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = Files.readAllBytes(getFile("extraction/encrypted-password-is-solrRules.pdf")); + ExtractionRequest request = + ExtractionRequest.builder() + .streamType("application/pdf") + .resourceName("encrypted-password-is-solrRules.pdf") + .contentType("application/pdf") + .streamName("encrypted-password-is-solrRules.pdf") + .extractFormat("xml") + .resourcePassword("solrRules") + .tikaServerConfigJson("{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}") + .build(); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + ExtractionResult res = backend.extract(in, request); + assertNotNull(res); + assertTrue( + "Expected the password-unlocked content to still be present alongside the merged" + + " tikaserver.config", + res.getContent().contains("This is a test of PDF and Word extraction")); + } + } + } + + @Test + public void testInvalidConfigJsonRejected() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = "hello".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = + newRequestWithConfig("test.txt", "text/plain", "xml", "not valid json"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_CONFIG_JSON)); + } + } + } + + @Test + public void testConfigJsonRejectedForRecursive() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = "hello".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = + ExtractionRequest.builder() + .streamType("text/plain") + .resourceName("test.txt") + .contentType("text/plain") + .streamName("test.txt") + .extractFormat("xml") + .tikaServerRecursive(true) + .tikaServerConfigJson("{\"parse-context\":{}}") + .build(); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_RECURSIVE)); + } + } + } } diff --git a/solr/packaging/test/test_extraction.bats b/solr/packaging/test/test_extraction.bats index 4b8e62ae59cc..0823377452c6 100644 --- a/solr/packaging/test/test_extraction.bats +++ b/solr/packaging/test/test_extraction.bats @@ -40,7 +40,7 @@ apply_extract_handler() { setup_file() { if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then export TIKA_PORT=$((SOLR_PORT+5)) - docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:3.2.3.0-full >/dev/null 2>&1 || true + docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:4.0.0-full >/dev/null 2>&1 || true echo "Waiting for Tika Server to be ready on port ${TIKA_PORT}" >&3 if ! wait_for 120 3 curl -s -f "http://localhost:${TIKA_PORT}/tika" -o /dev/null; then export DOCKER_UNAVAILABLE=1 diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc index db34f483b69f..d58038d509e3 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc @@ -39,7 +39,7 @@ Indexing binary files (PDF, DOCX, PPTX, etc.) with the Post Tool requires the So .Start a Tika Server quickly using Docker (exposes port 9998 on localhost) [,bash] ---- -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full ---- === Start Solr with the extraction module enabled diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index 1a50f8f14c6e..02f70b3da291 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -78,7 +78,7 @@ The quickest way to run Tika Server for development is using Docker. The example [,bash] ---- -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full ---- NOTE: If Solr runs in Docker too, ensure both containers share a network and use the Tika container name as the host in `tikaserver.url`. @@ -92,7 +92,7 @@ First we start a tika server on port 9998, using Docker. [source,bash] ---- # Start Tika Server in the background -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full # To stop the server when done, run `docker stop tika` ---- @@ -384,6 +384,18 @@ Example: `passwordsFile=/path/to/passwords.txt` // + // Only applicable for `tikaserver` backend. Can only be set in `solrconfig.xml`, not per request. +`tikaserver.config`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A raw JSON object sent as the per-request parser configuration for Tika Server (e.g., `{"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}`). +See <> below for details and an important security note: this requires `allowPerRequestConfig=true` on the Tika Server, which is off by default. ++ +Example: `tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}` + `tikaserver.maxChars`:: + [%autowidth,frame=none] @@ -540,9 +552,33 @@ So you can use the other URPs without worrying about unexpected field additions. === Parser-Specific Properties -Parser-specific properties for Tika must be configured directly on your Tika Server instance. Consult the https://tika.apache.org/[Apache Tika documentation] for details. +Server-wide parser properties (things that should apply to every request) must be configured directly on your Tika Server instance, via its own JSON configuration file. Consult the https://tika.apache.org/[Apache Tika documentation] for details. + +For a single request, you can instead pass parser-specific options through Solr using the `tikaserver.config` parameter, whose value is a raw JSON object matching Tika Server's per-request configuration format. +For example, to disable OCR for one request: + +[,console] +---- +$ bin/solr post -c gettingstarted example/exampledocs/solr-word.pdf --params 'literal.id=doc1&tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' +---- + +[IMPORTANT] +==== +`tikaserver.config` requires your Tika Server to be started with `allowPerRequestConfig: true` in its own JSON configuration (under the `server` section). +This is off by default, and Tika Server logs a warning when it is enabled, because it lets any client that can reach `/update/extract` inject arbitrary parser configuration, including options that spawn external processes such as OCR. +Only enable it if you need per-request configuration, and treat access to your Solr instance's extraction endpoint accordingly. + +[source,json] +---- +{ + "server": { + "allowPerRequestConfig": true + } +} +---- +==== -NOTE: In earlier versions of Solr Cell you could supply Tika configuration directly to Solr. This is no longer possible. +`tikaserver.config` is combined with any password resolved from `resource.password` or `passwordsFile` (see <>) into a single request to Tika Server, and is only supported for non-recursive extraction (`tikaserver.recursive=false`, the default); Tika Server has no way to accept per-request configuration for recursive extraction while also returning the XHTML content Solr Cell needs. === Indexing Encrypted Documents From 5ccf4632c256a9abe159708c5b9cddb80a8c6583 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 3 Sep 2026 17:42:15 -0400 Subject: [PATCH 05/14] Support Tika 4 method of passing in configuraiton. --- ...xtractingRequestHandlerTikaServerTest.java | 3 ++- .../extraction/TikaServerContainerRule.java | 22 ++++++++++++++++++- .../TikaServerExtractionBackendTest.java | 19 ++++++++-------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java index f32dbb5289c8..7f31e8fcd6d6 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java @@ -31,7 +31,8 @@ public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHan private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @ClassRule - public static final TikaServerContainerRule tikaContainer = new TikaServerContainerRule(); + public static final TikaServerContainerRule tikaContainer = + new TikaServerContainerRule(getFile("extraction/tika-server-config.json")); @BeforeClass public static void beforeClassTika() throws Exception { diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java index 7a6ae3937998..290992163322 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java @@ -17,6 +17,7 @@ package org.apache.solr.handler.extraction; import java.lang.invoke.MethodHandles; +import java.nio.file.Path; import org.junit.Assume; import org.junit.rules.ExternalResource; import org.slf4j.Logger; @@ -24,6 +25,7 @@ import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; /** * JUnit rule that manages a single Apache Tika Server Testcontainer. Declare as a @@ -38,11 +40,25 @@ public class TikaServerContainerRule extends ExternalResource { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String TIKA_DOCKER_IMAGE = "apache/tika:3.2.3.0-full"; + public static final String TIKA_DOCKER_IMAGE = "apache/tika:4.0.0-full"; + private final Path serverConfigFile; private GenericContainer tika; private String baseUrl; + public TikaServerContainerRule() { + this(null); + } + + /** + * @param serverConfigFile optional Tika Server JSON config file to mount and start the container + * with (via {@code -c}), e.g. to set {@code allowPerRequestConfig: true}. Null for the + * container's default configuration. + */ + public TikaServerContainerRule(Path serverConfigFile) { + this.serverConfigFile = serverConfigFile; + } + @Override @SuppressWarnings("resource") protected void before() { @@ -56,6 +72,10 @@ protected void before() { new GenericContainer<>(TIKA_DOCKER_IMAGE) .withExposedPorts(9998) .waitingFor(Wait.forListeningPort()); + if (serverConfigFile != null) { + tika.withCopyFileToContainer(MountableFile.forHostPath(serverConfigFile), "/tika-config.json") + .withCommand("-c", "/tika-config.json"); + } tika.start(); baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998); } diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index 3d684713ebde..1a677bfc3877 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -58,7 +58,8 @@ public boolean reject(Thread t) { } @ClassRule - public static final TikaServerContainerRule tikaContainer = new TikaServerContainerRule(); + public static final TikaServerContainerRule tikaContainer = + new TikaServerContainerRule(getFile("extraction/tika-server-config.json")); private static ExtractionRequest newRequest( String resourceName, @@ -210,8 +211,8 @@ private static ExtractionRequest newRequestWithConfig( @Test public void testConfigJsonDisablesOcr() throws Exception { - Assume.assumeTrue("Tika server container not started", tika != null); - try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); // With no config, the PDF's embedded image gets OCR'd and "Puppet Apply" (from the image) // appears in the extracted content. Disabling OCR via tikaserver.config should suppress it. @@ -233,8 +234,8 @@ public void testConfigJsonDisablesOcr() throws Exception { @Test public void testConfigJsonMergesWithPassword() throws Exception { - Assume.assumeTrue("Tika server container not started", tika != null); - try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { byte[] data = Files.readAllBytes(getFile("extraction/encrypted-password-is-solrRules.pdf")); ExtractionRequest request = ExtractionRequest.builder() @@ -259,8 +260,8 @@ public void testConfigJsonMergesWithPassword() throws Exception { @Test public void testInvalidConfigJsonRejected() throws Exception { - Assume.assumeTrue("Tika server container not started", tika != null); - try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { byte[] data = "hello".getBytes(StandardCharsets.UTF_8); ExtractionRequest request = newRequestWithConfig("test.txt", "text/plain", "xml", "not valid json"); @@ -274,8 +275,8 @@ public void testInvalidConfigJsonRejected() throws Exception { @Test public void testConfigJsonRejectedForRecursive() throws Exception { - Assume.assumeTrue("Tika server container not started", tika != null); - try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { byte[] data = "hello".getBytes(StandardCharsets.UTF_8); ExtractionRequest request = ExtractionRequest.builder() From c631731d830ac0633e6ae1e88e18ad7db35c2616 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 3 Sep 2026 17:50:05 -0400 Subject: [PATCH 06/14] Unused... --- .../solr/handler/extraction/TikaServerContainerRule.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java index 290992163322..368bdb0c75e1 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java @@ -46,10 +46,6 @@ public class TikaServerContainerRule extends ExternalResource { private GenericContainer tika; private String baseUrl; - public TikaServerContainerRule() { - this(null); - } - /** * @param serverConfigFile optional Tika Server JSON config file to mount and start the container * with (via {@code -c}), e.g. to set {@code allowPerRequestConfig: true}. Null for the From 3058834da24eea02fbeedb3aa8382239452ddc28 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 16:26:41 -0400 Subject: [PATCH 07/14] Remove previous version license file --- solr/licenses/tika-core-3.3.2.jar.sha1 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 solr/licenses/tika-core-3.3.2.jar.sha1 diff --git a/solr/licenses/tika-core-3.3.2.jar.sha1 b/solr/licenses/tika-core-3.3.2.jar.sha1 deleted file mode 100644 index 3887c7a34110..000000000000 --- a/solr/licenses/tika-core-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cb3323ade654469bb6f8d758d3eccbc146693f31 From 2c8d18070e6b27602c347dd7324056a2d70af43d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 16:33:54 -0400 Subject: [PATCH 08/14] Preserve error codes and gate TikaServer 422 partial success on ignoreTikaException - ExtractingDocumentLoader no longer rewraps SolrExceptions (e.g. BAD_REQUEST from invalid tikaserver.config) into SERVER_ERROR - A 422 with partial content from TikaServer's non-recursive endpoints now only succeeds when ignoreTikaException=true; defaults to failing, matching pre-4.x behavior - Reference TIKA-4881 where the config+recursive limitation is documented --- .../extraction/ExtractingDocumentLoader.java | 19 ++++++++++++--- .../handler/extraction/ExtractingParams.java | 7 +++--- .../handler/extraction/ExtractionRequest.java | 17 +++++++++++-- .../TikaServerExtractionBackend.java | 24 +++++++++++++++---- .../pages/indexing-with-tika.adoc | 2 +- 5 files changed, 55 insertions(+), 14 deletions(-) diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java index d5d343846834..abff0f38870b 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java @@ -67,6 +67,18 @@ public class ExtractingDocumentLoader extends ContentStreamLoader { protected SolrContentHandlerFactory factory; protected ExtractionBackend backend; + /** + * Rethrows {@code e} as-is if it's already a {@link SolrException} (preserving its error code, + * e.g. a {@code BAD_REQUEST} from invalid extraction parameters), otherwise wraps it in a + * {@code SERVER_ERROR}. + */ + private static SolrException wrapExtractionException(Exception e) { + if (e instanceof SolrException se) { + return se; + } + return new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + public ExtractingDocumentLoader( SolrQueryRequest req, UpdateRequestProcessor processor, @@ -144,6 +156,7 @@ public void load( .tikaServerTimeoutSeconds(tikaTimeoutSecs) .tikaServerRequestHeaders(Map.of()) .tikaServerConfigJson(params.get(ExtractingParams.TIKASERVER_CONFIG_JSON)) + .ignoreTikaException(ignoreTikaException) .build(); boolean captureAttr = params.getBool(ExtractingParams.CAPTURE_ATTRIBUTES, false); @@ -186,7 +199,7 @@ public void load( log.warn("skip extracting text due to {}.", e.getLocalizedMessage(), e); return; } - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + throw wrapExtractionException(e); } return; } @@ -204,7 +217,7 @@ public void load( return; } } - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + throw wrapExtractionException(e); } addDoc(handler); @@ -220,7 +233,7 @@ public void load( log.warn("skip extracting text due to {}.", e.getLocalizedMessage(), e); return; } - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + throw wrapExtractionException(e); } ExtractionMetadata metadata = result.getMetadata(); diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index 5a52f2ac02aa..4458045a05e1 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -162,9 +162,10 @@ public interface ExtractingParams { * Optional raw JSON object sent as the "config" part of a per-request TikaServer configuration * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). Tika Server 4.x removed its * X-Tika-* configuration headers in favor of this JSON mechanism; the server must additionally - * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Ignored for - * recursive (tikaserver.recursive) requests, since TikaServer has no XML-output variant of - * /rmeta/config. + * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Rejected with + * {@code BAD_REQUEST} for recursive (tikaserver.recursive) requests, since TikaServer has no + * XML-output variant of /rmeta/config (tracked upstream as + * TIKA-4881). */ String TIKASERVER_CONFIG_JSON = "tikaserver.config"; } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java index d34496772ce3..a68e5c66d644 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java @@ -39,6 +39,7 @@ public class ExtractionRequest { public final Integer tikaServerTimeoutSeconds; // optional per-request override public final Map tikaServerRequestHeaders = new HashMap<>(); public final String tikaServerConfigJson; // optional raw JSON "config" part; TikaServer 4.x only + public final boolean ignoreTikaException; /** * Constructs an ExtractionRequest object containing metadata and configurations for extraction @@ -62,6 +63,9 @@ public class ExtractionRequest { * service. TikaServer only * @param tikaServerConfigJson optional raw JSON object sent as the per-request "config" part. * TikaServer 4.x only; requires allowPerRequestConfig=true on the server + * @param ignoreTikaException if true, a backend that only partially extracted a document (e.g. a + * TikaServer 422 caused by a writeLimit truncation) may return that partial content instead + * of failing. TikaServer only */ private ExtractionRequest( String streamType, @@ -77,7 +81,8 @@ private ExtractionRequest( boolean tikaServerRecursive, Integer tikaServerTimeoutSeconds, Map tikaServerRequestHeaders, - String tikaServerConfigJson) { + String tikaServerConfigJson, + boolean ignoreTikaException) { this.streamType = streamType; this.resourceName = resourceName; this.contentType = contentType; @@ -94,6 +99,7 @@ private ExtractionRequest( this.tikaServerRequestHeaders.putAll(tikaServerRequestHeaders); } this.tikaServerConfigJson = tikaServerConfigJson; + this.ignoreTikaException = ignoreTikaException; } /** Creates a new Builder for constructing ExtractionRequest instances. */ @@ -117,6 +123,7 @@ public static class Builder { private Integer tikaServerTimeoutSeconds; private Map tikaServerRequestHeaders; private String tikaServerConfigJson; + private boolean ignoreTikaException = false; private Builder() {} @@ -190,6 +197,11 @@ public Builder tikaServerConfigJson(String tikaServerConfigJson) { return this; } + public Builder ignoreTikaException(boolean ignoreTikaException) { + this.ignoreTikaException = ignoreTikaException; + return this; + } + public ExtractionRequest build() { return new ExtractionRequest( streamType, @@ -205,7 +217,8 @@ public ExtractionRequest build() { tikaServerRecursive, tikaServerTimeoutSeconds, tikaServerRequestHeaders, - tikaServerConfigJson); + tikaServerConfigJson, + ignoreTikaException); } } } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index 395345087c9f..567d22c0c76d 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -199,13 +199,16 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t String url; Request req; if (configJson != null && request.tikaServerRecursive) { + // TikaServer 4.x has no XML-output variant of /rmeta/config; tracked upstream as + // https://issues.apache.org/jira/browse/TIKA-4881 throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Per-request TikaServer config (password or " + ExtractingParams.TIKASERVER_CONFIG_JSON + ") is not supported together with " + ExtractingParams.TIKASERVER_RECURSIVE - + "=true: TikaServer 4.x has no XML-output variant of /rmeta/config."); + + "=true: TikaServer 4.x has no XML-output variant of /rmeta/config" + + " (see https://issues.apache.org/jira/browse/TIKA-4881)."); } if (configJson != null) { // Tika 4.x dropped its X-Tika-* configuration headers (including Password) in favor of a @@ -323,8 +326,8 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t // truncation -- but the body still carries whatever content was successfully extracted // (there's no envelope to carry the exception itself on these endpoints; use /rmeta for // that). A request that extracted nothing at all (e.g. a wrong password) also gets 422, but - // with an empty body -- treat that case as the failure it is instead of a silent empty - // "success". Peek the first byte to tell the two apart. + // with an empty body -- that's always a hard failure, regardless of ignoreTikaException. + // Peek the first byte to tell the two apart. if (code == 422 && !request.tikaServerRecursive) { PushbackInputStream peekable = new PushbackInputStream(responseStream, 1); int firstByte = peekable.read(); @@ -337,10 +340,21 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t + " could not be parsed at all (check the password, if one was required)."); } peekable.unread(firstByte); + if (!request.ignoreTikaException) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "TikaServer " + + url + + " returned status 422 (Unprocessable Entity): a container-level exception" + + " occurred during parsing (e.g. a writeLimit truncation). Partial content was" + + " extracted but is being discarded because ignoreTikaException=false; set" + + " ignoreTikaException=true to index the partial content instead, or use" + + " tikaserver.recursive=true against /rmeta for the exception detail."); + } log.warn( "TikaServer {} returned 422 (a container-level exception occurred during parsing); " - + "using the partial content it still returned. Use tikaserver.recursive=true " - + "against /rmeta for the exception detail.", + + "using the partial content it still returned because ignoreTikaException=true. " + + "Use tikaserver.recursive=true against /rmeta for the exception detail.", url); responseStream = peekable; } else if (code < 200 || code >= 300) { diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index e1da9b61d227..b415f259cb1e 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -578,7 +578,7 @@ Only enable it if you need per-request configuration, and treat access to your S ---- ==== -`tikaserver.config` is combined with any password resolved from `resource.password` or `passwordsFile` (see <>) into a single request to Tika Server, and is only supported for non-recursive extraction (`tikaserver.recursive=false`, the default); Tika Server has no way to accept per-request configuration for recursive extraction while also returning the XHTML content Solr Cell needs. +`tikaserver.config` is combined with any password resolved from `resource.password` or `passwordsFile` (see <>) into a single request to Tika Server, and is only supported for non-recursive extraction (`tikaserver.recursive=false`, the default); Tika Server has no way to accept per-request configuration for recursive extraction while also returning the XHTML content Solr Cell needs (tracked upstream as https://issues.apache.org/jira/browse/TIKA-4881[TIKA-4881]). === Indexing Encrypted Documents From f55c8023ffc77f7ac302796c6f09d4b7bdf1721b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 16:39:14 -0400 Subject: [PATCH 09/14] Simplify the comment to be forward looking, and not talk about past tika --- .../handler/extraction/ExtractingParams.java | 9 ++++----- .../extraction/TikaServerExtractionBackend.java | 16 +++++++--------- .../handler/extraction/TikaServerParser.java | 4 ++-- .../ExtractingRequestHandlerTestAbstract.java | 8 ++++---- .../TikaServerExtractionBackendTest.java | 14 ++++++-------- 5 files changed, 23 insertions(+), 28 deletions(-) diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index 4458045a05e1..401c01dc3b65 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -160,11 +160,10 @@ public interface ExtractingParams { /** * Optional raw JSON object sent as the "config" part of a per-request TikaServer configuration - * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). Tika Server 4.x removed its - * X-Tika-* configuration headers in favor of this JSON mechanism; the server must additionally - * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Rejected with - * {@code BAD_REQUEST} for recursive (tikaserver.recursive) requests, since TikaServer has no - * XML-output variant of /rmeta/config (tracked upstream as + * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). The server must have {@code + * allowPerRequestConfig=true} set, or the request is rejected with 403. Rejected with {@code + * BAD_REQUEST} for recursive (tikaserver.recursive) requests, since TikaServer has no XML-output + * variant of /rmeta/config (tracked upstream as * TIKA-4881). */ String TIKASERVER_CONFIG_JSON = "tikaserver.config"; diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index 567d22c0c76d..0db5482fff2b 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -199,8 +199,7 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t String url; Request req; if (configJson != null && request.tikaServerRecursive) { - // TikaServer 4.x has no XML-output variant of /rmeta/config; tracked upstream as - // https://issues.apache.org/jira/browse/TIKA-4881 + // Tracked upstream: https://issues.apache.org/jira/browse/TIKA-4881 throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Per-request TikaServer config (password or " @@ -211,11 +210,10 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t + " (see https://issues.apache.org/jira/browse/TIKA-4881)."); } if (configJson != null) { - // Tika 4.x dropped its X-Tika-* configuration headers (including Password) in favor of a - // per-request JSON "config" part on a multipart request; the server must additionally opt - // in with allowPerRequestConfig=true. There is no XML content-handler variant of - // /rmeta/config in Tika 4.x, so this path only covers non-recursive extraction (checked - // above). + // TikaServer accepts per-request parser config (including passwords) as a JSON "config" + // part on a multipart request, and requires allowPerRequestConfig=true on the server. Only + // non-recursive extraction is handled here, since TikaServer has no XML content-handler + // variant of /rmeta/config (checked above). url = baseUrl + "/tika/config/xml"; req = client.newRequest(url).method("POST"); req.headers(h -> h.add("Accept", "text/xml")); @@ -240,8 +238,8 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t req.body(multiPart); } } else { - // Tika 4.x's default content handler is Markdown, not XHTML/XML (TIKA-4663); Solr's SAX-based - // content handling needs the previous XHTML/XML output, requested via the /xml path variants. + // TikaServer's /tika and /rmeta endpoints return Markdown by default (TIKA-4663); Solr's + // SAX-based content handling requires XHTML/XML, hence the /xml path variants. url = baseUrl + (request.tikaServerRecursive ? "/rmeta/xml" : "/tika/xml"); req = client.newRequest(url).method("PUT"); String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java index 41f9b83364c1..faf5eafe1f49 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java @@ -87,8 +87,8 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe for (Object o : list) { if (!(o instanceof Map)) continue; Map map = (Map) o; - // Tika 4.x renamed X-TIKA:content to tk:content (TIKA-4816); accept either so this also - // works against a Tika Server 3.x still on the old key. + // The content key is tk:content on TikaServer 4.x and X-TIKA:content on 3.x (TIKA-4816); + // accept either so this works against both. String contentKey = map.containsKey("tk:content") ? "tk:content" : "X-TIKA:content"; // Copy metadata for (Object k : map.keySet()) { diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java index f6f4cef080a1..b6553cf9f4bb 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java @@ -440,7 +440,7 @@ public void testLiterals() throws Exception { "one", "literal.extractionLiteral", "two", - // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816) "fmap.tk:parsed-by", "ignored_parser", "fmap.tk:parsed-by-full-set", @@ -483,7 +483,7 @@ public void testLiterals() throws Exception { "extractedLanguage", "literal.extractionLiteral", "one", - // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816) "fmap.tk:parsed-by", "ignored_parser", "fmap.tk:parsed-by-full-set", @@ -602,7 +602,7 @@ public void testPlainTextSpecifyingMimeType() throws Exception { "one", "fmap.language", "extractedLanguage", - // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816) "fmap.tk:parsed-by", "ignored_parser", "fmap.tk:detected-encoding", @@ -655,7 +655,7 @@ public void testPlainTextSpecifyingResourceName() throws Exception { "one", "fmap.language", "extractedLanguage", - // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816) "fmap.tk:parsed-by", "ignored_parser", "fmap.tk:detected-encoding", diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index 1a677bfc3877..2c59d69186a3 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -125,13 +125,11 @@ public void testPdfWithImageRecursive() throws Exception { try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); - // Tika 4.x removed the X-Tika-* header family entirely (see resolveConfigJson's javadoc); - // there is no replacement for this combination. Per-request config now requires the - // multipart /config endpoints, but Tika 4.x has no XML-output variant of /rmeta/config, so - // per-request PDF options (e.g. explicit inline-image extraction) cannot be requested - // together with tikaserver.recursive=true. The PDF's embedded image still gets OCR'd into - // the main document's content by default, just not exposed as a separate embedded - // resource entry the way the pre-4.x X-Tika-PDFextractInlineImages header used to. + // Explicit inline-image extraction options can't be requested here: TikaServer's per-request + // config only supports the non-recursive /tika/config/xml endpoint (see resolveConfigJson's + // javadoc), since there is no XML-output variant of /rmeta/config for recursive requests. The + // PDF's embedded image is still OCR'd into the main document's content by default, just not + // exposed as a separate embedded resource entry. ExtractionRequest request = newRequest("pdf-with-image.pdf", "application/pdf", "xml", true, Map.of()); try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { @@ -141,7 +139,7 @@ public void testPdfWithImageRecursive() throws Exception { String c = xmlHandler.toString(); assertNotNull(c); assertTrue(c.contains("Puppet Apply")); - // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816) assertEquals("org.apache.tika.parser.DefaultParser", md.getFirst("tk:parsed-by-full-set")); } } From 596e0e1506f0113d65ad72f0cba7c005f55aea19 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 17:07:24 -0400 Subject: [PATCH 10/14] fix lint --- .../solr/handler/extraction/ExtractingDocumentLoader.java | 4 ++-- .../org/apache/solr/handler/extraction/ExtractingParams.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java index abff0f38870b..c046125b3cef 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java @@ -69,8 +69,8 @@ public class ExtractingDocumentLoader extends ContentStreamLoader { /** * Rethrows {@code e} as-is if it's already a {@link SolrException} (preserving its error code, - * e.g. a {@code BAD_REQUEST} from invalid extraction parameters), otherwise wraps it in a - * {@code SERVER_ERROR}. + * e.g. a {@code BAD_REQUEST} from invalid extraction parameters), otherwise wraps it in a {@code + * SERVER_ERROR}. */ private static SolrException wrapExtractionException(Exception e) { if (e instanceof SolrException se) { diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index 401c01dc3b65..4c5d625c502c 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -163,8 +163,8 @@ public interface ExtractingParams { * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). The server must have {@code * allowPerRequestConfig=true} set, or the request is rejected with 403. Rejected with {@code * BAD_REQUEST} for recursive (tikaserver.recursive) requests, since TikaServer has no XML-output - * variant of /rmeta/config (tracked upstream as - * TIKA-4881). + * variant of /rmeta/config (tracked upstream as TIKA-4881). */ String TIKASERVER_CONFIG_JSON = "tikaserver.config"; } From 523d4d29ffe292c5a473973eddcd7104a898576a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 17:39:10 -0400 Subject: [PATCH 11/14] we don't use the old key anymore --- .../solr/handler/extraction/TikaServerParser.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java index faf5eafe1f49..746a91d7d5e1 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java @@ -35,6 +35,9 @@ import org.xml.sax.helpers.DefaultHandler; public class TikaServerParser { + // TikaServer 4.x's /rmeta content key (TIKA-4816). + private static final String CONTENT_KEY = "tk:content"; + private final SAXParser saxParser; public TikaServerParser() { @@ -87,14 +90,11 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe for (Object o : list) { if (!(o instanceof Map)) continue; Map map = (Map) o; - // The content key is tk:content on TikaServer 4.x and X-TIKA:content on 3.x (TIKA-4816); - // accept either so this works against both. - String contentKey = map.containsKey("tk:content") ? "tk:content" : "X-TIKA:content"; // Copy metadata for (Object k : map.keySet()) { String key = String.valueOf(k); Object val = map.get(k); - if (contentKey.equalsIgnoreCase(key)) { + if (CONTENT_KEY.equalsIgnoreCase(key)) { // handled below continue; } @@ -106,7 +106,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe md.add(key, String.valueOf(val)); } } - Object content = map.get(contentKey); + Object content = map.get(CONTENT_KEY); if (content != null) { String xhtml = String.valueOf(content); if (!xhtml.isEmpty() && handler != null) { From 2ea646762bb53206fc95d75f8a9bc3adb5d79518 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 17:51:37 -0400 Subject: [PATCH 12/14] Actually test teh version by making a fake tika server endpoint. Maybe should have been mocked more? Or some thing else? --- .../TikaServerVersionCheckTest.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java new file mode 100644 index 000000000000..950e910249d0 --- /dev/null +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.extraction; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.junit.After; +import org.junit.Test; + +/** + * Verifies that {@link TikaServerExtractionBackend} rejects a TikaServer older than {@code 4.x} + * with a clear diagnostic, rather than failing later with confusing 404s or missing metadata. + * + *

Uses a tiny in-process {@link HttpServer} stub for the {@code /version} endpoint instead of a + * real Tika Server, since that's all this check depends on. + */ +public class TikaServerVersionCheckTest extends SolrTestCaseJ4 { + + private HttpServer server; + + @After + public void stopServer() { + if (server != null) { + server.stop(0); + server = null; + } + } + + private String startServerWithVersion(String versionResponseBody) throws Exception { + server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + server.createContext("/version", exchange -> serveText(exchange, versionResponseBody)); + server.createContext( + "/tika/xml", + exchange -> + serveText( + exchange, + "" + + "hello world")); + server.start(); + return "http://localhost:" + server.getAddress().getPort(); + } + + private static void serveText(HttpExchange exchange, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + try (var os = exchange.getResponseBody()) { + os.write(bytes); + } + } + + private static ExtractionRequest newRequest() { + return ExtractionRequest.builder() + .streamType("text/plain") + .resourceName("test.txt") + .contentType("text/plain") + .streamName("test.txt") + .extractFormat("xml") + .build(); + } + + @Test + public void testRejectsPre4xTikaServer() throws Exception { + String baseUrl = startServerWithVersion("Apache Tika 3.2.3"); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + ExtractionRequest request = newRequest(); + try (ByteArrayInputStream in = + new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8))) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, e.code()); + assertTrue( + "Expected message to name the offending version and the minimum required, but was: " + + e.getMessage(), + e.getMessage().contains("Apache Tika 3.") && e.getMessage().contains("requires")); + } + } + } + + @Test + public void testAcceptsSupportedTikaServerVersion() throws Exception { + String baseUrl = startServerWithVersion("Apache Tika 4.0.0"); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + ExtractionRequest request = newRequest(); + try (ByteArrayInputStream in = + new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8))) { + ExtractionResult result = backend.extract(in, request); + assertNotNull(result); + assertTrue(result.getContent().contains("hello world")); + } + } + } +} From fd1f8ae8d808d3f64db97dc63cf53668f9fd436a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 17:52:00 -0400 Subject: [PATCH 13/14] Actually add the version check. --- .../TikaServerExtractionBackend.java | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index 0db5482fff2b..7af36a603260 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -35,6 +35,8 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; @@ -42,6 +44,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.util.RefCounted; import org.apache.tika.sax.BodyContentHandler; +import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.InputStreamRequestContent; import org.eclipse.jetty.client.InputStreamResponseListener; @@ -70,12 +73,24 @@ public class TikaServerExtractionBackend implements ExtractionBackend { */ public static final long DEFAULT_MAXCHARS_LIMIT = 100 * 1024 * 1024; + /** Minimum TikaServer major version this backend supports (relies on Tika 4.x-only APIs). */ + private static final int MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION = 4; + + private static final Pattern TIKASERVER_VERSION_PATTERN = + Pattern.compile("Apache Tika (\\d+)\\."); + + // Short-lived: /version is a trivial static-text endpoint, so this shouldn't use the full + // extraction defaultTimeout (which can be minutes) and block concurrent requests behind it. + private static final Duration VERSION_CHECK_TIMEOUT = Duration.ofSeconds(10); + private static final Object INIT_LOCK = new Object(); private final String baseUrl; private static final int DEFAULT_TIMEOUT_SECONDS = 3 * 60; private final Duration defaultTimeout; private final TikaServerParser tikaServerResponseParser = new TikaServerParser(); private boolean tikaMetadataCompatibility; + private boolean tikaServerVersionVerified = false; + private String rejectedTikaServerVersionMessage; private HashMap initArgsMap = new HashMap<>(); private final long maxCharsLimit; @@ -185,6 +200,8 @@ public void extractWithSaxHandler( * request.tikaserverRecursive */ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) throws Exception { + ensureSupportedTikaServerVersion(); + ExtractionMetadata md = buildMetadataFromRequest(request); String pwd = resolvePassword(request, md); String configJson = resolveConfigJson(request, pwd); @@ -371,6 +388,75 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t return new LimitingInputStream(responseStream, maxCharsLimit); } + /** + * Verifies, once per backend instance, that the configured TikaServer reports a major version of + * at least {@link #MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION}. This backend relies on endpoints + * (e.g. {@code /tika/xml}, {@code /tika/config/xml}) and metadata keys (e.g. {@code tk:content}) + * that only exist on TikaServer 4.x and newer; an older server would otherwise fail with + * confusing 404s or missing-metadata errors instead of a clear diagnostic. + * + *

Connectivity/parsing failures are retried on the next call rather than cached, since + * TikaServer may simply not be up yet. A definitively too-old version, however, is a permanent + * fact, so that verdict is cached to avoid re-probing the network on every extraction request. + */ + private synchronized void ensureSupportedTikaServerVersion() throws Exception { + if (tikaServerVersionVerified) { + return; + } + if (rejectedTikaServerVersionMessage != null) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, rejectedTikaServerVersionMessage); + } + HttpClient client = acquiredResourcesRef.get().client; + String versionUrl = baseUrl + "/version"; + ContentResponse response; + try { + response = + client + .newRequest(versionUrl) + .timeout(VERSION_CHECK_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS) + .send(); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Could not determine the TikaServer version at " + versionUrl + ": " + e.getMessage(), + e); + } + if (response.getStatus() != 200) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "TikaServer " + versionUrl + " returned status " + response.getStatus()); + } + String versionText = response.getContentAsString().trim(); + Matcher m = TIKASERVER_VERSION_PATTERN.matcher(versionText); + if (!m.find()) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Could not parse a TikaServer version from " + + versionUrl + + "'s response: '" + + versionText + + "'"); + } + int majorVersion = Integer.parseInt(m.group(1)); + if (majorVersion < MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION) { + rejectedTikaServerVersionMessage = + "TikaServer at " + + baseUrl + + " reports version '" + + versionText + + "', but Solr's 'tikaserver' extraction backend requires TikaServer " + + MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION + + ".x or newer (it relies on endpoints and metadata keys introduced in that" + + " version). Upgrade the TikaServer, or point tikaserver.url at a TikaServer " + + MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION + + ".x+ instance."; + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, rejectedTikaServerVersionMessage); + } + tikaServerVersionVerified = true; + } + /** Resolves the password to use for an encrypted document, or null if none applies. */ private String resolvePassword(ExtractionRequest request, ExtractionMetadata md) { if (request.resourcePassword == null && request.passwordsMap == null) { From 57b9a6ca7eefabb0942964ed50c77933581d1672 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Sep 2026 18:21:47 -0400 Subject: [PATCH 14/14] Test and fix for conccurrent extractions stepping on each other. --- .../TikaServerExtractionBackend.java | 14 ++++--- .../handler/extraction/TikaServerParser.java | 26 ++++++++---- .../TikaServerExtractionBackendTest.java | 42 +++++++++++++++++++ 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index 7af36a603260..6b6d6124dcb2 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -76,8 +76,7 @@ public class TikaServerExtractionBackend implements ExtractionBackend { /** Minimum TikaServer major version this backend supports (relies on Tika 4.x-only APIs). */ private static final int MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION = 4; - private static final Pattern TIKASERVER_VERSION_PATTERN = - Pattern.compile("Apache Tika (\\d+)\\."); + private static final Pattern TIKASERVER_VERSION_PATTERN = Pattern.compile("Apache Tika (\\d+)"); // Short-lived: /version is a trivial static-text endpoint, so this shouldn't use the full // extraction defaultTimeout (which can be minutes) and block concurrent requests behind it. @@ -89,8 +88,8 @@ public class TikaServerExtractionBackend implements ExtractionBackend { private final Duration defaultTimeout; private final TikaServerParser tikaServerResponseParser = new TikaServerParser(); private boolean tikaMetadataCompatibility; - private boolean tikaServerVersionVerified = false; - private String rejectedTikaServerVersionMessage; + private volatile boolean tikaServerVersionVerified = false; + private volatile String rejectedTikaServerVersionMessage; private HashMap initArgsMap = new HashMap<>(); private final long maxCharsLimit; @@ -398,8 +397,13 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t *

Connectivity/parsing failures are retried on the next call rather than cached, since * TikaServer may simply not be up yet. A definitively too-old version, however, is a permanent * fact, so that verdict is cached to avoid re-probing the network on every extraction request. + * + *

Deliberately not synchronized: while unverified, concurrent extraction requests may each + * probe {@code /version} independently rather than queue behind one shared lock. That's cheap and + * self-resolving once verified, and avoids turning a TikaServer outage into concurrent requests + * serialized behind a single blocking network call instead of each failing in parallel. */ - private synchronized void ensureSupportedTikaServerVersion() throws Exception { + private void ensureSupportedTikaServerVersion() throws Exception { if (tikaServerVersionVerified) { return; } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java index 746a91d7d5e1..96b817940149 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java @@ -38,20 +38,28 @@ public class TikaServerParser { // TikaServer 4.x's /rmeta content key (TIKA-4816). private static final String CONTENT_KEY = "tk:content"; - private final SAXParser saxParser; + // SAXParser isn't thread-safe, but a single TikaServerParser is shared across concurrent + // extraction requests (TikaServerExtractionBackend is held for the request handler's lifetime). + // The factory's config is fixed once at construction, so it's safe to share; each parse below + // mints its own SAXParser from it instead of reusing one. + private final SAXParserFactory saxParserFactory; public TikaServerParser() { - SAXParserFactory factory = SAXParserFactory.newInstance(); - factory.setNamespaceAware(true); + saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setNamespaceAware(true); try { - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + saxParserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); + saxParserFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + saxParserFactory.setFeature( + "http://apache.org/xml/features/nonvalidating/load-external-dtd", false); } catch (Throwable ignore) { // Some parsers may not support all features; ignore } + } + + private SAXParser newSaxParser() { try { - saxParser = factory.newSAXParser(); + return saxParserFactory.newSAXParser(); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } @@ -66,7 +74,7 @@ public void parseXml(InputStream inputStream, ContentHandler handler, Extraction DefaultHandler xmlHandler = new TikaXmlResponseSaxContentHandler(handler, metadata); try (Reader reader = new XmlSanitizingReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - saxParser.parse(new InputSource(reader), xmlHandler); + newSaxParser().parse(new InputSource(reader), xmlHandler); } } @@ -114,7 +122,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8)); try (Reader reader = new XmlSanitizingReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - saxParser.parse(new InputSource(reader), handler); + newSaxParser().parse(new InputSource(reader), handler); } } } diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index 2c59d69186a3..fbab3ba48697 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -21,9 +21,14 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.QuickPatchThreadsFilter; import org.apache.solr.SolrIgnoredThreadsFilter; import org.apache.solr.SolrTestCaseJ4; @@ -293,4 +298,41 @@ public void testConfigJsonRejectedForRecursive() throws Exception { } } } + + /** + * A single {@code TikaServerExtractionBackend} is constructed once by {@code + * ExtractingRequestHandler.inform()} and reused for every request it handles, including + * concurrently. {@code javax.xml.parsers.SAXParser} is not thread-safe, so parsing the response + * must not share one {@code SAXParser} instance across concurrent {@code extract()} calls. + */ + @Test + public void testConcurrentExtractDoesNotShareSaxParser() throws Exception { + int numThreads = 8; + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { + ExecutorService pool = Executors.newFixedThreadPool(numThreads); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < numThreads; i++) { + futures.add( + pool.submit( + () -> { + byte[] data = "Hello TestContainers".getBytes(StandardCharsets.UTF_8); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + return backend.extract(in, newRequest("test.txt", "text/plain", "text")); + } + })); + } + for (Future future : futures) { + ExtractionResult res = future.get(60, TimeUnit.SECONDS); + assertNotNull(res); + assertNotNull(res.getContent()); + assertTrue(res.getContent().contains("Hello TestContainers")); + } + } finally { + pool.shutdown(); + pool.awaitTermination(10, TimeUnit.SECONDS); + } + } + } }