initArgsMap = new HashMap<>();
private final long maxCharsLimit;
@@ -173,26 +199,84 @@ public void extractWithSaxHandler(
* request.tikaserverRecursive
*/
InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) throws Exception {
- String url = baseUrl + (request.tikaServerRecursive ? "/rmeta" : "/tika");
+ ensureSupportedTikaServerVersion();
- HttpClient client = acquiredResourcesRef.get().client;
+ ExtractionMetadata md = buildMetadataFromRequest(request);
+ String pwd = resolvePassword(request, md);
+ String configJson = resolveConfigJson(request, pwd);
- Request req = client.newRequest(url).method("PUT");
+ HttpClient client = acquiredResourcesRef.get().client;
Duration effectiveTimeout =
(request.tikaServerTimeoutSeconds != null && request.tikaServerTimeoutSeconds > 0)
? Duration.ofSeconds(request.tikaServerTimeoutSeconds)
: defaultTimeout;
- req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS);
- // Also set idle timeout in case of heavy server side work like OCR
- req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS);
-
- // Headers
- String accept = (request.tikaServerRecursive ? "application/json" : "text/xml");
- req.headers(h -> h.add("Accept", accept));
String contentType = (request.streamType != null) ? request.streamType : request.contentType;
- if (contentType != null) {
- req.headers(h -> h.add("Content-Type", contentType));
+
+ String url;
+ Request req;
+ if (configJson != null && request.tikaServerRecursive) {
+ // Tracked upstream: https://issues.apache.org/jira/browse/TIKA-4881
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Per-request TikaServer config (password or "
+ + ExtractingParams.TIKASERVER_CONFIG_JSON
+ + ") is not supported together with "
+ + ExtractingParams.TIKASERVER_RECURSIVE
+ + "=true: TikaServer 4.x has no XML-output variant of /rmeta/config"
+ + " (see https://issues.apache.org/jira/browse/TIKA-4881).");
}
+ if (configJson != null) {
+ // TikaServer accepts per-request parser config (including passwords) as a JSON "config"
+ // part on a multipart request, and requires allowPerRequestConfig=true on the server. Only
+ // non-recursive extraction is handled here, since TikaServer has no XML content-handler
+ // variant of /rmeta/config (checked above).
+ url = baseUrl + "/tika/config/xml";
+ req = client.newRequest(url).method("POST");
+ req.headers(h -> h.add("Accept", "text/xml"));
+
+ HttpFields.Mutable fileFields = HttpFields.build();
+ if (contentType != null) {
+ fileFields.add(HttpHeader.CONTENT_TYPE, contentType);
+ }
+ try (MultiPartRequestContent multiPart = new MultiPartRequestContent()) {
+ multiPart.addPart(
+ new MultiPart.ContentSourcePart(
+ "file",
+ request.resourceName,
+ fileFields,
+ new InputStreamRequestContent(inputStream)));
+ multiPart.addPart(
+ new MultiPart.ContentSourcePart(
+ "config",
+ null,
+ HttpFields.build().add(HttpHeader.CONTENT_TYPE, "application/json"),
+ new StringRequestContent(configJson)));
+ req.body(multiPart);
+ }
+ } else {
+ // TikaServer's /tika and /rmeta endpoints return Markdown by default (TIKA-4663); Solr's
+ // SAX-based content handling requires XHTML/XML, hence the /xml path variants.
+ url = baseUrl + (request.tikaServerRecursive ? "/rmeta/xml" : "/tika/xml");
+ req = client.newRequest(url).method("PUT");
+ String accept = (request.tikaServerRecursive ? "application/json" : "text/xml");
+ req.headers(h -> h.add("Accept", accept));
+ if (contentType != null) {
+ req.headers(h -> h.add("Content-Type", contentType));
+ }
+ if (request.resourceName != null) {
+ req.headers(
+ h ->
+ h.add(
+ "Content-Disposition",
+ "attachment; filename=\"" + request.resourceName + "\""));
+ }
+ if (contentType != null) {
+ req.body(new InputStreamRequestContent(contentType, inputStream));
+ } else {
+ req.body(new InputStreamRequestContent(inputStream));
+ }
+ }
+
if (!request.tikaServerRequestHeaders.isEmpty()) {
req.headers(
h ->
@@ -202,32 +286,9 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t
}));
}
- ExtractionMetadata md = buildMetadataFromRequest(request);
- if (request.resourcePassword != null || request.passwordsMap != null) {
- RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider();
- if (request.resourcePassword != null) {
- passwordProvider.setExplicitPassword(request.resourcePassword);
- }
- if (request.passwordsMap != null) {
- passwordProvider.setPasswordMap(request.passwordsMap);
- }
- String pwd = passwordProvider.getPassword(md);
- if (pwd != null) {
- req.headers(h -> h.add("Password", pwd)); // Tika Server expects this header if provided
- }
- }
- if (request.resourceName != null) {
- req.headers(
- h ->
- h.add(
- "Content-Disposition", "attachment; filename=\"" + request.resourceName + "\""));
- }
-
- if (contentType != null) {
- req.body(new InputStreamRequestContent(contentType, inputStream));
- } else {
- req.body(new InputStreamRequestContent(inputStream));
- }
+ req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS);
+ // Also set idle timeout in case of heavy server side work like OCR
+ req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS);
InputStreamResponseListener listener = new InputStreamResponseListener();
req.send(listener);
@@ -273,7 +334,44 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t
}
int code = response.getStatus();
- if (code < 200 || code >= 300) {
+ InputStream responseStream = listener.getInputStream();
+ // Tika 4.x's raw /tika* endpoints (non-recursive) return 422 whenever a container-level
+ // exception occurred during parsing -- including a non-aborting one like a writeLimit
+ // truncation -- but the body still carries whatever content was successfully extracted
+ // (there's no envelope to carry the exception itself on these endpoints; use /rmeta for
+ // that). A request that extracted nothing at all (e.g. a wrong password) also gets 422, but
+ // with an empty body -- that's always a hard failure, regardless of ignoreTikaException.
+ // Peek the first byte to tell the two apart.
+ if (code == 422 && !request.tikaServerRecursive) {
+ PushbackInputStream peekable = new PushbackInputStream(responseStream, 1);
+ int firstByte = peekable.read();
+ if (firstByte == -1) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "TikaServer "
+ + url
+ + " returned status 422 (Unprocessable Entity) with no content -- the document"
+ + " could not be parsed at all (check the password, if one was required).");
+ }
+ peekable.unread(firstByte);
+ if (!request.ignoreTikaException) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "TikaServer "
+ + url
+ + " returned status 422 (Unprocessable Entity): a container-level exception"
+ + " occurred during parsing (e.g. a writeLimit truncation). Partial content was"
+ + " extracted but is being discarded because ignoreTikaException=false; set"
+ + " ignoreTikaException=true to index the partial content instead, or use"
+ + " tikaserver.recursive=true against /rmeta for the exception detail.");
+ }
+ log.warn(
+ "TikaServer {} returned 422 (a container-level exception occurred during parsing); "
+ + "using the partial content it still returned because ignoreTikaException=true. "
+ + "Use tikaserver.recursive=true against /rmeta for the exception detail.",
+ url);
+ responseStream = peekable;
+ } else if (code < 200 || code >= 300) {
SolrException.ErrorCode errorCode = SolrException.ErrorCode.getErrorCode(code);
String reason = response.getReason();
String msg =
@@ -285,11 +383,130 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t
throw new SolrException(errorCode, msg);
}
- InputStream responseStream = listener.getInputStream();
// Bound the amount of data we read from Tika Server to avoid excessive memory/CPU usage
return new LimitingInputStream(responseStream, maxCharsLimit);
}
+ /**
+ * Verifies, once per backend instance, that the configured TikaServer reports a major version of
+ * at least {@link #MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION}. This backend relies on endpoints
+ * (e.g. {@code /tika/xml}, {@code /tika/config/xml}) and metadata keys (e.g. {@code tk:content})
+ * that only exist on TikaServer 4.x and newer; an older server would otherwise fail with
+ * confusing 404s or missing-metadata errors instead of a clear diagnostic.
+ *
+ * Connectivity/parsing failures are retried on the next call rather than cached, since
+ * TikaServer may simply not be up yet. A definitively too-old version, however, is a permanent
+ * fact, so that verdict is cached to avoid re-probing the network on every extraction request.
+ *
+ *
Deliberately not synchronized: while unverified, concurrent extraction requests may each
+ * probe {@code /version} independently rather than queue behind one shared lock. That's cheap and
+ * self-resolving once verified, and avoids turning a TikaServer outage into concurrent requests
+ * serialized behind a single blocking network call instead of each failing in parallel.
+ */
+ private void ensureSupportedTikaServerVersion() throws Exception {
+ if (tikaServerVersionVerified) {
+ return;
+ }
+ if (rejectedTikaServerVersionMessage != null) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, rejectedTikaServerVersionMessage);
+ }
+ HttpClient client = acquiredResourcesRef.get().client;
+ String versionUrl = baseUrl + "/version";
+ ContentResponse response;
+ try {
+ response =
+ client
+ .newRequest(versionUrl)
+ .timeout(VERSION_CHECK_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)
+ .send();
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Could not determine the TikaServer version at " + versionUrl + ": " + e.getMessage(),
+ e);
+ }
+ if (response.getStatus() != 200) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "TikaServer " + versionUrl + " returned status " + response.getStatus());
+ }
+ String versionText = response.getContentAsString().trim();
+ Matcher m = TIKASERVER_VERSION_PATTERN.matcher(versionText);
+ if (!m.find()) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Could not parse a TikaServer version from "
+ + versionUrl
+ + "'s response: '"
+ + versionText
+ + "'");
+ }
+ int majorVersion = Integer.parseInt(m.group(1));
+ if (majorVersion < MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION) {
+ rejectedTikaServerVersionMessage =
+ "TikaServer at "
+ + baseUrl
+ + " reports version '"
+ + versionText
+ + "', but Solr's 'tikaserver' extraction backend requires TikaServer "
+ + MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION
+ + ".x or newer (it relies on endpoints and metadata keys introduced in that"
+ + " version). Upgrade the TikaServer, or point tikaserver.url at a TikaServer "
+ + MIN_SUPPORTED_TIKASERVER_MAJOR_VERSION
+ + ".x+ instance.";
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, rejectedTikaServerVersionMessage);
+ }
+ tikaServerVersionVerified = true;
+ }
+
+ /** Resolves the password to use for an encrypted document, or null if none applies. */
+ private String resolvePassword(ExtractionRequest request, ExtractionMetadata md) {
+ if (request.resourcePassword == null && request.passwordsMap == null) {
+ return null;
+ }
+ RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider();
+ if (request.resourcePassword != null) {
+ passwordProvider.setExplicitPassword(request.resourcePassword);
+ }
+ if (request.passwordsMap != null) {
+ passwordProvider.setPasswordMap(request.passwordsMap);
+ }
+ return passwordProvider.getPassword(md);
+ }
+
+ /**
+ * Builds the per-request TikaServer JSON "config" payload, merging any resolved password with any
+ * caller-supplied {@link ExtractingParams#TIKASERVER_CONFIG_JSON}. Returns null if neither
+ * applies, meaning no per-request config is needed.
+ */
+ @SuppressWarnings("unchecked")
+ private String resolveConfigJson(ExtractionRequest request, String pwd) {
+ Map config = new LinkedHashMap<>();
+ if (request.tikaServerConfigJson != null && !request.tikaServerConfigJson.isBlank()) {
+ Object parsed;
+ try {
+ parsed = Utils.fromJSONString(request.tikaServerConfigJson);
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid JSON in " + ExtractingParams.TIKASERVER_CONFIG_JSON + ": " + e.getMessage(),
+ e);
+ }
+ if (!(parsed instanceof Map)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ ExtractingParams.TIKASERVER_CONFIG_JSON + " must be a JSON object");
+ }
+ config.putAll((Map) parsed);
+ }
+ if (pwd != null && !config.containsKey("simple-password-provider")) {
+ config.put("simple-password-provider", Map.of("password", pwd));
+ }
+ return config.isEmpty() ? null : Utils.toJSONString(config);
+ }
+
private static class LimitingInputStream extends InputStream {
private final InputStream in;
private final long max;
diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java
index e34514040b6e..96b817940149 100644
--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java
+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java
@@ -35,20 +35,31 @@
import org.xml.sax.helpers.DefaultHandler;
public class TikaServerParser {
- private final SAXParser saxParser;
+ // TikaServer 4.x's /rmeta content key (TIKA-4816).
+ private static final String CONTENT_KEY = "tk:content";
+
+ // SAXParser isn't thread-safe, but a single TikaServerParser is shared across concurrent
+ // extraction requests (TikaServerExtractionBackend is held for the request handler's lifetime).
+ // The factory's config is fixed once at construction, so it's safe to share; each parse below
+ // mints its own SAXParser from it instead of reusing one.
+ private final SAXParserFactory saxParserFactory;
public TikaServerParser() {
- SAXParserFactory factory = SAXParserFactory.newInstance();
- factory.setNamespaceAware(true);
+ saxParserFactory = SAXParserFactory.newInstance();
+ saxParserFactory.setNamespaceAware(true);
try {
- factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
- factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
- factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+ saxParserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+ saxParserFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+ saxParserFactory.setFeature(
+ "http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
} catch (Throwable ignore) {
// Some parsers may not support all features; ignore
}
+ }
+
+ private SAXParser newSaxParser() {
try {
- saxParser = factory.newSAXParser();
+ return saxParserFactory.newSAXParser();
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
@@ -63,7 +74,7 @@ public void parseXml(InputStream inputStream, ContentHandler handler, Extraction
DefaultHandler xmlHandler = new TikaXmlResponseSaxContentHandler(handler, metadata);
try (Reader reader =
new XmlSanitizingReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
- saxParser.parse(new InputSource(reader), xmlHandler);
+ newSaxParser().parse(new InputSource(reader), xmlHandler);
}
}
@@ -91,7 +102,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe
for (Object k : map.keySet()) {
String key = String.valueOf(k);
Object val = map.get(k);
- if ("X-TIKA:content".equalsIgnoreCase(key)) {
+ if (CONTENT_KEY.equalsIgnoreCase(key)) {
// handled below
continue;
}
@@ -103,7 +114,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe
md.add(key, String.valueOf(val));
}
}
- Object content = map.get("X-TIKA:content");
+ Object content = map.get(CONTENT_KEY);
if (content != null) {
String xhtml = String.valueOf(content);
if (!xhtml.isEmpty() && handler != null) {
@@ -111,7 +122,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe
new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8));
try (Reader reader =
new XmlSanitizingReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
- saxParser.parse(new InputSource(reader), handler);
+ newSaxParser().parse(new InputSource(reader), handler);
}
}
}
diff --git a/solr/modules/extraction/src/test-files/extraction/tika-server-config.json b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json
new file mode 100644
index 000000000000..63a1e4fcbd77
--- /dev/null
+++ b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json
@@ -0,0 +1,5 @@
+{
+ "server": {
+ "allowPerRequestConfig": true
+ }
+}
diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java
index c9c872bc99de..b6553cf9f4bb 100644
--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java
+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java
@@ -440,17 +440,20 @@ public void testLiterals() throws Exception {
"one",
"literal.extractionLiteral",
"two",
- "fmap.X-Parsed-By",
+ // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816)
+ "fmap.tk:parsed-by",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By",
+ "fmap.tk:parsed-by-full-set",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By-Full-Set",
+ "fmap.tk:content-handler-type",
"ignored_parser",
- "fmap.X-TIKA:content_handler",
+ "fmap.tk:parse-time-millis",
"ignored_parser",
- "fmap.X-TIKA:parse_time_millis",
+ "fmap.tk:embedded-depth",
"ignored_parser",
- "fmap.X-TIKA:embedded_depth",
+ "fmap.tk:content-type-parser-override",
+ "ignored_parser",
+ "fmap.tk:content-type-magic-detected",
"ignored_parser",
"fmap.Last-Modified",
"extractedDate");
@@ -480,17 +483,20 @@ public void testLiterals() throws Exception {
"extractedLanguage",
"literal.extractionLiteral",
"one",
- "fmap.X-Parsed-By",
+ // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816)
+ "fmap.tk:parsed-by",
+ "ignored_parser",
+ "fmap.tk:parsed-by-full-set",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By",
+ "fmap.tk:content-handler-type",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By-Full-Set",
+ "fmap.tk:parse-time-millis",
"ignored_parser",
- "fmap.X-TIKA:content_handler",
+ "fmap.tk:embedded-depth",
"ignored_parser",
- "fmap.X-TIKA:parse_time_millis",
+ "fmap.tk:content-type-parser-override",
"ignored_parser",
- "fmap.X-TIKA:embedded_depth",
+ "fmap.tk:content-type-magic-detected",
"ignored_parser",
"fmap.Last-Modified",
"extractedDate");
@@ -596,21 +602,26 @@ public void testPlainTextSpecifyingMimeType() throws Exception {
"one",
"fmap.language",
"extractedLanguage",
- "fmap.X-Parsed-By",
+ // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816)
+ "fmap.tk:parsed-by",
+ "ignored_parser",
+ "fmap.tk:detected-encoding",
+ "ignored_parser",
+ "fmap.tk:encoding-detector",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By",
+ "fmap.tk:encoding-detection-trace",
"ignored_parser",
- "fmap.X-TIKA:detectedEncoding",
+ "fmap.tk:parsed-by-full-set",
"ignored_parser",
- "fmap.X-TIKA:encodingDetector",
+ "fmap.tk:content-handler-type",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By-Full-Set",
+ "fmap.tk:parse-time-millis",
"ignored_parser",
- "fmap.X-TIKA:content_handler",
+ "fmap.tk:embedded-depth",
"ignored_parser",
- "fmap.X-TIKA:parse_time_millis",
+ "fmap.tk:content-type-parser-override",
"ignored_parser",
- "fmap.X-TIKA:embedded_depth",
+ "fmap.tk:content-type-magic-detected",
"ignored_parser",
"fmap.content",
"extractedContent",
@@ -644,21 +655,28 @@ public void testPlainTextSpecifyingResourceName() throws Exception {
"one",
"fmap.language",
"extractedLanguage",
- "fmap.X-Parsed-By",
+ // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816)
+ "fmap.tk:parsed-by",
+ "ignored_parser",
+ "fmap.tk:detected-encoding",
+ "ignored_parser",
+ "fmap.tk:encoding-detector",
+ "ignored_parser",
+ "fmap.tk:encoding-detection-trace",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By",
+ "fmap.tk:parsed-by-full-set",
"ignored_parser",
- "fmap.X-TIKA:detectedEncoding",
+ "fmap.tk:content-handler-type",
"ignored_parser",
- "fmap.X-TIKA:encodingDetector",
+ "fmap.tk:parse-time-millis",
"ignored_parser",
- "fmap.X-TIKA:Parsed-By-Full-Set",
+ "fmap.tk:embedded-depth",
"ignored_parser",
- "fmap.X-TIKA:content_handler",
+ "fmap.tk:content-type-parser-override",
"ignored_parser",
- "fmap.X-TIKA:parse_time_millis",
+ "fmap.tk:content-type-magic-detected",
"ignored_parser",
- "fmap.X-TIKA:embedded_depth",
+ "fmap.tk:resource-name",
"ignored_parser",
"fmap.content",
"extractedContent",
diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java
index f32dbb5289c8..7f31e8fcd6d6 100644
--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java
+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java
@@ -31,7 +31,8 @@ public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHan
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ClassRule
- public static final TikaServerContainerRule tikaContainer = new TikaServerContainerRule();
+ public static final TikaServerContainerRule tikaContainer =
+ new TikaServerContainerRule(getFile("extraction/tika-server-config.json"));
@BeforeClass
public static void beforeClassTika() throws Exception {
diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java
index 7a6ae3937998..368bdb0c75e1 100644
--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java
+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler.extraction;
import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
import org.junit.Assume;
import org.junit.rules.ExternalResource;
import org.slf4j.Logger;
@@ -24,6 +25,7 @@
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.MountableFile;
/**
* JUnit rule that manages a single Apache Tika Server Testcontainer. Declare as a
@@ -38,11 +40,21 @@ public class TikaServerContainerRule extends ExternalResource {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String TIKA_DOCKER_IMAGE = "apache/tika:3.2.3.0-full";
+ public static final String TIKA_DOCKER_IMAGE = "apache/tika:4.0.0-full";
+ private final Path serverConfigFile;
private GenericContainer tika;
private String baseUrl;
+ /**
+ * @param serverConfigFile optional Tika Server JSON config file to mount and start the container
+ * with (via {@code -c}), e.g. to set {@code allowPerRequestConfig: true}. Null for the
+ * container's default configuration.
+ */
+ public TikaServerContainerRule(Path serverConfigFile) {
+ this.serverConfigFile = serverConfigFile;
+ }
+
@Override
@SuppressWarnings("resource")
protected void before() {
@@ -56,6 +68,10 @@ protected void before() {
new GenericContainer<>(TIKA_DOCKER_IMAGE)
.withExposedPorts(9998)
.waitingFor(Wait.forListeningPort());
+ if (serverConfigFile != null) {
+ tika.withCopyFileToContainer(MountableFile.forHostPath(serverConfigFile), "/tika-config.json")
+ .withCommand("-c", "/tika-config.json");
+ }
tika.start();
baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998);
}
diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java
index 326ab818596c..fbab3ba48697 100644
--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java
+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java
@@ -21,9 +21,14 @@
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.tests.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -58,7 +63,8 @@ public boolean reject(Thread t) {
}
@ClassRule
- public static final TikaServerContainerRule tikaContainer = new TikaServerContainerRule();
+ public static final TikaServerContainerRule tikaContainer =
+ new TikaServerContainerRule(getFile("extraction/tika-server-config.json"));
private static ExtractionRequest newRequest(
String resourceName,
@@ -124,14 +130,13 @@ public void testPdfWithImageRecursive() throws Exception {
try (TikaServerExtractionBackend backend =
new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) {
byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf"));
- // Enable recursive extraction and set header to extract images from PDF
+ // Explicit inline-image extraction options can't be requested here: TikaServer's per-request
+ // config only supports the non-recursive /tika/config/xml endpoint (see resolveConfigJson's
+ // javadoc), since there is no XML-output variant of /rmeta/config for recursive requests. The
+ // PDF's embedded image is still OCR'd into the main document's content by default, just not
+ // exposed as a separate embedded resource entry.
ExtractionRequest request =
- newRequest(
- "pdf-with-image.pdf",
- "application/pdf",
- "xml",
- true,
- Map.of("X-Tika-PDFextractInlineImages", "true"));
+ newRequest("pdf-with-image.pdf", "application/pdf", "xml", true, Map.of());
try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
ToXMLContentHandler xmlHandler = new ToXMLContentHandler();
ExtractionMetadata md = backend.buildMetadataFromRequest(request);
@@ -139,9 +144,8 @@ public void testPdfWithImageRecursive() throws Exception {
String c = xmlHandler.toString();
assertNotNull(c);
assertTrue(c.contains("Puppet Apply"));
- assertTrue(c.contains("embedded:image0.jpg"));
- assertEquals(
- "org.apache.tika.parser.DefaultParser", md.getFirst("X-TIKA:Parsed-By-Full-Set"));
+ // TikaServer 4.x uses a single lowercase tk: prefix for its metadata keys (TIKA-4816)
+ assertEquals("org.apache.tika.parser.DefaultParser", md.getFirst("tk:parsed-by-full-set"));
}
}
}
@@ -195,4 +199,140 @@ public void testMaxCharsLimitEnforcedWithSaxHandler() throws Exception {
}
}
}
+
+ private static ExtractionRequest newRequestWithConfig(
+ String resourceName, String contentType, String extractFormat, String configJson) {
+ return ExtractionRequest.builder()
+ .streamType(contentType)
+ .resourceName(resourceName)
+ .contentType(contentType)
+ .streamName(resourceName)
+ .extractFormat(extractFormat)
+ .tikaServerConfigJson(configJson)
+ .build();
+ }
+
+ @Test
+ public void testConfigJsonDisablesOcr() throws Exception {
+ try (TikaServerExtractionBackend backend =
+ new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) {
+ byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf"));
+ // With no config, the PDF's embedded image gets OCR'd and "Puppet Apply" (from the image)
+ // appears in the extracted content. Disabling OCR via tikaserver.config should suppress it.
+ ExtractionRequest request =
+ newRequestWithConfig(
+ "pdf-with-image.pdf",
+ "application/pdf",
+ "xml",
+ "{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}");
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+ ExtractionResult res = backend.extract(in, request);
+ assertNotNull(res.getContent());
+ assertFalse(
+ "Expected tikaserver.config's NO_OCR strategy to suppress the OCR'd image text",
+ res.getContent().contains("Puppet Apply"));
+ }
+ }
+ }
+
+ @Test
+ public void testConfigJsonMergesWithPassword() throws Exception {
+ try (TikaServerExtractionBackend backend =
+ new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) {
+ byte[] data = Files.readAllBytes(getFile("extraction/encrypted-password-is-solrRules.pdf"));
+ ExtractionRequest request =
+ ExtractionRequest.builder()
+ .streamType("application/pdf")
+ .resourceName("encrypted-password-is-solrRules.pdf")
+ .contentType("application/pdf")
+ .streamName("encrypted-password-is-solrRules.pdf")
+ .extractFormat("xml")
+ .resourcePassword("solrRules")
+ .tikaServerConfigJson("{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}")
+ .build();
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+ ExtractionResult res = backend.extract(in, request);
+ assertNotNull(res);
+ assertTrue(
+ "Expected the password-unlocked content to still be present alongside the merged"
+ + " tikaserver.config",
+ res.getContent().contains("This is a test of PDF and Word extraction"));
+ }
+ }
+ }
+
+ @Test
+ public void testInvalidConfigJsonRejected() throws Exception {
+ try (TikaServerExtractionBackend backend =
+ new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) {
+ byte[] data = "hello".getBytes(StandardCharsets.UTF_8);
+ ExtractionRequest request =
+ newRequestWithConfig("test.txt", "text/plain", "xml", "not valid json");
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+ SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request));
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
+ assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_CONFIG_JSON));
+ }
+ }
+ }
+
+ @Test
+ public void testConfigJsonRejectedForRecursive() throws Exception {
+ try (TikaServerExtractionBackend backend =
+ new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) {
+ byte[] data = "hello".getBytes(StandardCharsets.UTF_8);
+ ExtractionRequest request =
+ ExtractionRequest.builder()
+ .streamType("text/plain")
+ .resourceName("test.txt")
+ .contentType("text/plain")
+ .streamName("test.txt")
+ .extractFormat("xml")
+ .tikaServerRecursive(true)
+ .tikaServerConfigJson("{\"parse-context\":{}}")
+ .build();
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+ SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request));
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
+ assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_RECURSIVE));
+ }
+ }
+ }
+
+ /**
+ * A single {@code TikaServerExtractionBackend} is constructed once by {@code
+ * ExtractingRequestHandler.inform()} and reused for every request it handles, including
+ * concurrently. {@code javax.xml.parsers.SAXParser} is not thread-safe, so parsing the response
+ * must not share one {@code SAXParser} instance across concurrent {@code extract()} calls.
+ */
+ @Test
+ public void testConcurrentExtractDoesNotShareSaxParser() throws Exception {
+ int numThreads = 8;
+ try (TikaServerExtractionBackend backend =
+ new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) {
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+ try {
+ List> futures = new ArrayList<>();
+ for (int i = 0; i < numThreads; i++) {
+ futures.add(
+ pool.submit(
+ () -> {
+ byte[] data = "Hello TestContainers".getBytes(StandardCharsets.UTF_8);
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+ return backend.extract(in, newRequest("test.txt", "text/plain", "text"));
+ }
+ }));
+ }
+ for (Future future : futures) {
+ ExtractionResult res = future.get(60, TimeUnit.SECONDS);
+ assertNotNull(res);
+ assertNotNull(res.getContent());
+ assertTrue(res.getContent().contains("Hello TestContainers"));
+ }
+ } finally {
+ pool.shutdown();
+ pool.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ }
+ }
}
diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java
new file mode 100644
index 000000000000..950e910249d0
--- /dev/null
+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerVersionCheckTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.extraction;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Verifies that {@link TikaServerExtractionBackend} rejects a TikaServer older than {@code 4.x}
+ * with a clear diagnostic, rather than failing later with confusing 404s or missing metadata.
+ *
+ * Uses a tiny in-process {@link HttpServer} stub for the {@code /version} endpoint instead of a
+ * real Tika Server, since that's all this check depends on.
+ */
+public class TikaServerVersionCheckTest extends SolrTestCaseJ4 {
+
+ private HttpServer server;
+
+ @After
+ public void stopServer() {
+ if (server != null) {
+ server.stop(0);
+ server = null;
+ }
+ }
+
+ private String startServerWithVersion(String versionResponseBody) throws Exception {
+ server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+ server.createContext("/version", exchange -> serveText(exchange, versionResponseBody));
+ server.createContext(
+ "/tika/xml",
+ exchange ->
+ serveText(
+ exchange,
+ ""
+ + "
hello world"));
+ server.start();
+ return "http://localhost:" + server.getAddress().getPort();
+ }
+
+ private static void serveText(HttpExchange exchange, String body) throws IOException {
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(200, bytes.length);
+ try (var os = exchange.getResponseBody()) {
+ os.write(bytes);
+ }
+ }
+
+ private static ExtractionRequest newRequest() {
+ return ExtractionRequest.builder()
+ .streamType("text/plain")
+ .resourceName("test.txt")
+ .contentType("text/plain")
+ .streamName("test.txt")
+ .extractFormat("xml")
+ .build();
+ }
+
+ @Test
+ public void testRejectsPre4xTikaServer() throws Exception {
+ String baseUrl = startServerWithVersion("Apache Tika 3.2.3");
+ try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) {
+ ExtractionRequest request = newRequest();
+ try (ByteArrayInputStream in =
+ new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8))) {
+ SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request));
+ assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, e.code());
+ assertTrue(
+ "Expected message to name the offending version and the minimum required, but was: "
+ + e.getMessage(),
+ e.getMessage().contains("Apache Tika 3.") && e.getMessage().contains("requires"));
+ }
+ }
+ }
+
+ @Test
+ public void testAcceptsSupportedTikaServerVersion() throws Exception {
+ String baseUrl = startServerWithVersion("Apache Tika 4.0.0");
+ try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) {
+ ExtractionRequest request = newRequest();
+ try (ByteArrayInputStream in =
+ new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8))) {
+ ExtractionResult result = backend.extract(in, request);
+ assertNotNull(result);
+ assertTrue(result.getContent().contains("hello world"));
+ }
+ }
+ }
+}
diff --git a/solr/packaging/test/test_extraction.bats b/solr/packaging/test/test_extraction.bats
index 4b8e62ae59cc..0823377452c6 100644
--- a/solr/packaging/test/test_extraction.bats
+++ b/solr/packaging/test/test_extraction.bats
@@ -40,7 +40,7 @@ apply_extract_handler() {
setup_file() {
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
export TIKA_PORT=$((SOLR_PORT+5))
- docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:3.2.3.0-full >/dev/null 2>&1 || true
+ docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:4.0.0-full >/dev/null 2>&1 || true
echo "Waiting for Tika Server to be ready on port ${TIKA_PORT}" >&3
if ! wait_for 120 3 curl -s -f "http://localhost:${TIKA_PORT}/tika" -o /dev/null; then
export DOCKER_UNAVAILABLE=1
diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc
index db34f483b69f..d58038d509e3 100644
--- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc
+++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc
@@ -39,7 +39,7 @@ Indexing binary files (PDF, DOCX, PPTX, etc.) with the Post Tool requires the So
.Start a Tika Server quickly using Docker (exposes port 9998 on localhost)
[,bash]
----
-docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full
+docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full
----
=== Start Solr with the extraction module enabled
diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc
index bc9e1905ec61..b415f259cb1e 100644
--- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc
+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc
@@ -78,7 +78,7 @@ The quickest way to run Tika Server for development is using Docker. The example
[,bash]
----
-docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full
+docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full
----
NOTE: If Solr runs in Docker too, ensure both containers share a network and use the Tika container name as the host in `tikaserver.url`.
@@ -92,7 +92,7 @@ First we start a tika server on port 9998, using Docker.
[source,bash]
----
# Start Tika Server in the background
-docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full
+docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full
# To stop the server when done, run `docker stop tika`
----
@@ -384,6 +384,18 @@ Example: `passwordsFile=/path/to/passwords.txt`
// +
// Only applicable for `tikaserver` backend. Can only be set in `solrconfig.xml`, not per request.
+`tikaserver.config`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: none
+|===
++
+A raw JSON object sent as the per-request parser configuration for Tika Server (e.g., `{"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}`).
+See <> below for details and an important security note: this requires `allowPerRequestConfig=true` on the Tika Server, which is off by default.
++
+Example: `tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}`
+
`tikaserver.maxChars`::
+
[%autowidth,frame=none]
@@ -540,9 +552,33 @@ So you can use the other URPs without worrying about unexpected field additions.
=== Parser-Specific Properties
-Parser-specific properties for Tika must be configured directly on your Tika Server instance. Consult the https://tika.apache.org/[Apache Tika documentation] for details.
+Server-wide parser properties (things that should apply to every request) must be configured directly on your Tika Server instance, via its own JSON configuration file. Consult the https://tika.apache.org/[Apache Tika documentation] for details.
+
+For a single request, you can instead pass parser-specific options through Solr using the `tikaserver.config` parameter, whose value is a raw JSON object matching Tika Server's per-request configuration format.
+For example, to disable OCR for one request:
+
+[,console]
+----
+$ bin/solr post -c gettingstarted example/exampledocs/solr-word.pdf --params 'literal.id=doc1&tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}'
+----
+
+[IMPORTANT]
+====
+`tikaserver.config` requires your Tika Server to be started with `allowPerRequestConfig: true` in its own JSON configuration (under the `server` section).
+This is off by default, and Tika Server logs a warning when it is enabled, because it lets any client that can reach `/update/extract` inject arbitrary parser configuration, including options that spawn external processes such as OCR.
+Only enable it if you need per-request configuration, and treat access to your Solr instance's extraction endpoint accordingly.
+
+[source,json]
+----
+{
+ "server": {
+ "allowPerRequestConfig": true
+ }
+}
+----
+====
-NOTE: In earlier versions of Solr Cell you could supply Tika configuration directly to Solr. This is no longer possible.
+`tikaserver.config` is combined with any password resolved from `resource.password` or `passwordsFile` (see <>) into a single request to Tika Server, and is only supported for non-recursive extraction (`tikaserver.recursive=false`, the default); Tika Server has no way to accept per-request configuration for recursive extraction while also returning the XHTML content Solr Cell needs (tracked upstream as https://issues.apache.org/jira/browse/TIKA-4881[TIKA-4881]).
=== Indexing Encrypted Documents