From ec60b17d9429a787b2d92a69374dc74617293297 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Thu, 3 Sep 2026 11:22:50 +0530 Subject: [PATCH] Upgrade to Tika 4.0.0 Closes #2063 Tika 4.0.0 requires several API and configuration changes: - core: TikaConfig has been removed - JSoupParserBolt now uses DefaultDetector for mime type detection; Detector.detect takes a TikaInputStream and a ParseContext; the CONTENT_TYPE and CONTENT_LENGTH constants moved from Metadata to HttpHeaders. - external/tika: ParserBolt loads configurations with TikaLoader (tika-serialization) since XML configurations are no longer supported, and the bundled tika-config.xml has been migrated to JSON. Config resources bundled in a jar are copied to a temporary file as TikaLoader reads configs from the filesystem only. - external/tika: tika-parsers-standard-package is now a POM aggregating the individual parser modules and is consumed as such. - the default value of parser.tika.config.file is now tika-config.json - THIRD-PARTY.txt regenerated --- THIRD-PARTY.txt | 80 ++++++++++--------- .../archetype-resources/crawler-conf.yaml | 2 +- .../stormcrawler/bolt/JSoupParserBolt.java | 17 ++-- docs/src/main/asciidoc/configuration.adoc | 2 +- .../archetype-resources/crawler-conf.yaml | 2 +- .../archetype-resources/crawler-conf.yaml | 2 +- external/tika/README.md | 2 +- external/tika/pom.xml | 17 ++++ .../apache/stormcrawler/tika/ParserBolt.java | 46 +++++++---- .../tika/src/main/resources/tika-config.json | 9 +++ .../tika/src/main/resources/tika-config.xml | 36 --------- pom.xml | 2 +- 12 files changed, 113 insertions(+), 104 deletions(-) create mode 100644 external/tika/src/main/resources/tika-config.json delete mode 100644 external/tika/src/main/resources/tika-config.xml diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt index cdbe76f03..79698f511 100644 --- a/THIRD-PARTY.txt +++ b/THIRD-PARTY.txt @@ -22,7 +22,7 @@ List of third-party dependencies grouped by their license type. * Apache Avro (org.apache.avro:avro:1.12.1 - https://avro.apache.org) * Apache Commons CLI (commons-cli:commons-cli:1.11.0 - https://commons.apache.org/proper/commons-cli/) * Apache Commons Codec (commons-codec:commons-codec:1.22.1 - https://commons.apache.org/proper/commons-codec/) - * Apache Commons Collections (org.apache.commons:commons-collections4:4.5.0 - https://commons.apache.org/proper/commons-collections/) + * Apache Commons Collections (org.apache.commons:commons-collections4:4.6.0 - https://commons.apache.org/proper/commons-collections/) * Apache Commons Compress (org.apache.commons:commons-compress:1.28.0 - https://commons.apache.org/proper/commons-compress/) * Apache Commons CSV (org.apache.commons:commons-csv:1.14.1 - https://commons.apache.org/proper/commons-csv/) * Apache Commons Exec (org.apache.commons:commons-exec:1.6.0 - https://commons.apache.org/proper/commons-exec/) @@ -49,7 +49,6 @@ List of third-party dependencies grouped by their license type. * Apache HttpCore NIO (org.apache.httpcomponents:httpcore-nio:4.4.16 - http://hc.apache.org/httpcomponents-core-ga) * Apache James :: Mime4j :: Core (org.apache.james:apache-mime4j-core:0.8.14 - http://james.apache.org/mime4j/apache-mime4j-core) * Apache James :: Mime4j :: DOM (org.apache.james:apache-mime4j-dom:0.8.14 - http://james.apache.org/mime4j/apache-mime4j-dom) - * Apache JempBox (org.apache.pdfbox:jempbox:1.8.17 - http://www.apache.org/pdfbox-parent/jempbox/) * Apache Log4j API (org.apache.logging.log4j:log4j-api:2.26.0 - https://logging.apache.org/log4j/2.x/) * Apache Log4j Core (org.apache.logging.log4j:log4j-core:2.26.0 - https://logging.apache.org/log4j/2.x/) * Apache Log4j JUL Adapter (org.apache.logging.log4j:log4j-jul:2.25.4 - https://logging.apache.org/log4j/2.x/) @@ -73,37 +72,40 @@ List of third-party dependencies grouped by their license type. * Apache POI (org.apache.poi:poi-scratchpad:5.5.1 - https://poi.apache.org/) * Apache POI - API based on OPC and OOXML schemas (org.apache.poi:poi-ooxml:5.5.1 - https://poi.apache.org/) * Apache POI - Common (org.apache.poi:poi:5.5.1 - https://poi.apache.org/) - * Apache POI - OOXML schemas (full) (org.apache.poi:poi-ooxml-full:5.5.1 - https://poi.apache.org/) * Apache Solr (module: api) (org.apache.solr:solr-api:10.0.0 - https://solr.apache.org/) * Apache Solr (module: solrj) (org.apache.solr:solr-solrj:10.0.0 - https://solr.apache.org/) * Apache Solr (module: solrj-jetty) (org.apache.solr:solr-solrj-jetty:10.0.0 - https://solr.apache.org/) * Apache Solr (module: solrj-zookeeper) (org.apache.solr:solr-solrj-zookeeper:10.0.0 - https://solr.apache.org/) - * Apache Tika Apple parser module (org.apache.tika:tika-parser-apple-module:3.3.2 - https://tika.apache.org/tika-parser-apple-module/) - * Apache Tika audiovideo parser module (org.apache.tika:tika-parser-audiovideo-module:3.3.2 - https://tika.apache.org/tika-parser-audiovideo-module/) - * Apache Tika cad parser module (org.apache.tika:tika-parser-cad-module:3.3.2 - https://tika.apache.org/tika-parser-cad-module/) - * Apache Tika code parser module (org.apache.tika:tika-parser-code-module:3.3.2 - https://tika.apache.org/tika-parser-code-module/) - * Apache Tika core (org.apache.tika:tika-core:3.3.2 - https://tika.apache.org/) - * Apache Tika crypto parser module (org.apache.tika:tika-parser-crypto-module:3.3.2 - https://tika.apache.org/tika-parser-crypto-module/) - * Apache Tika digest commons (org.apache.tika:tika-parser-digest-commons:3.3.2 - https://tika.apache.org/tika-parser-digest-commons/) - * Apache Tika font parser module (org.apache.tika:tika-parser-font-module:3.3.2 - https://tika.apache.org/tika-parser-font-module/) - * Apache Tika html parser module (org.apache.tika:tika-parser-html-module:3.3.2 - https://tika.apache.org/tika-parser-html-module/) - * Apache Tika image parser module (org.apache.tika:tika-parser-image-module:3.3.2 - https://tika.apache.org/tika-parser-image-module/) - * Apache Tika mail commons (org.apache.tika:tika-parser-mail-commons:3.3.2 - https://tika.apache.org/tika-parser-mail-commons/) - * Apache Tika mail parser module (org.apache.tika:tika-parser-mail-module:3.3.2 - https://tika.apache.org/tika-parser-mail-module/) - * Apache Tika Microsoft parser module (org.apache.tika:tika-parser-microsoft-module:3.3.2 - https://tika.apache.org/tika-parser-microsoft-module/) - * Apache Tika miscellaneous office format parser module (org.apache.tika:tika-parser-miscoffice-module:3.3.2 - https://tika.apache.org/tika-parser-miscoffice-module/) - * Apache Tika news parser module (org.apache.tika:tika-parser-news-module:3.3.2 - https://tika.apache.org/tika-parser-news-module/) - * Apache Tika OCR parser module (org.apache.tika:tika-parser-ocr-module:3.3.2 - https://tika.apache.org/tika-parser-ocr-module/) - * Apache Tika package parser module (org.apache.tika:tika-parser-pkg-module:3.3.2 - https://tika.apache.org/tika-parser-pkg-module/) - * Apache Tika PDF parser module (org.apache.tika:tika-parser-pdf-module:3.3.2 - https://tika.apache.org/tika-parser-pdf-module/) - * Apache Tika plugin for Ogg, Vorbis and FLAC (org.gagravarr:vorbis-java-tika:0.8 - https://github.com/Gagravarr/VorbisJava) - * Apache Tika standard parser package (org.apache.tika:tika-parsers-standard-package:3.3.2 - https://tika.apache.org/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/) - * Apache Tika text parser module (org.apache.tika:tika-parser-text-module:3.3.2 - https://tika.apache.org/tika-parser-text-module/) - * Apache Tika WARC parser module (org.apache.tika:tika-parser-webarchive-module:3.3.2 - https://tika.apache.org/tika-parser-webarchive-module/) - * Apache Tika XML parser module (org.apache.tika:tika-parser-xml-module:3.3.2 - https://tika.apache.org/tika-parser-xml-module/) - * Apache Tika XMP commons (org.apache.tika:tika-parser-xmp-commons:3.3.2 - https://tika.apache.org/tika-parser-xmp-commons/) - * Apache Tika ZIP commons (org.apache.tika:tika-parser-zip-commons:3.3.2 - https://tika.apache.org/tika-parser-zip-commons/) - * Apache XmpBox (org.apache.pdfbox:xmpbox:3.0.8 - https://www.apache.org/pdfbox-parent/xmpbox/) + * Apache Tika Apple parser module (org.apache.tika:tika-parser-apple-module:4.0.0 - https://tika.apache.org/tika-parser-apple-module/) + * Apache Tika audiovideo parser module (org.apache.tika:tika-parser-audiovideo-module:4.0.0 - https://tika.apache.org/tika-parser-audiovideo-module/) + * Apache Tika cad parser module (org.apache.tika:tika-parser-cad-module:4.0.0 - https://tika.apache.org/tika-parser-cad-module/) + * Apache Tika code parser module (org.apache.tika:tika-parser-code-module:4.0.0 - https://tika.apache.org/tika-parser-code-module/) + * Apache Tika core (org.apache.tika:tika-core:4.0.0 - https://tika.apache.org/) + * Apache Tika crypto parser module (org.apache.tika:tika-parser-crypto-module:4.0.0 - https://tika.apache.org/tika-parser-crypto-module/) + * Apache Tika data URI commons (org.apache.tika:tika-parser-datauri-commons:4.0.0 - https://tika.apache.org/tika-parser-datauri-commons/) + * Apache Tika digest commons (org.apache.tika:tika-parser-digest-commons:4.0.0 - https://tika.apache.org/tika-parser-digest-commons/) + * Apache Tika font parser module (org.apache.tika:tika-parser-font-module:4.0.0 - https://tika.apache.org/tika-parser-font-module/) + * Apache Tika HTML encoding detector (org.apache.tika:tika-encoding-detector-html:4.0.0 - https://tika.apache.org/tika-encoding-detectors/tika-encoding-detector-html/) + * Apache Tika html parser module (org.apache.tika:tika-parser-html-module:4.0.0 - https://tika.apache.org/tika-parser-html-module/) + * Apache Tika image parser module (org.apache.tika:tika-parser-image-module:4.0.0 - https://tika.apache.org/tika-parser-image-module/) + * Apache Tika mail commons (org.apache.tika:tika-parser-mail-commons:4.0.0 - https://tika.apache.org/tika-parser-mail-commons/) + * Apache Tika mail parser module (org.apache.tika:tika-parser-mail-module:4.0.0 - https://tika.apache.org/tika-parser-mail-module/) + * Apache Tika Microsoft parser module (org.apache.tika:tika-parser-microsoft-module:4.0.0 - https://tika.apache.org/tika-parser-microsoft-module/) + * Apache Tika miscellaneous office format parser module (org.apache.tika:tika-parser-miscoffice-module:4.0.0 - https://tika.apache.org/tika-parser-miscoffice-module/) + * Apache Tika ML charset encoding detector (org.apache.tika:tika-encoding-detector-mojibuster:4.0.0 - https://tika.apache.org/tika-encoding-detectors/tika-encoding-detector-mojibuster/) + * Apache Tika ML core (no Tika dependencies) (org.apache.tika:tika-ml-core:4.0.0 - https://tika.apache.org/tika-ml-core/) + * Apache Tika ML junk detector — runtime and training tools (org.apache.tika:tika-ml-junkdetect:4.0.0 - https://tika.apache.org/tika-ml-junkdetect/) + * Apache Tika news parser module (org.apache.tika:tika-parser-news-module:4.0.0 - https://tika.apache.org/tika-parser-news-module/) + * Apache Tika OCR parser module (org.apache.tika:tika-parser-ocr-module:4.0.0 - https://tika.apache.org/tika-parser-ocr-module/) + * Apache Tika package parser module (org.apache.tika:tika-parser-pkg-module:4.0.0 - https://tika.apache.org/tika-parser-pkg-module/) + * Apache Tika PDF parser module (org.apache.tika:tika-parser-pdf-module:4.0.0 - https://tika.apache.org/tika-parser-pdf-module/) + * Apache Tika serialization (org.apache.tika:tika-serialization:4.0.0 - https://tika.apache.org) + * Apache Tika standard parser package (org.apache.tika:tika-parsers-standard-package:4.0.0 - https://tika.apache.org/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/) + * Apache Tika text parser module (org.apache.tika:tika-parser-text-module:4.0.0 - https://tika.apache.org/tika-parser-text-module/) + * Apache Tika WARC parser module (org.apache.tika:tika-parser-webarchive-module:4.0.0 - https://tika.apache.org/tika-parser-webarchive-module/) + * Apache Tika XML parser module (org.apache.tika:tika-parser-xml-module:4.0.0 - https://tika.apache.org/tika-parser-xml-module/) + * Apache Tika XMP commons (org.apache.tika:tika-parser-xmp-commons:4.0.0 - https://tika.apache.org/tika-parser-xmp-commons/) + * Apache Tika ZIP commons (org.apache.tika:tika-parser-zip-commons:4.0.0 - https://tika.apache.org/tika-parser-zip-commons/) * Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.9.4 - http://zookeeper.apache.org/zookeeper-jute) * Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.9.4 - http://zookeeper.apache.org/zookeeper) * AWS Event Stream (software.amazon.eventstream:eventstream:1.0.1 - https://github.com/awslabs/aws-eventstream-java) @@ -141,7 +143,7 @@ List of third-party dependencies grouped by their license type. * AWS Java SDK :: Utilities (software.amazon.awssdk:utils:2.54.6 - https://aws.amazon.com/sdkforjava/utils) * AWS Java SDK :: Utils Lite (software.amazon.awssdk:utils-lite:2.54.6 - https://aws.amazon.com/sdkforjava) * Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.2.4 - https://github.com/ben-manes/caffeine) - * com.drewnoakes:metadata-extractor (com.drewnoakes:metadata-extractor:2.20.0 - https://drewnoakes.com/code/exif/) + * com.drewnoakes:metadata-extractor (com.drewnoakes:metadata-extractor:2.21.0 - https://drewnoakes.com/code/exif/) * compiler (com.github.spullara.mustache.java:compiler:0.9.14 - http://github.com/spullara/mustache.java) * Crawler-commons (com.github.crawler-commons:crawler-commons:1.6 - https://github.com/crawler-commons/crawler-commons) * Curator Client (org.apache.curator:curator-client:5.9.0 - https://curator.apache.org/curator-client) @@ -305,7 +307,13 @@ List of third-party dependencies grouped by their license type. * Bouncy Castle JavaMail Jakarta S/MIME APIs (org.bouncycastle:bcjmail-jdk18on:1.85 - https://www.bouncycastle.org/download/bouncy-castle-java/) * Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.84 - https://www.bouncycastle.org/download/bouncy-castle-java/) * Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.84 - https://www.bouncycastle.org/download/bouncy-castle-java/) - * Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.85 - https://www.bouncycastle.org/download/bouncy-castle-java/) + * Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.85.2 - https://www.bouncycastle.org/download/bouncy-castle-java/) + + BSD-2-Clause + + * commonmark-java core (org.commonmark:commonmark:0.30.0 - https://github.com/commonmark/commonmark-java/commonmark) + * commonmark-java extension for strikethrough (org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0 - https://github.com/commonmark/commonmark-java/commonmark-ext-gfm-strikethrough) + * commonmark-java extension for tables (org.commonmark:commonmark-ext-gfm-tables:0.30.0 - https://github.com/commonmark/commonmark-java/commonmark-ext-gfm-tables) BSD-2-Clause, Public Domain, per Creative Commons CC0 @@ -331,7 +339,7 @@ List of third-party dependencies grouped by their license type. CDDL, v1.0, LGPL, v2.1 or later - * JHighlight (org.codelibs:jhighlight:1.1.1 - https://github.com/codelibs/jhighlight) + * JHighlight (org.codelibs:jhighlight:2.0.0 - https://github.com/codelibs/jhighlight) CDDL/GPLv2+CE @@ -375,10 +383,6 @@ List of third-party dependencies grouped by their license type. * JSON-B API (jakarta.json.bind:jakarta.json.bind-api:2.0.0 - https://eclipse-ee4j.github.io/jsonb-api) * JSON-P Default Provider (org.glassfish:jakarta.json:2.0.0 - https://github.com/eclipse-ee4j/jsonp) - GENERAL PUBLIC LICENSE, version 3 (GPL-3.0), GNU LESSER GENERAL PUBLIC LICENSE, version 3 (LGPL-3.0), Mozilla Public License Version 1.1 - - * juniversalchardet (com.github.albfernandez:juniversalchardet:2.5.0 - https://github.com/albfernandez/juniversalchardet) - MIT-0 * reactive-streams (org.reactivestreams:reactive-streams:1.0.4 - http://www.reactive-streams.org/) @@ -386,7 +390,7 @@ List of third-party dependencies grouped by their license type. MIT License * Animal Sniffer Annotations (org.codehaus.mojo:animal-sniffer-annotations:1.24 - https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations) - * dd-plist (com.googlecode.plist:dd-plist:1.29 - http://www.github.com/3breadt/dd-plist) + * dd-plist (com.googlecode.plist:dd-plist:1.30 - http://www.github.com/3breadt/dd-plist) * JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.4 - http://jopt-simple.github.io/jopt-simple) * jsoup Java HTML Parser (org.jsoup:jsoup:1.23.2 - https://jsoup.org/) * JTokkit (com.knuddels:jtokkit:1.1.0 - https://github.com/knuddelsgmbh/jtokkit) @@ -415,4 +419,4 @@ List of third-party dependencies grouped by their license type. UnRar License - * Java Unrar (com.github.junrar:junrar:7.6.0 - https://github.com/junrar/junrar) + * Java Unrar (com.github.junrar:junrar:8.1.0 - https://github.com/junrar/junrar) diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index bc57e3714..fa27ac3b5 100644 --- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -129,7 +129,7 @@ config: - application/.*pdf.* # Tika parser configuration file - parse.tika.config.file: "tika-config.xml" + parse.tika.config.file: "tika-config.json" # custom fetch interval to be used when a document has the key/value in its metadata # and has been fetched successfully (value in minutes) diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index 577d828fe..c1ae43869 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -19,9 +19,7 @@ import static org.apache.stormcrawler.Constants.StatusStreamName; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; import java.net.URL; @@ -62,10 +60,12 @@ import org.apache.stormcrawler.util.RefreshTag; import org.apache.stormcrawler.util.RobotsTags; import org.apache.stormcrawler.util.URLUtil; -import org.apache.tika.config.TikaConfig; +import org.apache.tika.detect.DefaultDetector; import org.apache.tika.detect.Detector; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; import org.jsoup.nodes.Element; import org.jsoup.parser.Parser; import org.jsoup.select.Elements; @@ -89,7 +89,7 @@ public class JSoupParserBolt extends StatusEmitterBolt { private JSoupFilter jsoupFilters = null; - private final Detector detector = TikaConfig.getDefaultConfig().getDetector(); + private final Detector detector = new DefaultDetector(); private boolean detectMimeType = true; @@ -547,17 +547,16 @@ public String guessMimeType(String url, String httpContentType, byte[] content) if (StringUtils.isNotBlank(httpContentType)) { // pass content type from server as a clue - metadata.set(org.apache.tika.metadata.Metadata.CONTENT_TYPE, httpContentType); + metadata.set(HttpHeaders.CONTENT_TYPE, httpContentType); } // use full URL as a clue metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, url); - metadata.set( - org.apache.tika.metadata.Metadata.CONTENT_LENGTH, Integer.toString(content.length)); + metadata.set(HttpHeaders.CONTENT_LENGTH, Integer.toString(content.length)); - try (InputStream stream = new ByteArrayInputStream(content)) { - MediaType mt = detector.detect(stream, metadata); + try (TikaInputStream stream = TikaInputStream.get(content)) { + MediaType mt = detector.detect(stream, metadata, new ParseContext()); return mt.toString(); } catch (IOException e) { throw new IllegalStateException("Unexpected IOException", e); diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 047a86ea4..ddd362c3a 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -530,7 +530,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/tika[tika | key | default value | description | parser.mimetype.whitelist | - | Regex list of allowed MIME types. If set, only matching types are parsed by the Tika ParserBolt. -| parser.tika.config.file | tika-config.xml | Path to the Tika configuration XML file. +| parser.tika.config.file | tika-config.json | Path to the Tika configuration file (JSON format since Tika 4). |=== NOTE: When using the Tika `ParserBolt` alongside `JSoupParserBolt`, set `jsoup.treat.non.html.as.error` to `false` so that non-HTML content is passed through to the Tika parser rather than being treated as an error. diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..c5358b920 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -123,7 +123,7 @@ config: - application/.*pdf.* # Tika parser configuration file - parse.tika.config.file: "tika-config.xml" + parse.tika.config.file: "tika-config.json" # custom fetch interval to be used when a document has the key/value in its metadata # and has been fetched successfully (value in minutes) diff --git a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..c5358b920 100644 --- a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -123,7 +123,7 @@ config: - application/.*pdf.* # Tika parser configuration file - parse.tika.config.file: "tika-config.xml" + parse.tika.config.file: "tika-config.json" # custom fetch interval to be used when a document has the key/value in its metadata # and has been fetched successfully (value in minutes) diff --git a/external/tika/README.md b/external/tika/README.md index 99d3ca6ff..2467587c7 100644 --- a/external/tika/README.md +++ b/external/tika/README.md @@ -31,4 +31,4 @@ The next step is to use a [RedirectionBolt](https://github.com/apache/stormcrawl ## Configure Tika -The Tika parser bolt loads a Tika configuration file from the Java classpath. The default file name (path) is `tika-config.xml` and can be changed by the configuration `parser.tika.config.file`. See [configuring Tika](https://tika.apache.org/2.1.0/configuring.html) and the default configuration file [tika-config.xml](./src/main/resources/tika-config.xml). +The Tika parser bolt loads a Tika configuration file from the Java classpath. The default file name (path) is `tika-config.json` and can be changed by the configuration `parser.tika.config.file`. Since Tika 4, configurations are written in JSON instead of XML - see [configuring Tika](https://tika.apache.org/4.0.0/config.html) and the default configuration file [tika-config.json](./src/main/resources/tika-config.json). diff --git a/external/tika/pom.xml b/external/tika/pom.xml index f1eba0922..c9d8fe4a7 100644 --- a/external/tika/pom.xml +++ b/external/tika/pom.xml @@ -47,10 +47,27 @@ under the License. + + + org.apache.tika + tika-core + ${tika.version} + + + + org.apache.tika + tika-serialization + ${tika.version} + + + org.apache.tika tika-parsers-standard-package ${tika.version} + pom asm diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 46179df97..a738a1ea6 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -19,10 +19,15 @@ import static org.apache.stormcrawler.Constants.StatusStreamName; -import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; @@ -56,7 +61,8 @@ import org.apache.stormcrawler.util.MetadataTransfer; import org.apache.stormcrawler.util.URLUtil; import org.apache.tika.Tika; -import org.apache.tika.config.TikaConfig; +import org.apache.tika.config.loader.TikaLoader; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; @@ -194,14 +200,13 @@ public void execute(Tuple tuple) { long start = System.currentTimeMillis(); - ByteArrayInputStream bais = new ByteArrayInputStream(content); org.apache.tika.metadata.Metadata md = new org.apache.tika.metadata.Metadata(); // provide the mime-type as a clue for guessing String httpCT = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); if (StringUtils.isNotBlank(httpCT)) { // pass content type from server as a clue - md.set(org.apache.tika.metadata.Metadata.CONTENT_TYPE, httpCT); + md.set(HttpHeaders.CONTENT_TYPE, httpCT); } // as well as the filename @@ -242,18 +247,12 @@ public void execute(Tuple tuple) { // parse String text; - try { - tika.getParser().parse(bais, teeHandler, md, parseContext); + try (TikaInputStream tis = TikaInputStream.get(content)) { + tika.getParser().parse(tis, teeHandler, md, parseContext); text = textHandler.toString(); } catch (Throwable e) { handleException(url, e, metadata, tuple, "parse error"); return; - } finally { - try { - bais.close(); - } catch (IOException e) { - LOG.error("Exception while closing stream", e); - } } // add parse md to metadata @@ -317,7 +316,7 @@ public void execute(Tuple tuple) { private Tika instantiateTika(Map conf) { Tika tika = null; String tikaConfigFile = - ConfUtils.getString(conf, "parser.tika.config.file", "tika-config.xml"); + ConfUtils.getString(conf, "parser.tika.config.file", "tika-config.json"); long start = System.currentTimeMillis(); URL tikaConfigUrl = getClass().getClassLoader().getResource(tikaConfigFile); if (tikaConfigUrl == null) { @@ -325,8 +324,9 @@ private Tika instantiateTika(Map conf) { } else { LOG.info("Instantiating Tika using custom configuration {}", tikaConfigUrl); try { - TikaConfig tikaConfig = new TikaConfig(tikaConfigUrl, getClass().getClassLoader()); - tika = new Tika(tikaConfig); + TikaLoader tikaLoader = + TikaLoader.load(urlToPath(tikaConfigUrl), getClass().getClassLoader()); + tika = new Tika(tikaLoader.loadDetectors(), tikaLoader.loadAutoDetectParser()); } catch (Exception e) { LOG.error( "Failed to instantiate Tika using custom configuration {}", @@ -343,6 +343,22 @@ private Tika instantiateTika(Map conf) { return tika; } + /** + * Returns the configuration file as a path, copying it to a temporary file first if it is + * bundled inside a jar, as TikaLoader can only read configs from the filesystem. + */ + private Path urlToPath(URL configUrl) throws IOException, URISyntaxException { + if ("file".equals(configUrl.getProtocol())) { + return Paths.get(configUrl.toURI()); + } + Path tmp = Files.createTempFile("tika-config", ".json"); + try (InputStream is = configUrl.openStream()) { + Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING); + } + tmp.toFile().deleteOnExit(); + return tmp; + } + private void handleException( String url, Throwable e, Metadata metadata, Tuple tuple, String errorType) { // real exception? diff --git a/external/tika/src/main/resources/tika-config.json b/external/tika/src/main/resources/tika-config.json new file mode 100644 index 000000000..a17d7cab1 --- /dev/null +++ b/external/tika/src/main/resources/tika-config.json @@ -0,0 +1,9 @@ +{ + "parsers": [ + { + "default-parser": { + "exclude": ["tesseract-ocr-parser"] + } + } + ] +} diff --git a/external/tika/src/main/resources/tika-config.xml b/external/tika/src/main/resources/tika-config.xml deleted file mode 100644 index c55bc3626..000000000 --- a/external/tika/src/main/resources/tika-config.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/pom.xml b/pom.xml index cfd6686b2..00183f7fd 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ under the License. 2.22 2.22.0 - 3.3.2 + 4.0.0 5.23.0 2.0.18 26.1.0