From a421d8e8e521c6921757bfc6178a21c8d8c46d8e Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Fri, 30 Oct 2026 17:07:01 +0000 Subject: [PATCH] Round the next fetch dates without a Calendar DateUtils.round builds a GregorianCalendar for every URL, which was the largest share of the CPU used by AbstractStatusUpdaterBolt.execute(). Rounding to the second or minute is now done on the epoch time, half up, which gives the same dates as DateUtils. Rounding to the hour depends on the time zone and still goes through DateUtils. DateUtils can also move a date in the hour repeated when the clocks go back to its other occurrence, an hour away. The new rounding keeps it where it is. Signed-off-by: Julien Nioche Co-Authored-By: Claude Opus 5.5 --- .../AbstractStatusUpdaterBolt.java | 21 +++++- .../AbstractStatusUpdaterBoltTest.java | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java index 71bbeb736..832b9e011 100644 --- a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java @@ -259,9 +259,7 @@ public void execute(Tuple tuple) { metadata = mdTransfer.filter(metadata); // round next fetch date - unless it is never - if (nextFetch.isPresent()) { - nextFetch = Optional.of(DateUtils.round(nextFetch.get(), this.roundDateUnit)); - } + nextFetch = nextFetch.map(date -> roundDate(date, roundDateUnit)); // extensions of this class will handle the storage // on a per document basis @@ -273,6 +271,23 @@ public void execute(Tuple tuple) { } } + /** + * Rounds the date as {@link DateUtils#round(Date, int)} does. Seconds and minutes are rounded + * on the epoch time, half up, as building a Calendar for every URL is costly. Hours depend on + * the time zone, e.g. with an offset of 5:30, so they still go through DateUtils. + */ + static Date roundDate(Date date, int unit) { + long unitMsec; + if (unit == Calendar.SECOND) { + unitMsec = 1000L; + } else if (unit == Calendar.MINUTE) { + unitMsec = 60_000L; + } else { + return DateUtils.round(date, unit); + } + return new Date(Math.floorDiv(date.getTime() + unitMsec / 2, unitMsec) * unitMsec); + } + /** * Get the document id. * diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java index a3badc81d..ceb08b341 100644 --- a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java @@ -29,11 +29,15 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Random; import java.util.Set; +import java.util.TimeZone; +import org.apache.commons.lang3.time.DateUtils; import org.apache.storm.metric.api.MultiCountMetric; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -255,4 +259,69 @@ void cacheCountsArePerTimeBucketUnderV1() { assertEquals(Map.of("hits", 2L, "misses", 1L), metric.getValue().getValueAndReset()); assertEquals(Map.of("hits", 0L, "misses", 0L), metric.getValue().getValueAndReset()); } + + /** The rounding done without a Calendar must give the same dates as DateUtils. */ + @Test + void roundDateAsDateUtils() { + long[] times = { + 0L, + -1L, + -500L, + -501L, + 1_700_000_000_499L, + 1_700_000_000_500L, + 1_700_000_029_999L, + 1_700_000_030_000L, + 1_700_000_059_999L + }; + for (long time : times) { + assertRoundedAsDateUtils(new Date(time)); + } + Random random = new Random(0); + for (int i = 0; i < 100_000; i++) { + // between 2000 and 2100, some time zones had offsets with seconds in the past + long time = 946_684_800_000L + (long) (random.nextDouble() * 3_155_760_000_000L); + assertRoundedAsDateUtils(new Date(time)); + } + } + + private static void assertRoundedAsDateUtils(Date date) { + int[] units = {Calendar.SECOND, Calendar.MINUTE, Calendar.HOUR}; + long[] unitMsec = {1000L, 60_000L, 3_600_000L}; + for (int i = 0; i < units.length; i++) { + int unit = units[i]; + Date expected = DateUtils.round(date, unit); + // DateUtils can move a date of the hour repeated when the clocks go back to its + // other occurrence, see roundDateInRepeatedHour + if (Math.abs(expected.getTime() - date.getTime()) > unitMsec[i] / 2) { + continue; + } + assertEquals( + expected, + AbstractStatusUpdaterBolt.roundDate(date, unit), + "unit " + unit + " for " + date.getTime()); + } + } + + /** + * When the clocks go back, DateUtils rounds 02:22:47 CEST to 02:23 CET, an hour later, the date + * must stay in the same hour. + */ + @Test + void roundDateInRepeatedHour() { + TimeZone timeZone = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Paris")); + // 2060-10-31T02:22:47.299+02:00 + Date date = new Date(2_866_407_767_299L); + assertEquals( + new Date(2_866_407_780_000L), + AbstractStatusUpdaterBolt.roundDate(date, Calendar.MINUTE)); + assertEquals( + new Date(2_866_407_767_000L), + AbstractStatusUpdaterBolt.roundDate(date, Calendar.SECOND)); + } finally { + TimeZone.setDefault(timeZone); + } + } }