diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java index 71bbeb736..832b9e011 100644 --- a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java @@ -259,9 +259,7 @@ public void execute(Tuple tuple) { metadata = mdTransfer.filter(metadata); // round next fetch date - unless it is never - if (nextFetch.isPresent()) { - nextFetch = Optional.of(DateUtils.round(nextFetch.get(), this.roundDateUnit)); - } + nextFetch = nextFetch.map(date -> roundDate(date, roundDateUnit)); // extensions of this class will handle the storage // on a per document basis @@ -273,6 +271,23 @@ public void execute(Tuple tuple) { } } + /** + * Rounds the date as {@link DateUtils#round(Date, int)} does. Seconds and minutes are rounded + * on the epoch time, half up, as building a Calendar for every URL is costly. Hours depend on + * the time zone, e.g. with an offset of 5:30, so they still go through DateUtils. + */ + static Date roundDate(Date date, int unit) { + long unitMsec; + if (unit == Calendar.SECOND) { + unitMsec = 1000L; + } else if (unit == Calendar.MINUTE) { + unitMsec = 60_000L; + } else { + return DateUtils.round(date, unit); + } + return new Date(Math.floorDiv(date.getTime() + unitMsec / 2, unitMsec) * unitMsec); + } + /** * Get the document id. * diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java index a3badc81d..ceb08b341 100644 --- a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java @@ -29,11 +29,15 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Random; import java.util.Set; +import java.util.TimeZone; +import org.apache.commons.lang3.time.DateUtils; import org.apache.storm.metric.api.MultiCountMetric; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -255,4 +259,69 @@ void cacheCountsArePerTimeBucketUnderV1() { assertEquals(Map.of("hits", 2L, "misses", 1L), metric.getValue().getValueAndReset()); assertEquals(Map.of("hits", 0L, "misses", 0L), metric.getValue().getValueAndReset()); } + + /** The rounding done without a Calendar must give the same dates as DateUtils. */ + @Test + void roundDateAsDateUtils() { + long[] times = { + 0L, + -1L, + -500L, + -501L, + 1_700_000_000_499L, + 1_700_000_000_500L, + 1_700_000_029_999L, + 1_700_000_030_000L, + 1_700_000_059_999L + }; + for (long time : times) { + assertRoundedAsDateUtils(new Date(time)); + } + Random random = new Random(0); + for (int i = 0; i < 100_000; i++) { + // between 2000 and 2100, some time zones had offsets with seconds in the past + long time = 946_684_800_000L + (long) (random.nextDouble() * 3_155_760_000_000L); + assertRoundedAsDateUtils(new Date(time)); + } + } + + private static void assertRoundedAsDateUtils(Date date) { + int[] units = {Calendar.SECOND, Calendar.MINUTE, Calendar.HOUR}; + long[] unitMsec = {1000L, 60_000L, 3_600_000L}; + for (int i = 0; i < units.length; i++) { + int unit = units[i]; + Date expected = DateUtils.round(date, unit); + // DateUtils can move a date of the hour repeated when the clocks go back to its + // other occurrence, see roundDateInRepeatedHour + if (Math.abs(expected.getTime() - date.getTime()) > unitMsec[i] / 2) { + continue; + } + assertEquals( + expected, + AbstractStatusUpdaterBolt.roundDate(date, unit), + "unit " + unit + " for " + date.getTime()); + } + } + + /** + * When the clocks go back, DateUtils rounds 02:22:47 CEST to 02:23 CET, an hour later, the date + * must stay in the same hour. + */ + @Test + void roundDateInRepeatedHour() { + TimeZone timeZone = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Paris")); + // 2060-10-31T02:22:47.299+02:00 + Date date = new Date(2_866_407_767_299L); + assertEquals( + new Date(2_866_407_780_000L), + AbstractStatusUpdaterBolt.roundDate(date, Calendar.MINUTE)); + assertEquals( + new Date(2_866_407_767_000L), + AbstractStatusUpdaterBolt.roundDate(date, Calendar.SECOND)); + } finally { + TimeZone.setDefault(timeZone); + } + } }