|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, software |
| 13 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | + * See the License for the specific language governing permissions and |
| 16 | + * limitations under the License. |
| 17 | + */ |
| 18 | + |
| 19 | +package org.apache.hudi.table.action.ttl.strategy; |
| 20 | + |
| 21 | +import org.junit.jupiter.api.Test; |
| 22 | +import org.junit.jupiter.params.ParameterizedTest; |
| 23 | +import org.junit.jupiter.params.provider.CsvSource; |
| 24 | + |
| 25 | +import java.time.LocalDate; |
| 26 | +import java.time.LocalDateTime; |
| 27 | +import java.time.ZoneOffset; |
| 28 | +import java.time.format.DateTimeFormatter; |
| 29 | +import java.util.concurrent.TimeUnit; |
| 30 | + |
| 31 | +import static org.junit.jupiter.api.Assertions.assertEquals; |
| 32 | +import static org.junit.jupiter.api.Assertions.assertFalse; |
| 33 | +import static org.junit.jupiter.api.Assertions.assertThrows; |
| 34 | +import static org.junit.jupiter.api.Assertions.assertTrue; |
| 35 | + |
| 36 | +/** |
| 37 | + * Tests for {@link KeepByEventTimeStrategy}'s pure parsing logic. Exercises the static helpers |
| 38 | + * directly so we don't need to spin up a HoodieTable / write client. |
| 39 | + */ |
| 40 | +public class TestKeepByEventTimeStrategy { |
| 41 | + |
| 42 | + @Test |
| 43 | + public void segmentCountCountsSlashes() { |
| 44 | + assertEquals(1, KeepByEventTimeStrategy.segmentCount("yyyy-MM-dd")); |
| 45 | + assertEquals(1, KeepByEventTimeStrategy.segmentCount("yyyyMMdd")); |
| 46 | + assertEquals(2, KeepByEventTimeStrategy.segmentCount("yyyy-MM-dd/HH")); |
| 47 | + assertEquals(2, KeepByEventTimeStrategy.segmentCount("yyyyMMdd/HH")); |
| 48 | + } |
| 49 | + |
| 50 | + // path, format, startIdx, hiveStyle, expectedExpired |
| 51 | + // |
| 52 | + // Full matrix mirroring the class-level JavaDoc: 4 formats x 4 position shapes |
| 53 | + // (time only / prefix+time / time+suffix / prefix+time+suffix) x hive vs non-hive. |
| 54 | + // Cutoff is "2026-04-30 00:00 UTC" so any 2026-04-22 path expires; the two |
| 55 | + // "not expired" rows pin down the strict-less-than boundary. |
| 56 | + @ParameterizedTest |
| 57 | + @CsvSource({ |
| 58 | + // -------- Day, format=yyyy-MM-dd -------- |
| 59 | + // time only |
| 60 | + "'dt=2026-04-22', 'yyyy-MM-dd', 0, true, true", |
| 61 | + "'dt=2026-04-30', 'yyyy-MM-dd', 0, true, false", |
| 62 | + "'2026-04-22', 'yyyy-MM-dd', 0, false, true", |
| 63 | + "'2026-04-30', 'yyyy-MM-dd', 0, false, false", |
| 64 | + // prefix + time |
| 65 | + "'region=us/dt=2026-04-22', 'yyyy-MM-dd', 1, true, true", |
| 66 | + "'eventType=login/dt=2026-04-30', 'yyyy-MM-dd', 1, true, false", |
| 67 | + "'region=us/2026-04-22', 'yyyy-MM-dd', 1, false, true", |
| 68 | + // time + suffix |
| 69 | + "'dt=2026-04-22/source=app', 'yyyy-MM-dd', 0, true, true", |
| 70 | + "'2026-04-22/source=app', 'yyyy-MM-dd', 0, false, true", |
| 71 | + // prefix + time + suffix |
| 72 | + "'region=us/dt=2026-04-22/source=app', 'yyyy-MM-dd', 1, true, true", |
| 73 | + "'region=us/2026-04-22/source=app', 'yyyy-MM-dd', 1, false, true", |
| 74 | + |
| 75 | + // -------- Day, format=yyyyMMdd -------- |
| 76 | + // time only |
| 77 | + "'dt=20260422', 'yyyyMMdd', 0, true, true", |
| 78 | + "'dt=20260430', 'yyyyMMdd', 0, true, false", |
| 79 | + "'20260422', 'yyyyMMdd', 0, false, true", |
| 80 | + // prefix + time |
| 81 | + "'region=us/dt=20260422', 'yyyyMMdd', 1, true, true", |
| 82 | + "'region=us/20260422', 'yyyyMMdd', 1, false, true", |
| 83 | + // time + suffix |
| 84 | + "'dt=20260422/source=app', 'yyyyMMdd', 0, true, true", |
| 85 | + "'20260422/source=app', 'yyyyMMdd', 0, false, true", |
| 86 | + // prefix + time + suffix |
| 87 | + "'region=us/dt=20260422/source=app', 'yyyyMMdd', 1, true, true", |
| 88 | + "'region=us/20260422/source=app', 'yyyyMMdd', 1, false, true", |
| 89 | + |
| 90 | + // -------- Hour, format=yyyy-MM-dd/HH -------- |
| 91 | + // time only |
| 92 | + "'dt=2026-04-22/hh=06', 'yyyy-MM-dd/HH', 0, true, true", |
| 93 | + "'2026-04-22/06', 'yyyy-MM-dd/HH', 0, false, true", |
| 94 | + // prefix + time |
| 95 | + "'region=us/dt=2026-04-22/hh=06', 'yyyy-MM-dd/HH', 1, true, true", |
| 96 | + "'region=us/2026-04-22/06', 'yyyy-MM-dd/HH', 1, false, true", |
| 97 | + // time + suffix |
| 98 | + "'dt=2026-04-22/hh=06/source=app', 'yyyy-MM-dd/HH', 0, true, true", |
| 99 | + "'2026-04-22/06/source=app', 'yyyy-MM-dd/HH', 0, false, true", |
| 100 | + // prefix + time + suffix |
| 101 | + "'region=us/dt=2026-04-22/hh=06/source=app', 'yyyy-MM-dd/HH', 1, true, true", |
| 102 | + "'region=us/2026-04-22/06/source=app', 'yyyy-MM-dd/HH', 1, false, true", |
| 103 | + |
| 104 | + // -------- Hour, format=yyyyMMdd/HH -------- |
| 105 | + // time only |
| 106 | + "'dt=20260422/hh=06', 'yyyyMMdd/HH', 0, true, true", |
| 107 | + "'20260422/06', 'yyyyMMdd/HH', 0, false, true", |
| 108 | + // prefix + time |
| 109 | + "'eventType=login/dt=20260422/hh=06', 'yyyyMMdd/HH', 1, true, true", |
| 110 | + "'login/20260422/06', 'yyyyMMdd/HH', 1, false, true", |
| 111 | + // time + suffix |
| 112 | + "'dt=20260422/hh=06/eventType=login', 'yyyyMMdd/HH', 0, true, true", |
| 113 | + "'20260422/06/source=app', 'yyyyMMdd/HH', 0, false, true", |
| 114 | + // prefix + time + suffix |
| 115 | + "'region=us/dt=20260422/hh=06/source=app', 'yyyyMMdd/HH', 1, true, true", |
| 116 | + "'region=us/20260422/06/source=app', 'yyyyMMdd/HH', 1, false, true", |
| 117 | + }) |
| 118 | + public void parsesPartitionAndCompares(String path, |
| 119 | + String format, |
| 120 | + int startIdx, |
| 121 | + boolean hiveStyle, |
| 122 | + boolean expectedExpired) { |
| 123 | + // Cutoff is "2026-04-30 00:00 UTC" so 2026-04-22 is expired (strictly before) while 2026-04-30 is not. |
| 124 | + long cutoff = LocalDate.of(2026, 4, 30).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); |
| 125 | + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format).withZone(ZoneOffset.UTC); |
| 126 | + int segCount = KeepByEventTimeStrategy.segmentCount(format); |
| 127 | + |
| 128 | + boolean expired = KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 129 | + path, formatter, startIdx, segCount, cutoff, false, hiveStyle); |
| 130 | + |
| 131 | + assertEquals(expectedExpired, expired, "path=" + path + " format=" + format); |
| 132 | + } |
| 133 | + |
| 134 | + @Test |
| 135 | + public void parseFailureThrows() { |
| 136 | + // A partition whose time segment can't be parsed has no defined lifetime under event-time |
| 137 | + // semantics, so the strategy fails fast and asks the user to switch to commit-time TTL. |
| 138 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); |
| 139 | + assertThrows(IllegalArgumentException.class, () -> |
| 140 | + KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 141 | + "dt=not-a-date", f, 0, 1, Long.MAX_VALUE, false, true)); |
| 142 | + } |
| 143 | + |
| 144 | + @Test |
| 145 | + public void shorterPathThanFormatThrows() { |
| 146 | + // Partition layout is fixed per table — a mismatch is a configuration error, not a per-row skip. |
| 147 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd/HH").withZone(ZoneOffset.UTC); |
| 148 | + assertThrows(IllegalArgumentException.class, () -> |
| 149 | + KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 150 | + "dt=2026-04-22", f, 0, 2, Long.MAX_VALUE, false, true)); |
| 151 | + } |
| 152 | + |
| 153 | + @Test |
| 154 | + public void startIndexOutOfRangeThrows() { |
| 155 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); |
| 156 | + assertThrows(IllegalArgumentException.class, () -> |
| 157 | + KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 158 | + "dt=2026-04-22", f, 5, 1, Long.MAX_VALUE, false, true)); |
| 159 | + } |
| 160 | + |
| 161 | + @Test |
| 162 | + public void hiveStyleSegmentWithoutEqualsThrows() { |
| 163 | + // Table is hive-style but the segment lacks 'field=' prefix -> misconfiguration, fail fast. |
| 164 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); |
| 165 | + assertThrows(IllegalArgumentException.class, () -> |
| 166 | + KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 167 | + "2026-04-22", f, 0, 1, Long.MAX_VALUE, false, /*hiveStyle*/ true)); |
| 168 | + } |
| 169 | + |
| 170 | + @Test |
| 171 | + public void nonHiveStyleSegmentWithEqualsThrows() { |
| 172 | + // hiveStyle=false: segment is taken verbatim, so '=' becomes part of the time string and |
| 173 | + // fails parsing -> hard error, same as any other unparseable partition. |
| 174 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); |
| 175 | + assertThrows(IllegalArgumentException.class, () -> |
| 176 | + KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 177 | + "dt=2026-04-22", f, 0, 1, Long.MAX_VALUE, false, /*hiveStyle*/ false)); |
| 178 | + } |
| 179 | + |
| 180 | + @Test |
| 181 | + public void hiveDefaultPartitionSkippedWhenSwitchOff() { |
| 182 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); |
| 183 | + boolean expired = KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 184 | + "dt=__HIVE_DEFAULT_PARTITION__", f, 0, 1, Long.MAX_VALUE, false, true); |
| 185 | + assertFalse(expired); |
| 186 | + } |
| 187 | + |
| 188 | + @Test |
| 189 | + public void hiveDefaultPartitionDeletedWhenSwitchOn() { |
| 190 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); |
| 191 | + boolean expired = KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 192 | + "dt=__HIVE_DEFAULT_PARTITION__", f, 0, 1, /*cutoff*/ 0, /*deleteHiveDefault*/ true, /*hiveStyle*/ true); |
| 193 | + assertTrue(expired); |
| 194 | + } |
| 195 | + |
| 196 | + @Test |
| 197 | + public void wholeTimeBlockDefaultPartitionDeletedWhenSwitchOn() { |
| 198 | + // Multi-segment format where the entire time block is the default marker -> treated as |
| 199 | + // "the default partition" and respects the delete switch (just like the single-segment case). |
| 200 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd/HH").withZone(ZoneOffset.UTC); |
| 201 | + boolean expired = KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 202 | + "dt=__HIVE_DEFAULT_PARTITION__/hh=__HIVE_DEFAULT_PARTITION__", |
| 203 | + f, 0, 2, /*cutoff*/ 0, /*deleteHiveDefault*/ true, /*hiveStyle*/ true); |
| 204 | + assertTrue(expired); |
| 205 | + } |
| 206 | + |
| 207 | + @Test |
| 208 | + public void partialDefaultPartitionTimeBlockFollowsSwitch() { |
| 209 | + // dt=2026-06-28/hh=__HIVE_DEFAULT_PARTITION__: the date column was recorded but the hour |
| 210 | + // column was NULL, so the event time has no place on the hour axis. We treat this exactly |
| 211 | + // like a fully-null time block — the explicit delete switch decides. The cutoff is a recent |
| 212 | + // date precisely to nail down that this is NOT a "the date is fresh, keep it" path. |
| 213 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd/HH").withZone(ZoneOffset.UTC); |
| 214 | + long cutoff = LocalDate.of(2026, 4, 30).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); |
| 215 | + |
| 216 | + // switch on -> expire (even though dt looks fresh, event time is undefined) |
| 217 | + assertTrue(KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 218 | + "dt=2026-06-28/hh=__HIVE_DEFAULT_PARTITION__", |
| 219 | + f, 0, 2, cutoff, /*deleteHiveDefault*/ true, /*hiveStyle*/ true)); |
| 220 | + // switch off -> skip with WARN (user retains explicit control) |
| 221 | + assertFalse(KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 222 | + "dt=2026-06-28/hh=__HIVE_DEFAULT_PARTITION__", |
| 223 | + f, 0, 2, cutoff, /*deleteHiveDefault*/ false, /*hiveStyle*/ true)); |
| 224 | + } |
| 225 | + |
| 226 | + @Test |
| 227 | + public void resolveCutoffMillisInterpretsInstantInUtc() { |
| 228 | + // The instant string '20260430120000000' must be read as 2026-04-30T12:00:00Z regardless of the |
| 229 | + // JVM default zone. With ttl=0 the cutoff equals that exact instant; this pins down the contract |
| 230 | + // that resolveCutoffMillis and the partition formatter both speak UTC. |
| 231 | + long expected = LocalDateTime.of(2026, 4, 30, 12, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); |
| 232 | + assertEquals(expected, KeepByEventTimeStrategy.resolveCutoffMillis("20260430120000000", 0)); |
| 233 | + |
| 234 | + // ttl=1d shifts back exactly 24h, again with no dependence on the JVM zone. |
| 235 | + long oneDay = TimeUnit.DAYS.toMillis(1); |
| 236 | + assertEquals(expected - oneDay, |
| 237 | + KeepByEventTimeStrategy.resolveCutoffMillis("20260430120000000", oneDay)); |
| 238 | + } |
| 239 | + |
| 240 | + @Test |
| 241 | + public void hourBoundaryRespected() { |
| 242 | + // Cutoff is 2026-04-22 12:00 UTC. 11:00 expired, 12:00 not expired. |
| 243 | + long cutoff = LocalDateTime.of(2026, 4, 22, 12, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); |
| 244 | + DateTimeFormatter f = DateTimeFormatter.ofPattern("yyyy-MM-dd/HH").withZone(ZoneOffset.UTC); |
| 245 | + |
| 246 | + assertTrue(KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 247 | + "2026-04-22/11", f, 0, 2, cutoff, false, /*hiveStyle*/ false)); |
| 248 | + assertFalse(KeepByEventTimeStrategy.isPartitionExpiredByEventTime( |
| 249 | + "2026-04-22/12", f, 0, 2, cutoff, false, /*hiveStyle*/ false)); |
| 250 | + } |
| 251 | +} |
0 commit comments