Skip to content

Commit

Permalink
Update Date32 and DateTime64 range for ClickHouse, remove datetime64 …
Browse files Browse the repository at this point in the history 
…limits check layer (#1518)

* Update Date32 and DateTime64 range for ClickHouse, remove datetime64 limits check layer #1103 #1386 #1441

* fix of byte arrays for updated dates in BinaryStreamUtilsTest
  • Loading branch information
@TimonSP
TimonSP authored Jan 11, 2024
1 parent 2e42faa commit 52634fb
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 40 deletions.
17 changes: 6 additions & 11 deletions clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryDataProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,17 +289,12 @@ public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream in
@Override
public void serialize(ClickHouseValue value, ClickHouseOutputStream output) throws IOException {
LocalDateTime dt = value.asDateTime(scale);
long v = ClickHouseChecker.between(
ClickHouseValues.UTC_ZONE.equals(zoneId) ? dt.toEpochSecond(ZoneOffset.UTC)
: dt.atZone(zoneId).toEpochSecond(),
ClickHouseValues.TYPE_DATE_TIME, BinaryStreamUtils.DATETIME64_MIN,
BinaryStreamUtils.DATETIME64_MAX);
if (ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9) > 0) {
v *= BASES[scale];
int nanoSeconds = dt.getNano();
if (nanoSeconds > 0L) {
v += nanoSeconds / BASES[9 - scale];
}
long v = ClickHouseValues.UTC_ZONE.equals(zoneId) ? dt.toEpochSecond(ZoneOffset.UTC)
: dt.atZone(zoneId).toEpochSecond();
v *= BASES[scale];
int nanoSeconds = dt.getNano();
if (nanoSeconds > 0L) {
v += nanoSeconds / BASES[9 - scale];
}

BinaryStreamUtils.writeInt64(output, v);
Expand Down
14 changes: 9 additions & 5 deletions clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryStreamUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ public final class BinaryStreamUtils {
(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF,
(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF });

public static final int DATE32_MAX = (int) LocalDate.of(2283, 11, 11).toEpochDay();
public static final int DATE32_MIN = (int) LocalDate.of(1925, 1, 1).toEpochDay();
public static final int DATE32_MAX = (int) LocalDate.of(2299, 12, 31).toEpochDay();
public static final int DATE32_MIN = (int) LocalDate.of(1900, 1, 1).toEpochDay();

public static final BigDecimal DECIMAL32_MAX = new BigDecimal("1000000000");
public static final BigDecimal DECIMAL32_MIN = new BigDecimal("-1000000000");
Expand All @@ -69,9 +69,11 @@ public final class BinaryStreamUtils {
public static final BigDecimal DECIMAL256_MIN = new BigDecimal(
"-10000000000000000000000000000000000000000000000000000000000000000000000000000");

public static final long DATETIME64_MAX = LocalDateTime.of(LocalDate.of(2283, 11, 11), LocalTime.MAX)
public static final long DATETIME64_MAX = LocalDateTime.of(LocalDate.of(2299, 12, 31), LocalTime.MAX)
.toEpochSecond(ZoneOffset.UTC);
public static final long DATETIME64_MIN = LocalDateTime.of(LocalDate.of(1925, 1, 1), LocalTime.MIN)
public static final long DATETIME64_9_MAX = LocalDateTime.of(2262, 4, 11, 23, 47, 16, 0)
.toEpochSecond(ZoneOffset.UTC);
public static final long DATETIME64_MIN = LocalDateTime.of(LocalDate.of(1900, 1, 1), LocalTime.MIN)
.toEpochSecond(ZoneOffset.UTC);

public static final long MILLIS_IN_DAY = TimeUnit.DAYS.toMillis(1);
Expand Down Expand Up @@ -1529,7 +1531,9 @@ public static void writeDateTime64(OutputStream output, LocalDateTime value, int
long v = ClickHouseChecker.between(
tz == null || tz.equals(ClickHouseValues.UTC_TIMEZONE) ? value.toEpochSecond(ZoneOffset.UTC)
: value.atZone(tz.toZoneId()).toEpochSecond(),
ClickHouseValues.TYPE_DATE_TIME, DATETIME64_MIN, DATETIME64_MAX);
ClickHouseValues.TYPE_DATE_TIME,
DATETIME64_MIN,
scale == 9 ? DATETIME64_9_MAX : DATETIME64_MAX);
if (ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9) > 0) {
v *= BASES[scale];
int nanoSeconds = value.getNano();
Expand Down
48 changes: 24 additions & 24 deletions clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -909,10 +909,10 @@ public void testReadDate32() throws IOException {
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0x17, 0x61, 0, 0), null),
LocalDate.of(2038, 1, 19));

Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCC, 0xBF, 0xFF, 0xFF), null),
LocalDate.of(1925, 1, 1));
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCB, 0xBF, 1, 0), null),
LocalDate.of(2283, 11, 11));
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0x21, 0x9C, 0xFF, 0xFF), null),
LocalDate.of(1900, 1, 1));
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xD1, 0xD6, 1, 0), null),
LocalDate.of(2299, 12, 31));
}

@Test(dataProvider = "timeZoneProvider", groups = { "unit" })
Expand All @@ -935,12 +935,12 @@ public void testReadDate32WithTimeZone(String timeZoneId) throws IOException {
.withZoneSameInstant(tz.toZoneId())
.toLocalDate());

Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCC, 0xBF, 0xFF, 0xFF), tz),
LocalDate.of(1925, 1, 1).atStartOfDay(ClickHouseValues.SYS_ZONE)
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0x21, 0x9C, 0xFF, 0xFF), tz),
LocalDate.of(1900, 1, 1).atStartOfDay(ClickHouseValues.SYS_ZONE)
.withZoneSameInstant(tz.toZoneId())
.toLocalDate());
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCB, 0xBF, 1, 0), tz),
LocalDate.of(2283, 11, 11).atStartOfDay(ClickHouseValues.SYS_ZONE)
Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xD1, 0xD6, 1, 0), tz),
LocalDate.of(2299, 12, 31).atStartOfDay(ClickHouseValues.SYS_ZONE)
.withZoneSameInstant(tz.toZoneId())
.toLocalDate());
}
Expand All @@ -961,18 +961,18 @@ public void testWriteDate32() throws IOException {
generateBytes(0x17, 0x61, 0, 0));

Assert.assertEquals(
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1), null)),
generateBytes(0xCC, 0xBF, 0xFF, 0xFF));
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1900, 1, 1), null)),
generateBytes(0x21, 0x9C, 0xFF, 0xFF));
Assert.assertEquals(getWrittenBytes(
o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11), null)),
generateBytes(0xCB, 0xBF, 1, 0));
o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2299, 12, 31), null)),
generateBytes(0xD1, 0xD6, 1, 0));

Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(
o -> BinaryStreamUtils.writeDate32(o,
LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS), null)));
LocalDate.of(1900, 1, 1).minus(1L, ChronoUnit.DAYS), null)));
Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(
o -> BinaryStreamUtils.writeDate32(o,
LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS), null)));
LocalDate.of(2299, 12, 31).plus(1L, ChronoUnit.DAYS), null)));
}

@Test(dataProvider = "timeZoneProvider", groups = { "unit" })
Expand Down Expand Up @@ -1000,22 +1000,22 @@ public void testWriteDate32WithTimeZone(String timeZoneId) throws IOException {
.withZoneSameInstant(ClickHouseValues.SYS_ZONE).toLocalDate())));

Assert.assertEquals(
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 2), tz)),
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 2)
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1900, 1, 2), tz)),
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1900, 1, 2)
.atStartOfDay(tz.toZoneId())
.withZoneSameInstant(ClickHouseValues.SYS_ZONE).toLocalDate())));
Assert.assertEquals(
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 10), tz)),
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 10)
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2299, 12, 31), tz)),
getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2299, 12, 31)
.atStartOfDay(tz.toZoneId())
.withZoneSameInstant(ClickHouseValues.SYS_ZONE).toLocalDate())));

Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(
o -> BinaryStreamUtils.writeDate32(o,
LocalDate.of(1925, 1, 1).minus(2L, ChronoUnit.DAYS), tz)));
LocalDate.of(1900, 1, 1).minus(2L, ChronoUnit.DAYS), tz)));
Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(
o -> BinaryStreamUtils.writeDate32(o,
LocalDate.of(2283, 11, 11).plus(2L, ChronoUnit.DAYS), tz)));
LocalDate.of(2299, 12, 31).plus(2L, ChronoUnit.DAYS), tz)));
}

@Test(groups = { "unit" })
Expand Down Expand Up @@ -1256,12 +1256,12 @@ public void testWriteDateTime64() throws IOException {

Assert.assertThrows(IllegalArgumentException.class,
() -> getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o,
LocalDateTime.of(LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS),
LocalDateTime.of(LocalDate.of(1900, 1, 1).minus(1L, ChronoUnit.DAYS),
LocalTime.MAX),
null)));
Assert.assertThrows(IllegalArgumentException.class,
() -> getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o,
LocalDateTime.of(LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS),
LocalDateTime.of(LocalDate.of(2299, 12, 31).plus(1L, ChronoUnit.DAYS),
LocalTime.MIN),
null)));
}
Expand Down Expand Up @@ -1325,7 +1325,7 @@ public void testWriteDateTime64WithTimeZone(String timeZoneId) throws IOExceptio
() -> getWrittenBytes(
o -> BinaryStreamUtils.writeDateTime64(o,
LocalDateTime.of(
LocalDate.of(1925, 1, 1).minus(1L,
LocalDate.of(1900, 1, 1).minus(1L,
ChronoUnit.DAYS),
LocalTime.MAX)
.atOffset(ZoneOffset.UTC)
Expand All @@ -1336,7 +1336,7 @@ public void testWriteDateTime64WithTimeZone(String timeZoneId) throws IOExceptio
() -> getWrittenBytes(
o -> BinaryStreamUtils.writeDateTime64(o,
LocalDateTime.of(
LocalDate.of(2283, 11, 11).plus(1L,
LocalDate.of(2299, 12, 31).plus(1L,
ChronoUnit.DAYS),
LocalTime.MIN)
.atOffset(ZoneOffset.UTC)
Expand Down

0 comments on commit 52634fb

Please sign in to comment.