Skip to content

Commit

Permalink
Added more test cases to cover datetime limits in DebeziumConverterTest
Browse files Browse the repository at this point in the history
  • Loading branch information
@subkanthi
subkanthi committed Nov 24, 2023
1 parent 63bb852 commit 334ad64
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 50 deletions.
24 changes: 21 additions & 3 deletions .../com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,15 @@ public void testCreateTable() throws Exception {
boolean dateResultValueChecked = false;
while(dateResult.next()) {
dateResultValueChecked = true;
Assert.assertTrue(dateResult.getDate("Minimum_Value").toString().equalsIgnoreCase("1925-01-01"));
Assert.assertTrue(dateResult.getDate("Mid_Value").toString().equalsIgnoreCase("2022-09-29"));
Assert.assertTrue(dateResult.getDate("Maximum_Value").toString().equalsIgnoreCase("2283-11-11"));


System.out.println(dateResult.getTimestamp("Mid_Value").toString());
System.out.println(dateResult.getTimestamp("Maximum_Value").toString());
System.out.println(dateResult.getTimestamp("Minimum_Value").toString());

Assert.assertTrue(dateResult.getDate("Mid_Value").toString().contains("2022-09-29"));
Assert.assertTrue(dateResult.getDate("Maximum_Value").toString().contains("2283-11-11"));
Assert.assertTrue(dateResult.getDate("Minimum_Value").toString().contains("1925-01-01"));
}
Assert.assertTrue(dateResultValueChecked);

Expand All @@ -129,6 +135,11 @@ public void testCreateTable() throws Exception {
while(dateTimeResult.next()) {
System.out.println("DATE TIME");
dateTimeResultValueChecked = true;

System.out.println(dateTimeResult.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult.getTimestamp("Maximum_Value").toString());
System.out.println(dateTimeResult.getTimestamp("Minimum_Value").toString());

Assert.assertTrue(dateTimeResult.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
Assert.assertTrue(dateTimeResult.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:47:46.0"));
Assert.assertTrue(dateTimeResult.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
Expand All @@ -143,6 +154,10 @@ public void testCreateTable() throws Exception {
System.out.println("DATE TIME 1");
dateTimeResult1ValueChecked = true;

System.out.println(dateTimeResult1.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult1.getTimestamp("Maximum_Value").toString());
System.out.println(dateTimeResult1.getTimestamp("Minimum_Value").toString());

Assert.assertTrue(dateTimeResult1.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
Assert.assertTrue(dateTimeResult1.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:48:25.1"));
Assert.assertTrue(dateTimeResult1.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
Expand All @@ -153,6 +168,9 @@ public void testCreateTable() throws Exception {
ResultSet dateTimeResult2 = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME2");
while(dateTimeResult2.next()) {
System.out.println("DATE TIME 2");
System.out.println(dateTimeResult2.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult2.getTimestamp("Maximum_Value").toString());
System.out.println(dateTimeResult2.getTimestamp("Minimum_Value").toString());

Assert.assertTrue(dateTimeResult2.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
Assert.assertTrue(dateTimeResult2.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:49:05.12"));
Expand Down
18 changes: 3 additions & 15 deletions ...or/src/main/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public static class DateConverter {
*
* Function to convert Debezium Date fields
* to java.sql.Date
* @param value
* @param value - NUMBER OF DAYS since epoch.
* @return
*/
public static Date convert(Object value, ClickHouseDataType chDataType) {
Expand Down Expand Up @@ -165,15 +165,6 @@ public static class ZonedTimestampConverter {
*/
public static String convert(Object value, ZoneId serverTimezone) {

// TemporalAccessor parsedTime = ZonedTimestamp.FORMATTER.parse((String) value);
// DateTimeFormatter bqZonedTimestampFormat =
// new DateTimeFormatterBuilder()
// .append(DateTimeFormatter.ISO_LOCAL_DATE)
// .appendLiteral(' ')
// .append(DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS"))
// .toFormatter();
// return bqZonedTimestampFormat.format(parsedTime);

String result = "";
DateTimeFormatter destFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS")
.withZone(serverTimezone);
Expand Down Expand Up @@ -208,15 +199,12 @@ public static String convert(Object value, ZoneId serverTimezone) {
ZonedDateTime zd = ZonedDateTime.parse((String) value, formatter.withZone(serverTimezone));

long dateTimeInMs = zd.toInstant().toEpochMilli();
if(dateTimeInMs > BinaryStreamUtils.DATETIME64_MAX) {
if(dateTimeInMs > BinaryStreamUtils.DATETIME64_MAX * 1000) {
zd = ZonedDateTime.ofInstant(Instant.ofEpochSecond(BinaryStreamUtils.DATETIME64_MAX), serverTimezone);
} else if(dateTimeInMs < BinaryStreamUtils.DATETIME64_MIN) {
} else if(dateTimeInMs < BinaryStreamUtils.DATETIME64_MIN * 1000) {
zd = ZonedDateTime.ofInstant(Instant.ofEpochSecond(BinaryStreamUtils.DATETIME64_MIN), serverTimezone);
}
//long v = ClickHouseChecker.between(ClickHouseValues.UTC_ZONE.equals(this.zoneId) ? dt.toEpochSecond(ZoneOffset.UTC) : dt.atZone(this.zoneId).toEpochSecond(), "DateTime", BinaryStreamUtils.DATETIME64_MIN, BinaryStreamUtils.DATETIME64_MAX);

result = zd.format(destFormatter);
//result = removeTrailingZeros(result);
parsingSuccesful = true;
break;
} catch(Exception e) {
Expand Down
86 changes: 54 additions & 32 deletions ...rc/test/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,82 +7,93 @@
import com.clickhouse.jdbc.ClickHouseConnection;
import com.clickhouse.jdbc.ClickHouseDataSource;
import org.junit.Assert;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

import java.sql.PreparedStatement;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.ZoneId;
import java.time.*;
import java.util.*;
import java.util.concurrent.TimeUnit;

import static com.altinity.clickhouse.sink.connector.metadata.DataTypeRange.CLICKHOUSE_MAX_SUPPORTED_DATE32;
import static com.altinity.clickhouse.sink.connector.metadata.DataTypeRange.CLICKHOUSE_MIN_SUPPORTED_DATE32;

public class DebeziumConverterTest {

@Test
public void testMicroTimeConverter() {
@DisplayName("Test timestamp converter for multiple timezones.")
public void testTimestampConverter() {

Object timeInMicroSeconds = 3723000000L;
String formattedTime = DebeziumConverter.MicroTimeConverter.convert(timeInMicroSeconds);
Object timestampEpoch = LocalDateTime.of(2022, 1, 1, 0, 1, 0).atZone(ZoneId.of("UTC")).toEpochSecond() * 1000;

Assert.assertTrue(formattedTime.equalsIgnoreCase("01:02:03.000000"));
}
String formattedTimestamp = DebeziumConverter.TimestampConverter.convert(timestampEpoch, ClickHouseDataType.DateTime64, ZoneId.of("UTC"));
Assert.assertTrue(formattedTimestamp.equalsIgnoreCase("2022-01-01 00:01:00"));

@Test
public void testTimestampConverter() {

Object timestampEpoch = 1640995260000L;
String formattedTimestamp = String.valueOf(DebeziumConverter.TimestampConverter.convert(timestampEpoch, ClickHouseDataType.DateTime64, ZoneId.of("UTC")));
// 6 hours difference.
String timestampWithChicagoTZ = DebeziumConverter.TimestampConverter.convert(timestampEpoch, ClickHouseDataType.DateTime64, ZoneId.of("America/Chicago"));
Assert.assertTrue(timestampWithChicagoTZ.equalsIgnoreCase("2021-12-31 18:01:00"));

Assert.assertTrue(formattedTimestamp.equalsIgnoreCase("1640995260000"));
String timestampWithPacificTZ = DebeziumConverter.TimestampConverter.convert(timestampEpoch, ClickHouseDataType.DateTime64, ZoneId.of("America/Los_Angeles"));
Assert.assertTrue(timestampWithPacificTZ.equalsIgnoreCase("2021-12-31 16:01:00"));
}

@Test
@DisplayName("Test timestamp converter(MIN) when clickhouse columns are DateTime and DateTime64, min limit is different for DateTime and DateTime64")
public void testTimestampConverterMinRange() {

Object timestampEpoch = -2166681362000L;
String formattedTimestamp = String.valueOf(DebeziumConverter.TimestampConverter.convert(timestampEpoch, ClickHouseDataType.DateTime64, ZoneId.of("UTC")));
Object timestampEpochDateTime = LocalDateTime.of(1960, 1, 1, 0, 1, 0).atZone(ZoneId.of("UTC")).toEpochSecond() * 1000;
Assert.assertTrue(DebeziumConverter.TimestampConverter.convert(timestampEpochDateTime, ClickHouseDataType.DateTime32, ZoneId.of("UTC")).equalsIgnoreCase("1970-01-01 00:00:00"));

Assert.assertTrue(formattedTimestamp.equalsIgnoreCase("-1420070400000"));
//Clickhouse column DateTime64
Assert.assertTrue(DebeziumConverter.TimestampConverter.convert(timestampEpochDateTime, ClickHouseDataType.DateTime64, ZoneId.of("UTC")).equalsIgnoreCase("1960-01-01 00:01:00"));
}

@Test
@DisplayName("Test timestamp converter(MAX) when clickhouse columns are DateTime and DateTime64, min limit is different for DateTime and DateTime64")
public void testTimestampConverterMaxRange() {

Object timestampEpoch = 4807440238000L;
String formattedTimestamp = String.valueOf(DebeziumConverter.TimestampConverter.convert(timestampEpoch, ClickHouseDataType.DateTime64, ZoneId.of("UTC")));
Object timestampEpochDateTime = LocalDateTime.of(2289, 1, 1, 0, 1, 0).atZone(ZoneId.of("UTC")).toEpochSecond() * 1000;
String formattedTimestamp = String.valueOf(DebeziumConverter.TimestampConverter.convert(timestampEpochDateTime, ClickHouseDataType.DateTime64, ZoneId.of("UTC")));

Assert.assertTrue(formattedTimestamp.equalsIgnoreCase("4807440238000"));
Assert.assertTrue(formattedTimestamp.equalsIgnoreCase("2283-11-11 23:59:59"));
}

@Test
public void testDateConverter() {

Integer date = -354285;
Integer date = Math.toIntExact(LocalDate.of(1925, 1, 1).toEpochDay());
java.sql.Date formattedDate = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date32);

Assert.assertTrue(formattedDate.toString().equalsIgnoreCase("1925-01-01"));
}

@Test
@DisplayName("Test Date converter(MIN), min limits are different for Date and Date32 types")
public void testDateConverterMinRange() {

Integer date = -16436;
java.sql.Date formattedDate = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date32);
Integer date = Math.toIntExact(LocalDate.of(1960, 1, 1).toEpochDay());

//Date32
java.sql.Date formattedDate32 = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date32);
Assert.assertTrue(formattedDate32.toString().equalsIgnoreCase("1960-01-01"));

Assert.assertTrue(formattedDate.toString().equalsIgnoreCase("1925-01-01"));
//Date
java.sql.Date formattedDate = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date);
Assert.assertTrue(formattedDate.toString().equalsIgnoreCase("1970-01-01"));
}

@Test
@DisplayName("Test Date converter(MAX), min limits are different for Date and Date32 types")
public void testDateConverterMaxRange() {

Integer date = 450000;
java.sql.Date formattedDate = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date32);
Integer date = Math.toIntExact(LocalDate.of(2299, 1, 1).toEpochDay());

//Date32
java.sql.Date formattedDate32 = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date32);
Assert.assertTrue(formattedDate32.toString().equalsIgnoreCase("2283-11-11"));

//Date
java.sql.Date formattedDate = DebeziumConverter.DateConverter.convert(date, ClickHouseDataType.Date);
Assert.assertTrue(formattedDate.toString().equalsIgnoreCase("2149-06-06"));

Assert.assertTrue(formattedDate.toString().equalsIgnoreCase("2283-11-11"));
}

@Test
Expand All @@ -106,11 +117,22 @@ public void testZonedTimestampConverter() {
String formattedTimestamp3 = DebeziumConverter.ZonedTimestampConverter.convert("2038-01-19T03:14:07.99Z", ZoneId.of("UTC"));
Assert.assertTrue(formattedTimestamp3.equalsIgnoreCase("2038-01-19 03:14:07.990000"));

// Test max limit
String formattedTimestamp4 = DebeziumConverter.ZonedTimestampConverter.convert("2338-01-19T03:14:07.99Z", ZoneId.of("UTC"));
Assert.assertTrue(formattedTimestamp4.equalsIgnoreCase("2283-11-11 23:59:59.000000"));
}

@Test
public void testCheckIfDateTimeExceedsSupportedRange() {
DebeziumConverter.TimestampConverter.convert(1665076675000L, ClickHouseDataType.DateTime64, ZoneId.of("UTC"));
public void testMicroTimeConverter() {

Object timeInMicroSeconds = LocalTime.of(10, 1, 1, 1).toEpochSecond(LocalDate.now(), ZoneOffset.UTC);
String formattedTime = DebeziumConverter.MicroTimeConverter.convert(timeInMicroSeconds);

Assert.assertTrue(formattedTime.equalsIgnoreCase("00:28:20.820061"));

Object timePacificTZ = ZonedDateTime.of(2024, 1, 1, 1, 1, 1, 1, ZoneId.of("America/Los_Angeles")).toEpochSecond() * 1000 * 1000;
String formattedTimePacificTZ = DebeziumConverter.MicroTimeConverter.convert(timePacificTZ);
Assert.assertTrue(formattedTimePacificTZ.equalsIgnoreCase("09:01:01.000000"));
}


Expand Down

0 comments on commit 334ad64

Please sign in to comment.