Skip to content

Commit

Permalink
Fixed integration tests for datetime, added logic to include primary …
Browse files Browse the repository at this point in the history 
…key when DDL has partitions
  • Loading branch information
@subkanthi
subkanthi committed Nov 27, 2023
1 parent ecc4061 commit 16e4655
Show file tree
Hide file tree
Showing 7 changed files with 102 additions and 58 deletions.
6 changes: 6 additions & 0 deletions ...java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,12 @@ private void parseCreateTable(MySqlParser.CreateTableContext ctx, StringBuilder
if (colDefinitionChildTree.getText() != null) {

}
} else if (colDefinitionChildTree instanceof MySqlParser.PrimaryKeyColumnConstraintContext) {
for(ParseTree primaryKeyTree: ((MySqlParser.PrimaryKeyColumnConstraintContext) colDefinitionChildTree).children) {
System.out.println(primaryKeyTree.getText());
orderByColumns.append(columnName);
break;
}
}
}
if(isNullColumn) {
Expand Down
66 changes: 33 additions & 33 deletions ...est/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,39 +107,39 @@ public void testCreateTable() throws Exception {
ResultSet dateTimeResult = writer.executeQueryWithResultSet("select * from temporal_types_DATETIME");

/**
* DATE TIME
* 2022-09-29 01:47:46.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 1
* 2022-09-29 01:48:25.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 2
* 2022-09-29 01:49:05.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 3
* 2022-09-29 01:49:22.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 4
* 2022-09-29 01:50:12.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 5
* 2022-09-29 01:50:28.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 6
* 2022-09-29 01:50:56.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
* DATE TIME 6
* 2022-09-29 01:50:56.0
* 1970-05-01 12:43:11.999
* 1900-01-01 00:00:00.0
*/
DATE TIME
1900-01-01 18:09:24.0
2022-09-28 20:47:46.0
1970-05-01 07:43:11.999
DATE TIME 1
1900-01-01 18:09:24.0
2022-09-28 20:48:25.0
1970-05-01 07:43:11.999
DATE TIME 2
1900-01-01 18:09:24.0
2022-09-28 20:49:05.0
1970-05-01 07:43:11.999
DATE TIME 3
2022-09-28 20:49:22.0
1970-05-01 07:43:11.999
1900-01-01 18:09:24.0
DATE TIME 4
2022-09-28 20:50:12.123
2299-12-31 17:59:59.999
1900-01-01 18:09:24.0
DATE TIME 5
2022-09-28 20:50:28.123
2299-12-31 17:59:59.999
1900-01-01 18:09:24.0
DATE TIME 6
2022-09-28 20:50:56.123
2299-12-31 17:59:59.999
1900-01-01 18:09:24.0
DATE TIME 6
2022-09-28 20:50:56.1
2299-12-31 17:59:59.999
1900-01-01 18:09:24.0
**/
while(dateTimeResult.next()) {
System.out.println("DATE TIME");

Expand Down
43 changes: 22 additions & 21 deletions ...om/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService;
import com.altinity.clickhouse.sink.connector.db.BaseDbWriter;
import org.apache.log4j.BasicConfigurator;
import org.junit.Assert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -119,9 +120,9 @@ public void testCreateTable() throws Exception {
System.out.println(dateTimeResult.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult.getTimestamp("Maximum_Value").toString());

// Assert.assertTrue(dateTimeResult.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
// Assert.assertTrue(dateTimeResult.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:47:46.0"));
// Assert.assertTrue(dateTimeResult.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
Assert.assertTrue(dateTimeResult.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1969-12-31 18:00:00.0"));
Assert.assertTrue(dateTimeResult.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-28 20:47:46.0"));
Assert.assertTrue(dateTimeResult.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2106-02-07 00:28:15.0"));
}

// DATETIME1
Expand All @@ -132,9 +133,9 @@ public void testCreateTable() throws Exception {
System.out.println(dateTimeResult1.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult1.getTimestamp("Maximum_Value").toString());

// Assert.assertTrue(dateTimeResult1.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
// Assert.assertTrue(dateTimeResult1.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:48:25.1"));
// Assert.assertTrue(dateTimeResult1.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
Assert.assertTrue(dateTimeResult1.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1969-12-31 18:00:00.0"));
Assert.assertTrue(dateTimeResult1.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-28 20:48:25.0"));
Assert.assertTrue(dateTimeResult1.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2106-02-07 00:28:15.0"));
}

// DATETIME2
Expand All @@ -145,24 +146,24 @@ public void testCreateTable() throws Exception {
System.out.println(dateTimeResult2.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult2.getTimestamp("Maximum_Value").toString());

// Assert.assertTrue(dateTimeResult2.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
// Assert.assertTrue(dateTimeResult2.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:49:05.12"));
// Assert.assertTrue(dateTimeResult2.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
Assert.assertTrue(dateTimeResult2.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1969-12-31 18:00:00.0"));
Assert.assertTrue(dateTimeResult2.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-28 20:49:05.0"));
Assert.assertTrue(dateTimeResult2.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2106-02-07 00:28:15.0"));
}

// DATETIME3
ResultSet dateTimeResult3 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME3");
while(dateTimeResult3.next()) {
System.out.println("DATE TIME 3");

System.out.println(dateTimeResult3.getTimestamp("Mid_Value").toString());
System.out.println(dateTimeResult3.getTimestamp("Maximum_Value").toString());
System.out.println(dateTimeResult3.getTimestamp("Minimum_Value").toString());

// Assert.assertTrue(dateTimeResult3.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:49:22.123"));
// Assert.assertTrue(dateTimeResult3.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
// Assert.assertTrue(dateTimeResult3.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
}
// ResultSet dateTimeResult3 = writer.executeQueryWithResultSet("select * from employees.temporal_types_DATETIME3");
// while(dateTimeResult3.next()) {
// System.out.println("DATE TIME 3");
//
// System.out.println(dateTimeResult3.getTimestamp("Mid_Value").toString());
// System.out.println(dateTimeResult3.getTimestamp("Maximum_Value").toString());
// System.out.println(dateTimeResult3.getTimestamp("Minimum_Value").toString());
//
//// Assert.assertTrue(dateTimeResult3.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:49:22.123"));
//// Assert.assertTrue(dateTimeResult3.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999"));
//// Assert.assertTrue(dateTimeResult3.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0"));
// }


// // DATETIME4
Expand Down
2 changes: 1 addition & 1 deletion ...ght/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public void testEmployeesDB() throws Exception {
employeesCHCount = chRs.getInt(1);
}

// Assert.assertTrue(employeesMySqlCount == employeesCHCount);
Assert.assertTrue(employeesMySqlCount == employeesCHCount);
// Files.deleteIfExists(tmpFilePath);
if(engine.get() != null) {
engine.get().stop();
Expand Down
36 changes: 35 additions & 1 deletion .../com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,25 @@ public void testCreateTableWithParitionRange() {
StringBuffer clickHouseQuery = new StringBuffer();
MySQLDDLParserService mySQLDDLParserService = new MySQLDDLParserService();
mySQLDDLParserService.parseSql(createQuery, "Persons", clickHouseQuery);
//Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase("CREATE TABLE rcx(a Nullable(Int32),b Nullable(Int32),c Nullable(String),d Nullable(Int32),`_version` UInt64,`is_deleted` UInt8) Engine=ReplacingMergeTree(_version,is_deleted) PARTITION BY (a,d,c) ORDER BY tuple()"));
Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase("CREATE TABLE t(id Nullable(Int32),dt Date32 NOT NULL ,`_version` UInt64,`is_deleted` UInt8) Engine=ReplacingMergeTree(_version,is_deleted) PARTITION BY (dt) ORDER BY id"));
log.info("Create table " + clickHouseQuery);

String createQueryWithoutPrimaryKey = "create table t(\n" +
"id int ,\n" +
"dt date not null\n" +
") engine=InnoDB\n" +
"PARTITION BY RANGE COLUMNS(dt)\n" +
"(PARTITION p20201231 VALUES LESS THAN ('2021-01-01') ENGINE = InnoDB,\n" +
" PARTITION p20211230 VALUES LESS THAN ('2021-12-31') ENGINE = InnoDB,\n" +
" PARTITION p20211231 VALUES LESS THAN ('2022-01-03') ENGINE = InnoDB,\n" +
" PARTITION p20220103 VALUES LESS THAN ('2022-01-04') ENGINE = InnoDB,\n" +
" PARTITION p20220104 VALUES LESS THAN ('2022-01-05') ENGINE = InnoDB,\n" +
" PARTITION p20220105 VALUES LESS THAN ('2022-01-06') ENGINE = InnoDB\n" +
");";
StringBuffer clickHouseQueryWOPrimaryKey = new StringBuffer();
mySQLDDLParserService.parseSql(createQueryWithoutPrimaryKey, "Persons", clickHouseQueryWOPrimaryKey);
Assert.assertTrue(clickHouseQueryWOPrimaryKey.toString().equalsIgnoreCase("Create table CREATE TABLE t(id Nullable(Int32),dt Date32 NOT NULL ,`_version` UInt64,`is_deleted` UInt8) Engine=ReplacingMergeTree(_version,is_deleted) PARTITION BY (dt) ORDER BY tuple()"));
log.info("Create table " + clickHouseQueryWOPrimaryKey);
}
@Test
public void testCreateTableWithKeyPartition() {
Expand All @@ -84,6 +101,23 @@ public void testCreateTableWithKeyPartition() {
Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase("CREATE TABLE members(firstname String NOT NULL ,lastname String NOT NULL ,username String NOT NULL ,email Nullable(String),joined Date32 NOT NULL ,`_version` UInt64,`is_deleted` UInt8) Engine=ReplacingMergeTree(_version,is_deleted) PARTITION BY joined ORDER BY tuple()"));
log.info("Create table " + clickHouseQuery);
}

@Test
public void testDateTimeColumns() {
String createQuery = "CREATE TABLE `temporal_types_DATETIME4` (\n" +
" `Type` varchar(50) NOT NULL,\n" +
" `Minimum_Value` datetime(4) NOT NULL,\n" +
" `Mid_Value` datetime(4) NOT NULL,\n" +
" `Maximum_Value` datetime(4) NOT NULL,\n" +
" `Null_Value` datetime(4) DEFAULT NULL,\n" +
" PRIMARY KEY (`Type`)\n" +
") ENGINE=InnoDB DEFAULT CHARSET=latin1;";
StringBuffer clickHouseQuery = new StringBuffer();
MySQLDDLParserService mySQLDDLParserService = new MySQLDDLParserService();
mySQLDDLParserService.parseSql(createQuery, "Persons", clickHouseQuery);
Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase("CREATE TABLE `temporal_types_DATETIME4`(`Type` String NOT NULL ,`Minimum_Value` DateTime64 NOT NULL ,`Mid_Value` DateTime64 NOT NULL ,`Maximum_Value` DateTime64 NOT NULL ,`Null_Value` Nullable(DateTime64),`_version` UInt64,`is_deleted` UInt8) Engine=ReplacingMergeTree(_version,is_deleted) ORDER BY (`Type`)"));
log.info("Create table " + clickHouseQuery);
}
@Test
public void testCreateTableAutoIncrement() {
StringBuffer clickHouseQuery = new StringBuffer();
Expand Down
5 changes: 4 additions & 1 deletion sink-connector-lightweight/src/test/resources/init_clickhouse_schema_only.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,4 +99,7 @@ CREATE TABLE employees.temporal_types_DATETIME2
)
ENGINE = ReplacingMergeTree(_version, is_deleted)
ORDER BY Type
SETTINGS index_granularity = 8192;
SETTINGS index_granularity = 8192;



2 changes: 1 addition & 1 deletion ...or/src/main/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ public static Integer checkIfDateExceedsSupportedRange(Integer epochInDays, Clic
return BinaryStreamUtils.U_INT16_MAX;
}
} else {
log.error("Unknown DATE field");
log.warn("Unknown DATE field:" + chDataType);
}

return epochInDays;
Expand Down

0 comments on commit 16e4655

Please sign in to comment.