Merge pull request #379 from Altinity/fix_datetime_limits

Fix datetime limits

.github/workflows/docker-build.yml

@@ -24,10 +24,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v2
         with:
-          java-version: '11'
+          java-version: '17'
           distribution: 'temurin'
           cache: maven
 

.github/workflows/sink-connector-kafka-tests.yml

@@ -22,10 +22,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 11
+    - name: Set up JDK 17
       uses: actions/setup-java@v2
       with:
-        java-version: '11'
+        java-version: '17'
         distribution: 'temurin'
         cache: maven
     - name: Build with Maven

.github/workflows/sink-connector-lightweight-tests.yml

@@ -22,10 +22,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v2
         with:
-          java-version: '11'
+          java-version: '17'
           distribution: 'temurin'
           cache: maven
       - name: Install maven

Dockerfile

@@ -1,5 +1,5 @@
-FROM openjdk:11
+FROM openjdk:17
 COPY sink-connector-client/sink-connector-client /sink-connector-client
-COPY sink-connector-lightweight/target/clickhouse-debezium-embedded-1.0-SNAPSHOT.jar /app.jar
+COPY sink-connector-lightweight/target/clickhouse-debezium-embedded*.jar /app.jar
 ENV JAVA_OPTS="-Dlog4jDebug=true"
 ENTRYPOINT ["java", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005", "-jar","/app.jar", "/config.yml", "com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]

doc/configuration.md

@@ -59,6 +59,11 @@ https://debezium.io/documentation/reference/stable/connectors/postgresql.html#po
 | persist.raw.bytes                     | Debezium.BYTES data(usually UUID) is persisted as raw bytes(CH String) if set to true.                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | database.connectionTimeZone           | Example: "US/Samoa,  Specify MySQL timezone for DATETIME conversions.https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types                                                                                                                                                                                                                                                                                                                                                                                        |
 | enable.snapshot.ddl                   | When true, pre-existing DDL statements from source(MySQL) will be executed. Warning: This might run DROP TABLE commands.                                                                                                                                                                                                                                                                                                                                                                                                             |
+| clickhouse.datetime.timezone          | Override timezone for DateTime columns in ClickHouse server.                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| skip_replica_start                    | If set to true, replication is not started, the user is expected to start replication with the sink-connector-client program.                                                                                                                                                                                                                                                                                                                                                                                                        |
+
+
+
 ## Kafka Sink Connector for ClickHouse
 
 ### Recommended Memory limits.

doc/timezone.md

@@ -0,0 +1,61 @@
+## Time Zone
+
+### MySQL
+1. The environment variable `TZ=US/Central` in docker-compose under mysql can be used to set the timezone for MySQL.
+
+2. To make sure the timezone is properly set in MySQL, run the following SQL  on MySQL Server.
+
+ `select @@system_time_zone`
+
+3. Set the `database.connectionTimezone: 'US/Central'` in config.yml to make sure the sink connector performs the datetime/timestamp conversions using the same timezone.
+
+Note: With the new version of Debezium there are no errors if the `database.connectionTimeZone` is not set and if the MySQL server is set to a different timezone
+`TZ=US/Central`
+
+### ClickHouse
+
+To check the ClickHouse server timezone.
+```
+SELECT timezone()
+
+Query id: f7b665b5-639f-4e1c-b494-256744c4310f
+
+┌─timezone()──────┐
+│ America/Chicago │
+└─────────────────┘
+
+```
+The configuration variable `clickhouse.datetime.timezone: "UTC"` is to used to force the timezone for `DateTime/DateTime64` fields
+when its persisted to ClickHouse. By Default the server timezone will be used.
+```
+
+SELECT toDateTime(Mid_Value, 'UTC') AS utc_time
+FROM temporal_types_DATETIME5
+
+Query id: f6c735bc-cdb3-4df6-995d-00dfe2d13ae8
+
+┌────────────utc_time─┐
+│ 2022-09-29 06:50:28 │
+└─────────────────────┘
+
+d1c194dbabc3 :) select toDateTime(Mid_Value, 'America/Chicago') as chicago_time from temporal_types_DATETIME5;
+
+SELECT toDateTime(Mid_Value, 'America/Chicago') AS chicago_time
+FROM temporal_types_DATETIME5
+
+Query id: 77602998-f7f8-4bfb-b7a4-2a2034541c62
+
+┌────────chicago_time─┐
+│ 2022-09-29 01:50:28 │
+
+
+SELECT *
+FROM temporal_types_DATETIME5
+
+Query id: c2f743df-f64e-4037-ad85-6363f9d7f11c
+
+┌─Type────────┬─────────────Minimum_Value─┬─────────────────Mid_Value─┬─────────────Maximum_Value─┬─Null_Value─┬────────────_version─┬─is_deleted─┐
+│ DATETIME(5) │ 1900-01-01 00:00:00.00000 │ 2022-09-29 01:50:28.12345 │ 2299-12-31 23:59:59.99999 │       ᴺᵁᴸᴸ │ 1730606158705590330 │          0 │
+└─────────────┴───────────────────────────┴───────────────────────────┴───────────────────────────┴────────────┴─────────────────────┴────────────┘
+
+```

pom.xml

@@ -10,8 +10,8 @@
     <packaging>pom</packaging>
 
     <properties>
-        <maven.compiler.source>11</maven.compiler.source>
-        <maven.compiler.target>11</maven.compiler.target>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <version.debezium>2.4.0</version.debezium>

sink-connector-lightweight/Dockerfile

@@ -1,4 +1,4 @@
-FROM openjdk:11
+FROM openjdk:17
 COPY sink-connector-client/sink-connector-client /sink-connector-client
 COPY sink-connector-lightweight/target/clickhouse-debezium-embedded-*.jar /app.jar
 ENV JAVA_OPTS="-Dlog4jDebug=true"

sink-connector-lightweight/clickhouse/config.xml

@@ -0,0 +1,11 @@
+<clickhouse replace="true">
+    <user_directories>
+        <users_xml>
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
+    <timezone>America/Chicago</timezone>
+</clickhouse>

sink-connector-lightweight/dependency-reduced-pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.altinity</groupId>
   <artifactId>clickhouse-debezium-embedded</artifactId>
-  <version>1.1-SNAPSHOT</version>
+  <version>0.0.4</version>
   <build>
     <extensions>
       <extension>
@@ -293,11 +293,12 @@
     <version.testcontainers>1.19.1</version.testcontainers>
     <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <sink-connector-library-version>0.0.5</sink-connector-library-version>
     <version.junit>5.9.1</version.junit>
-    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.source>17</maven.compiler.source>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <version.checkstyle.plugin>3.1.1</version.checkstyle.plugin>
-    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.target>17</maven.compiler.target>
     <version.debezium>2.5.0.Alpha1</version.debezium>
     <quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
   </properties>

sink-connector-lightweight/docker/config.yml

@@ -44,3 +44,7 @@ schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector
 enable.snapshot.ddl: "true"
 persist.raw.bytes: "false"
 auto.create.tables: "true"
+#database.connectionTimeZone: "US/Samoa"
+#clickhouse.datetime.timezone: "UTC"
+#skip_replica_start: "false"
+#binary.handling.mode: "base64"

sink-connector-lightweight/docker/docker-compose.yml

@@ -19,6 +19,7 @@ services:
       - MYSQL_REPLICATION_MODE=master
       - MYSQL_REPLICATION_USER=repl_user
       - ALLOW_EMPTY_PASSWORD=yes
+      - TZ=US/Central
     volumes:
       - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
       - ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql
@@ -47,6 +48,7 @@ services:
         - MYSQL_MASTER_PORT_NUMBER=3306
         - MYSQL_MASTER_ROOT_PASSWORD=root
         - ALLOW_EMPTY_PASSWORD=yes
+        -  TZ=America/Samoa
       volumes:
         - ./mysqld-slave.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
       healthcheck:
@@ -77,6 +79,7 @@ services:
         hard: "262144"
     volumes:
       -  ../clickhouse/users.xml:/etc/clickhouse-server/users.xml
+      - ../clickhouse/config.xml:/etc/clickhouse-server/config.d/config.xml
     depends_on:
       zookeeper:
         condition: service_healthy

sink-connector-lightweight/pom.xml

@@ -9,8 +9,8 @@
     <version>0.0.4</version>
 
     <properties>
-        <maven.compiler.source>11</maven.compiler.source>
-        <maven.compiler.target>11</maven.compiler.target>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <version.debezium>2.5.0.Alpha1</version.debezium>
@@ -23,7 +23,7 @@
         <quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
         <quarkus.platform.version>2.14.0.Final</quarkus.platform.version>
         <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
-        <sink-connector-library-version>0.0.4</sink-connector-library-version>
+        <sink-connector-library-version>0.0.5</sink-connector-library-version>
     </properties>
     <dependencyManagement>
         <dependencies>
@@ -44,9 +44,16 @@
         <dependency>
             <groupId>com.google.inject</groupId>
             <artifactId>guice</artifactId>
-            <version>5.1.0</version>
+            <version>6.0.0</version>
         </dependency>
 
+        <dependency>
+            <groupId>jakarta.inject</groupId>
+            <artifactId>jakarta.inject-api</artifactId>
+            <version>2.0.1</version>
+        </dependency>
+
+
         <!-- REST API SERVER -->
         <dependency>
             <groupId>io.javalin</groupId>