Merge pull request #346 from Altinity/fix_integration_tests

Fix integration tests

.github/workflows/docker-build.yml

@@ -0,0 +1,31 @@
+name: Sink connector(Kafka version) Docker Build/Push
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: maven
+      - name: Build with Maven
+        working-directory: sink-connector
+        run: mvn -B package --file pom.xml
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build the Docker image
+        working-directory: sink-connector
+        run: docker build . --file docker/Dockerfile-sink-on-debezium-base-image --tag altinity/clickhouse-sink-connector:$(date +%F)
+      - name: Push docker image to dockerhub
+        working-directory: sink-connector
+        run: docker push altinity/clickhouse-sink-connector:$(date +%F)

.github/workflows/sink-connector-kafka-tests.yml

@@ -0,0 +1,25 @@
+name: Sink Connector(Kafka version) tests
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
+      with:
+        java-version: '11'
+        distribution: 'temurin'
+        cache: maven
+    - name: Build with Maven
+      working-directory: sink-connector
+      run: mvn -B package --file pom.xml
+    - name: Run Unit tests
+      working-directory: sink-connector
+      run: mvn test -Dgroups=IntegrationTest --file pom.xml

.github/workflows/sink-connector-lightweight-integration-tests.yml → .github/workflows/sink-connector-lightweight-tests.yml

@@ -1,4 +1,4 @@
-name: Integration tests Sink connector lightweight
+name: Sink Connector(Light-weight) Tests
 
 on:
   push:
@@ -22,6 +22,6 @@ jobs:
       - name: Build Library with Maven
         working-directory: sink-connector
         run: mvn install -DskipTests=true
-#      - name: Build Replicator with Maven
-#        working-directory: sink-connector-lightweight
-#        run: mvn test
+      - name: Build Replicator with Maven
+        working-directory: sink-connector-lightweight
+        run: mvn test

.github/workflows/testflow_tests.yml → .github/workflows/testflow-sink-connector-kafka.yml

@@ -1,17 +1,17 @@
-name: TestFlows Tests
+name: TestFlows Tests - Sink connector(Kafka)
 
 on:
   push:
-    branches: [ main, develop]
+    branches: [ main ]
   pull_request:
-    branches: [ main , develop]
+    branches: [ main ]
   workflow_dispatch:
     inputs:
       sink_version:
         description: "sink connector version"
         required: true
         type: string
-        default: "2023-08-28"
+        default: "2023-10-30Update "
 
 env:
 #  SINK_CONNECTOR_VERSION: "${{ inputs.sink_version }}"

.github/workflows/testflow_tests_lightweight.yml → .github/workflows/testflow-sink-connector-lightweight.yml

@@ -1,4 +1,4 @@
-name: TestFlows Tests Lightweight
+name: TestFlows Tests - Sink Connector(Light-weight)
 
 on:
   push:

sink-connector-lightweight/Dockerfile

@@ -1,5 +1,5 @@
 FROM openjdk:11
 COPY sink-connector-client/sink-connector-client /sink-connector-client
-COPY sink-connector-lightweight/target/clickhouse-debezium-embedded-1.0-SNAPSHOT.jar /app.jar
+COPY sink-connector-lightweight/target/clickhouse-debezium-embedded-*.jar /app.jar
 ENV JAVA_OPTS="-Dlog4jDebug=true"
 ENTRYPOINT ["java", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005", "-jar","/app.jar", "/config.yml", "com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]

sink-connector-lightweight/README.md

@@ -7,52 +7,8 @@ Configuration(yaml) file mounted as volume
 ```
 
 **Configuration(MySQL)**
-```
-database.hostname: "mysql-master"
-database.port: "3306"
-database.user: "root"
-database.password: "root"
-database.server.name: "ER54"
-database.include.list: sbtest
-#table.include.list=sbtest1
-clickhouse.server.url: "clickhouse"
-clickhouse.server.user: "root"
-clickhouse.server.password: "root"
-clickhouse.server.port: "8123"
-clickhouse.server.database: "test"
-database.allowPublicKeyRetrieval: "true"
-snapshot.mode: "schema_only"
-offset.flush.interval.ms: 5000
-connector.class: "io.debezium.connector.mysql.MySqlConnector"
-offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
-offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123"
-offset.storage.jdbc.user: "root"
-offset.storage.jdbc.password: "root"
-offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
-(
-    `id` String,
-    `offset_key` String,
-    `offset_val` String,
-    `record_insert_ts` DateTime,
-    `record_insert_seq` UInt64,
-    `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
-)
-ENGINE = ReplacingMergeTree(_version)
-ORDER BY id
-SETTINGS index_granularity = 8198"
-offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
-schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
-schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123"
-schema.history.internal.jdbc.user: "root"
-schema.history.internal.jdbc.password: "root"
-schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
-(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
-
-schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
-enable.snapshot.ddl: "false"
+`(sink-connector-lightweight/docker/config.yml)`
 
-```
 Start the docker container
 A Sample docker-compose is provided , it starts the docker container \
 `registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest`
@@ -64,7 +20,7 @@ cd docker
 
 ###  Getting Started (PostgreSQL)
 
-`(sink-connector-lightweight/docker/docker_postgres.env)` 
+`(sink-connector-lightweight/docker/config_postgres.yml)` 
 
 **Configuration**
 ```

sink-connector-lightweight/dependency-reduced-pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.altinity</groupId>
   <artifactId>clickhouse-debezium-embedded</artifactId>
-  <version>1.0-SNAPSHOT</version>
+  <version>1.1-SNAPSHOT</version>
   <build>
     <extensions>
       <extension>
@@ -16,7 +16,7 @@
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>build-helper-maven-plugin</artifactId>
-        <version>3.2.0</version>
+        <version>3.4.0</version>
         <executions>
           <execution>
             <phase>generate-sources</phase>
@@ -51,21 +51,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>${quarkus.platform.group-id}</groupId>
-        <artifactId>quarkus-maven-plugin</artifactId>
-        <version>${quarkus.platform.version}</version>
-        <extensions>true</extensions>
-        <executions>
-          <execution>
-            <goals>
-              <goal>build</goal>
-              <goal>generate-code</goal>
-              <goal>generate-code-tests</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <artifactId>maven-checkstyle-plugin</artifactId>
         <version>${version.checkstyle.plugin}</version>
@@ -154,7 +139,7 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>testcontainers</artifactId>
-      <version>1.17.6</version>
+      <version>1.19.1</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
@@ -169,10 +154,6 @@
           <artifactId>duct-tape</artifactId>
           <groupId>org.rnorth.duct-tape</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>docker-java-api</artifactId>
-          <groupId>com.github.docker-java</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>docker-java-transport-zerodep</artifactId>
           <groupId>com.github.docker-java</groupId>
@@ -181,56 +162,44 @@
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>junit-jupiter</artifactId>
-      <version>1.17.6</version>
+      <artifactId>jdbc</artifactId>
+      <version>1.19.1</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
-          <artifactId>junit-jupiter-api</artifactId>
-          <groupId>org.junit.jupiter</groupId>
+          <artifactId>database-commons</artifactId>
+          <groupId>org.testcontainers</groupId>
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>1.19.1</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>clickhouse</artifactId>
-      <version>1.17.6</version>
+      <version>1.19.1</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>jdbc</artifactId>
-          <groupId>org.testcontainers</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mysql</artifactId>
-      <version>1.17.6</version>
+      <version>1.19.1</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>jdbc</artifactId>
-          <groupId>org.testcontainers</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>postgresql</artifactId>
-      <version>1.17.6</version>
+      <version>1.19.1</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>jdbc</artifactId>
-          <groupId>org.testcontainers</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mongodb</artifactId>
-      <version>1.17.6</version>
+      <version>1.19.1</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -250,14 +219,14 @@
           <artifactId>junit-platform-engine</artifactId>
           <groupId>org.junit.platform</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>apiguardian-api</artifactId>
-          <groupId>org.apiguardian</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>junit-jupiter-api</artifactId>
           <groupId>org.junit.jupiter</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>apiguardian-api</artifactId>
+          <groupId>org.apiguardian</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -339,6 +308,7 @@
   <properties>
     <quarkus.platform.version>2.14.0.Final</quarkus.platform.version>
     <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
+    <version.testcontainers>1.19.1</version.testcontainers>
     <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <version.junit>5.9.1</version.junit>