Updates to mongo configuration.

sink-connector-lightweight/docker/config_mongo.yml

@@ -54,10 +54,10 @@ offset.flush.interval.ms: 5000
 # connector.class: The Java class for the connector. This must be set to io.debezium.connector.postgresql.PostgresConnector.
 connector.class: "io.debezium.connector.mongodb.MongoDbConnector"
 
-mongodb.connection.string : "mongodb://mongodb:27017/?replicaSet=rs0"
+mongodb.connection.string : "mongodb://mongo:27017/?replicaSet=rs0"
 topic.prefix : "dbserver1"
-mongodb.user : "debezium"
-mongodb.password : "dbz"
+mongodb.user : "myUserAdmin"
+mongodb.password : "admin"
 database.include.list  : "inventory"
 # offset.storage: The Java class that implements the offset storage strategy. This must be set to io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore.
 offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"

sink-connector-lightweight/docker/config_mongo_local.yml

@@ -0,0 +1,119 @@
+#### Some of the properties are part of Debezium PostgreSQL Connector
+#### https://debezium.io/documentation/reference/stable/connectors/postgresql.html
+# name: Unique name for the connector. Attempting to register again with the same name will fail.
+name: "debezium-embedded-mongo"
+
+auto.create.tables.replicated: "true"
+
+# database.hostname: IP address or hostname of the PostgreSQL database server.
+#database.hostname: "postgres"
+
+# database.port: Integer port number of the PostgreSQL database server listening for client connections.
+#database.port: "5432"
+
+# database.user: Name of the PostgreSQL database user to be used when connecting to the database.
+#database.user: "root"
+
+# database.password: Password of the PostgreSQL database user to be used when connecting to the database.
+#database.password: "root"
+
+# database.server.name: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode.
+database.server.name: "ER54"
+
+# schema.include.list: An optional list of regular expressions that match schema names to be monitored;
+schema.include.list: public,public2
+
+slot.name: connector2
+
+
+
+# table.include.list: An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored;
+#table.include.list: "public.tm,public.tm2"
+
+# clickhouse.server.url: Specify only the hostname of the Clickhouse Server.
+clickhouse.server.url: "localhost"
+
+# clickhouse.server.user: Clickhouse Server User
+clickhouse.server.user: "root"
+
+# clickhouse.server.password: Clickhouse Server Password
+clickhouse.server.password: "root"
+
+# clickhouse.server.port: Clickhouse Server Port
+clickhouse.server.port: "8123"
+
+# database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/
+database.allowPublicKeyRetrieval: "true"
+
+# snapshot.mode: Debezium can use different modes when it runs a snapshot. The snapshot mode is determined by the snapshot.mode configuration property.
+snapshot.mode: "initial"
+
+# offset.flush.interval.ms: The number of milliseconds to wait before flushing recent offsets to Kafka. This ensures that offsets are committed within the specified time interval.
+offset.flush.interval.ms: 5000
+
+# connector.class: The Java class for the connector. This must be set to io.debezium.connector.postgresql.PostgresConnector.
+connector.class: "io.debezium.connector.mongodb.MongoDbConnector"
+
+mongodb.connection.string : "mongodb://localhost:27017/?replicaSet=rs0"
+topic.prefix : "dbserver1"
+mongodb.user : "myUserAdmin"
+mongodb.password : "admin"
+database.include.list  : "inventory"
+# offset.storage: The Java class that implements the offset storage strategy. This must be set to io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore.
+offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
+
+# offset.storage.jdbc.offset.table.name: The name of the database table where connector offsets are to be stored.
+offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
+
+# offset.storage.jdbc.url: The JDBC URL for the database where connector offsets are to be stored.
+offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
+
+# offset.storage.jdbc.user: The name of the database user to be used when connecting to the database where connector offsets are to be stored.
+offset.storage.jdbc.user: "root"
+
+# offset.storage.jdbc.password: The password of the database user to be used when connecting to the database where connector offsets are to be stored.
+offset.storage.jdbc.password: "root"
+
+# offset.storage.jdbc.offset.table.ddl: The DDL statement used to create the database table where connector offsets are to be stored.
+offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
+(
+    `id` String,
+    `offset_key` String,
+    `offset_val` String,
+    `record_insert_ts` DateTime,
+    `record_insert_seq` UInt64,
+    `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
+)
+ENGINE = ReplacingMergeTree(_version)
+ORDER BY id
+SETTINGS index_granularity = 8192"
+offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
+# enable.snapshot.ddl: If set to true, the connector wil parse the DDL statements as part of initial load.
+enable.snapshot.ddl: "true"
+
+# auto.create.tables: If set to true, the connector will create the database tables for the destination tables if they do not already exist.
+auto.create.tables: "true"
+
+# database.dbname: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode.
+database.dbname: "public"
+
+# clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone.
+#clickhouse.datetime.timezone: "UTC"
+
+# skip_replica_start: If set to true, the connector will skip replication on startup. sink-connector-client start_replica will start replication.
+#skip_replica_start: "false"
+
+# binary.handling.mode: The mode for handling binary values. Possible values are bytes, base64, and decode. The default is bytes.
+#binary.handling.mode: "base64"
+
+# ignore_delete: If set to true, the connector will ignore delete events. The default is false.
+#ignore_delete: "true"
+
+#Metrics (Prometheus target), required for Grafana Dashboard
+metrics.enable: "true"
+
+#disable.ddl: If set to true, the connector will ignore DDL events. The default is false.
+#disable.ddl: "false"
+
+#disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false.
+#disable.drop.truncate: "false"

sink-connector-lightweight/docker/docker-compose-mongo.yml

@@ -42,20 +42,20 @@ services:
       - ./config_mongo.yml:/config.yml
 
   ### MONITORING ####
-  prometheus:
-    extends:
-      file: prometheus-service.yml
-      service: prometheus
-
-
-  grafana:
-    extends:
-      file: grafana-service.yml
-      service: grafana
-    volumes:
-      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
-      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
-      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
-    depends_on:
-      - prometheus
+#  prometheus:
+#    extends:
+#      file: prometheus-service.yml
+#      service: prometheus
+#
+#
+#  grafana:
+#    extends:
+#      file: grafana-service.yml
+#      service: grafana
+#    volumes:
+#      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+#      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+#      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
+#    depends_on:
+#      - prometheus
     ## END OF MONITORING ###

sink-connector-lightweight/docker/init-replica.js

@@ -0,0 +1,6 @@
+rs.initiate({
+  _id: "rs0",
+  members: [
+    { _id: 0, host: "mongo:27017" }
+  ]
+});

sink-connector-lightweight/docker/mongo-service.yml

@@ -8,4 +8,6 @@ services:
         - 27017:27017
     environment:
         - MONGODB_USER=debezium
-        - MONGODB_PASSWORD=dbz
+        - MONGODB_PASSWORD=dbz
+    volumes:
+      - ./run-mongodb.sh:/docker-entrypoint-initdb.d/run-mongodb.sh

sink-connector-lightweight/docker/run-mongodb.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+mongosh <<EOF
+db = connect("mongodb://root:example@localhost:27017/admin");
+rs.initiate({
+  _id: "rs0",
+  members: [
+    { _id: 0, host: "mongo:27017" }
+  ]
+});
+EOF
+