Merge pull request #361 from Altinity/subkanthi-patch-1

Update README.md

README.md

@@ -33,108 +33,18 @@ Also Update **ClickHouse information** for the following fields that are used to
 5.  Set `snapshot.mode` to `initial` if you like to replicate existing records, set `snapshot.mode` to `schema_only` to replicate schema and only the records that are modified after the connector is started.
 6.  Start replication by running the JAR file. `java -jar clickhouse-debezium-embedded-1.0-SNAPSHOT.jar <yaml_config_file>` or docker.
 
-### MySQL Configuration (docker/config.yaml)
+### MySQL Configuration 
 For `https` servers, make sure the `clickhouse.server.url` includes `https`
 Also add `?ssl=true` to both the `offset.storage.jdbc.url` and `schema.history.internal.jdbc.url` configuration variables.
 
-```
-database.hostname: "mysql-master"
-database.port: "3306"
-database.user: "root"
-database.password: "root"
-database.server.name: "ER54"
-database.include.list: sbtest
-#table.include.list=sbtest1
-clickhouse.server.url: "clickhouse"
-clickhouse.server.user: "root"
-clickhouse.server.password: "root"
-clickhouse.server.port: "8123"
-clickhouse.server.database: "test"
-database.allowPublicKeyRetrieval: "true"
-snapshot.mode: "schema_only"
-offset.flush.interval.ms: 5000
-connector.class: "io.debezium.connector.mysql.MySqlConnector"
-offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
-offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-offset.storage.jdbc.user: "root"
-offset.storage.jdbc.password: "root"
-offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
-(
-    `id` String,
-    `offset_key` String,
-    `offset_val` String,
-    `record_insert_ts` DateTime,
-    `record_insert_seq` UInt64,
-    `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
-)
-ENGINE = ReplacingMergeTree(_version)
-ORDER BY id
-SETTINGS index_granularity = 8198"
-offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
-schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
-schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-schema.history.internal.jdbc.user: "root"
-schema.history.internal.jdbc.password: "root"
-schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
-(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
-
-schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
-enable.snapshot.ddl: "true"
-auto.create.tables: "true"
-```
-### PostgreSQL Config(docker/config_postgres.yml)
+[MySQL Configuration](sink-connector-lightweight/docker/config.yml)
+
+### PostgreSQL Configuration
 For AWS RDS users, you might need to add heartbeat interval and query to avoid WAL logs constantly growing in size.
 https://stackoverflow.com/questions/76415644/postgresql-wal-log-limiting-rds
 https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-wal-disk-space
-```
-database.hostname: "postgres"
-database.port: "5432"
-database.user: "root"
-database.password: "root"
-database.server.name: "ER54"
-schema.include.list: public
-plugin.name: "pgoutput"
-table.include.list: "public.tm"
-clickhouse.server.url: "clickhouse"
-clickhouse.server.user: "root"
-clickhouse.server.password: "root"
-clickhouse.server.port: "8123"
-clickhouse.server.database: "test"
-database.allowPublicKeyRetrieval: "true"
-snapshot.mode: "initial"
-offset.flush.interval.ms: 5000
-connector.class: "io.debezium.connector.postgresql.PostgresConnector"
-offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
-offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-offset.storage.jdbc.user: "root"
-offset.storage.jdbc.password: "root"
-offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
-(
-    `id` String,
-    `offset_key` String,
-    `offset_val` String,
-    `record_insert_ts` DateTime,
-    `record_insert_seq` UInt64,
-    `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
-)
-ENGINE = ReplacingMergeTree(_version)
-ORDER BY id
-SETTINGS index_granularity = 8198"
-offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
-schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
-schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-schema.history.internal.jdbc.user: "root"
-schema.history.internal.jdbc.password: "root"
-schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
-(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
-
-schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
-enable.snapshot.ddl: "true"
-auto.create.tables: "true"
-database.dbname: "public"
-```
+
+[PostgreSQL Configuration](sink-connector-lightweight/docker/config_postgres.yml)
 
 ## Command Line(JAR)
 https://github.com/Altinity/clickhouse-sink-connector/releases