From d98c55ccb17ba8f14d66572b0c885c0a0bae81c6 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Mon, 10 Jun 2024 18:04:22 -0400 Subject: [PATCH 1/3] Fixing proxy support and added a test to cover --- client-v2/pom.xml | 6 + .../com/clickhouse/client/api/Client.java | 6 +- .../com/clickhouse/client/ProxyTests.java | 117 ++++++++++++++++++ .../clickhouse/client/insert/InsertTests.java | 3 +- 4 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/ProxyTests.java diff --git a/client-v2/pom.xml b/client-v2/pom.xml index 1284e287a..15d14df1f 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -109,6 +109,12 @@ ${testcontainers.version} test + + org.testcontainers + clickhouse + ${testcontainers.version} + test + org.testcontainers toxiproxy diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 384904789..5dc7c780a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -385,9 +385,9 @@ public Builder addProxy(ProxyType type, String host, int port) { ValidationUtils.checkNonBlank(host, "host"); ValidationUtils.checkRange(port, 1, ValidationUtils.TCP_PORT_NUMBER_MAX, "port"); - this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_TYPE), type.toString()); - this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_HOST), host); - this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_PORT), String.valueOf(port)); + this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_TYPE).toLowerCase(), type.toString()); + this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_HOST).toLowerCase(), host); + this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_PORT).toLowerCase(), String.valueOf(port)); return this; } diff --git a/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java b/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java new file mode 100644 index 000000000..6a8cda37d --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java @@ -0,0 +1,117 @@ +package com.clickhouse.client; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientException; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.enums.ProxyType; +import com.clickhouse.client.api.insert.InsertResponse; +import com.clickhouse.client.api.insert.InsertSettings; +import com.clickhouse.client.api.metrics.ClientMetrics; +import com.clickhouse.client.api.metrics.OperationMetrics; +import com.clickhouse.client.api.metrics.ServerMetrics; +import com.clickhouse.client.insert.SamplePOJO; +import eu.rekawek.toxiproxy.Proxy; +import eu.rekawek.toxiproxy.ToxiproxyClient; +import org.testcontainers.containers.ToxiproxyContainer; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +public class ProxyTests extends BaseIntegrationTest{ + private Client client; + ToxiproxyContainer toxiproxy = null; + ToxiproxyClient toxiproxyClient = null; + Proxy proxy = null; + + @BeforeMethod(groups = { "integration" }, enabled = true) + public void setUp() throws IOException { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + toxiproxy = new ToxiproxyContainer(ClickHouseServerForTest.getProxyImage()) + .withNetwork(ClickHouseServerForTest.getNetwork()); + toxiproxy.start(); + + toxiproxyClient = new ToxiproxyClient(toxiproxy.getHost(), toxiproxy.getControlPort()); + proxy = toxiproxyClient.createProxy("clickhouse", "0.0.0.0:8666", + ClickHouseServerForTest.hasClickHouseContainer() + ? "clickhouse:" + ClickHouseProtocol.HTTP.getDefaultPort() + : ClickHouseServerForTest.getClickHouseAddress(ClickHouseProtocol.HTTP, true)); + + client = new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), false) + .setUsername("default") + .setPassword("") + .addProxy(ProxyType.HTTP, toxiproxy.getHost(), toxiproxy.getMappedPort(8666)) + .build(); + } + + @AfterMethod(groups = { "integration" }, enabled = true) + public void teardown() { + if (toxiproxy != null) { + toxiproxy.stop(); + } + } + + private void createTable(String tableQuery) throws ClickHouseException { + try (ClickHouseClient client = ClickHouseClient.builder().config(new ClickHouseConfig()) + .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP)) + .build()) { + client.read(getServer(ClickHouseProtocol.HTTP)).query(tableQuery).executeAndWait().close(); + } + } + + + @Test(groups = { "integration" }, enabled = false) + public void simpleProxyTest() throws Exception { + String tableName = "simple_pojo_proxy_table"; + String createSQL = SamplePOJO.generateTableCreateSQL(tableName); + System.out.println(createSQL); + createTable(createSQL); + + client.register(SamplePOJO.class, SamplePOJO.generateTableSchema(tableName)); + List simplePOJOs = new ArrayList<>(); + + for (int i = 0; i < 1000; i++) { + simplePOJOs.add(new SamplePOJO()); + } + proxy.enable(); + InsertResponse response = client.insert(tableName, simplePOJOs).get(120, TimeUnit.SECONDS); + + OperationMetrics metrics = response.getMetrics(); + assertEquals(simplePOJOs.size(), metrics.getMetric(ServerMetrics.NUM_ROWS_WRITTEN).getLong()); + assertEquals(simplePOJOs.size(), response.getWrittenRows()); + assertTrue(metrics.getMetric(ClientMetrics.OP_DURATION).getLong() > 0); + assertTrue(metrics.getMetric(ClientMetrics.OP_SERIALIZATION).getLong() > 0); + } + + @Test(groups = { "integration" }, enabled = true) + public void simpleDisabledProxyTest() throws Exception { + String tableName = "simple_pojo_proxy_table"; + String createSQL = SamplePOJO.generateTableCreateSQL(tableName); + System.out.println(createSQL); + createTable(createSQL); + + client.register(SamplePOJO.class, SamplePOJO.generateTableSchema(tableName)); + List simplePOJOs = new ArrayList<>(); + + for (int i = 0; i < 1000; i++) { + simplePOJOs.add(new SamplePOJO()); + } + //proxy.disable(); + try { + InsertResponse response = client.insert(tableName, simplePOJOs).get(120, TimeUnit.SECONDS); + } catch (Exception e) { + //FOR NOW THIS WILL FAIL + //IF IT SUCCEEDS (meaning the test fails), YAY PROXY WORKS WE SHOULD REIMPLEMENT THE TESTS! + assertTrue(e.getMessage().contains("Operation has likely timed out.")); + } + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java b/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java index 8f8011832..41a23a1e6 100644 --- a/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java @@ -18,6 +18,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -31,7 +32,7 @@ public class InsertTests extends BaseIntegrationTest { private InsertSettings settings; @BeforeMethod(groups = { "integration" }, enabled = true) - public void setUp() { + public void setUp() throws IOException { ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); client = new Client.Builder() .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), false) From 4a3eaa807f5e58e8ed48fbdbbda8606367012f99 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Mon, 10 Jun 2024 18:18:38 -0400 Subject: [PATCH 2/3] Update pom.xml --- client-v2/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/client-v2/pom.xml b/client-v2/pom.xml index 15d14df1f..1284e287a 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -109,12 +109,6 @@ ${testcontainers.version} test - - org.testcontainers - clickhouse - ${testcontainers.version} - test - org.testcontainers toxiproxy From 3241b1b7e000fef944f6c2b000c48ae1544775ad Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Mon, 10 Jun 2024 18:22:45 -0400 Subject: [PATCH 3/3] Update Client.java --- .../src/main/java/com/clickhouse/client/api/Client.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 5dc7c780a..9f67559fa 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -385,9 +385,9 @@ public Builder addProxy(ProxyType type, String host, int port) { ValidationUtils.checkNonBlank(host, "host"); ValidationUtils.checkRange(port, 1, ValidationUtils.TCP_PORT_NUMBER_MAX, "port"); - this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_TYPE).toLowerCase(), type.toString()); - this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_HOST).toLowerCase(), host); - this.configuration.put(String.valueOf(ClickHouseClientOption.PROXY_PORT).toLowerCase(), String.valueOf(port)); + this.configuration.put(ClickHouseClientOption.PROXY_TYPE.getKey(), type.toString()); + this.configuration.put(ClickHouseClientOption.PROXY_HOST.getKey(), host); + this.configuration.put(ClickHouseClientOption.PROXY_PORT.getKey(), String.valueOf(port)); return this; }