Skip to content

Commit

Permalink
Merge pull request #777 from zhicwu/enhance-jdbc
Browse files Browse the repository at this point in the history 
The build failure is irrelevant, merge to release `0.3.2-test3`.
  • Loading branch information
@zhicwu
zhicwu authored Dec 21, 2021
2 parents 30bda80 + 7ddc6e0 commit 48a6281
Show file tree
Hide file tree
Showing 81 changed files with 2,397 additions and 868 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/analysis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: |
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
mvn -q --batch-mode --global-toolchains .github/toolchains.xml -Drelease \
-Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
continue-on-error: true
4 changes: 2 additions & 2 deletions .github/workflows/benchmark.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ steps.version.outputs.value }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ steps.version.outputs.value }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
- name: Build project
run: |
mvn --batch-mode --update-snapshots -q -DskipTests install
Expand Down Expand Up @@ -100,7 +100,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.driver }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ github.event.inputs.driver }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
continue-on-error: true
- name: Install driver as needed
run: mvn --batch-mode --update-snapshots -q -DskipTests install
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ github.event.inputs.version }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
- name: Release Maven package
uses: samuelmeuli/action-maven-publish@v1
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/third_party_libs.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ github.event.inputs.version }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
- name: Release Maven package
uses: samuelmeuli/action-maven-publish@v1
with:
Expand Down
2 changes: 1 addition & 1 deletion clickhouse-benchmark/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

<properties>
<clickhouse4j-driver.version>1.4.4</clickhouse4j-driver.version>
<native-driver.version>2.6.1</native-driver.version>
<native-driver.version>2.6.3</native-driver.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jmh.version>1.33</jmh.version>
<shade.name>benchmarks</shade.name>
Expand Down
7 changes: 5 additions & 2 deletions clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/JdbcDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ public enum JdbcDriver {
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
Constants.HTTP_PORT),
// ClickHouse JDBC Driver
ClickhouseHttpJdbc("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
ClickhouseHttpJdbc1("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_URL_CONNECTION&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
Constants.HTTP_PORT),
ClickhouseHttpJdbc2("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_CLIENT&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
Constants.HTTP_PORT),
ClickhouseGrpcJdbc("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&max_inbound_message_size=2147483647&compress=%s",
Expand Down
5 changes: 3 additions & 2 deletions clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import com.clickhouse.benchmark.BaseState;
import com.clickhouse.client.ClickHouseEnum;
import com.clickhouse.client.ClickHouseValue;
import com.clickhouse.client.data.ClickHouseBigDecimalValue;
import com.clickhouse.client.data.ClickHouseBigIntegerValue;
Expand Down Expand Up @@ -82,7 +83,7 @@ public void setupSamples() {
add(map, list, Double.class, () -> ClickHouseDoubleValue.ofNull());
add(map, list, BigInteger.class, () -> ClickHouseBigIntegerValue.ofNull());
add(map, list, BigDecimal.class, () -> ClickHouseBigDecimalValue.ofNull());
add(map, list, Enum.class, () -> ClickHouseEnumValue.ofNull());
add(map, list, Enum.class, () -> ClickHouseEnumValue.ofNull(ClickHouseEnum.EMPTY));
add(map, list, Inet4Address.class, () -> ClickHouseIpv4Value.ofNull());
add(map, list, Inet6Address.class, () -> ClickHouseIpv6Value.ofNull());

Expand Down Expand Up @@ -121,7 +122,7 @@ ClickHouseValue newValue(Class<?> clazz) {
} else if (BigDecimal.class.equals(clazz)) {
return ClickHouseBigDecimalValue.ofNull();
} else if (Enum.class.equals(clazz)) {
return ClickHouseEnumValue.ofNull();
return ClickHouseEnumValue.ofNull(ClickHouseEnum.EMPTY);
} else if (Inet4Address.class.equals(clazz)) {
return ClickHouseIpv4Value.ofNull();
} else if (Inet6Address.class.equals(clazz)) {
Expand Down
2 changes: 1 addition & 1 deletion clickhouse-client/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
26 changes: 7 additions & 19 deletions clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,8 @@ public void init(ClickHouseConfig config) {
this.config = config;
if (this.executor == null) { // only initialize once
int threads = config.getMaxThreadsPerClient();
this.executor = threads <= 0 ? ClickHouseClient.getExecutorService()
: ClickHouseUtils.newThreadPool(getClass().getSimpleName(), threads,
config.getMaxQueuedRequests());
this.executor = threads < 1 ? ClickHouseClient.getExecutorService()
: ClickHouseUtils.newThreadPool(this, threads, config.getMaxQueuedRequests());
}

initialized = true;
Expand All @@ -196,27 +195,16 @@ public final void close() {
try {
server = null;

if (executor != null) {
executor.shutdown();
}

if (connection != null) {
closeConnection(connection, false);
connection = null;
}

// shutdown* won't shutdown commonPool, so awaitTermination will always time out
// on the other hand, for a client-specific thread pool, we'd better shut it
// down for real
if (executor != null && config.getMaxThreadsPerClient() > 0
&& !executor.awaitTermination(config.getConnectionTimeout(), TimeUnit.MILLISECONDS)) {
executor.shutdownNow();
// avoid shutting down shared thread pool
if (executor != null && config.getMaxThreadsPerClient() > 0 && !executor.isTerminated()) {
executor.shutdown();
}

executor = null;
connection = null;
} catch (InterruptedException e) {
log.warn("Got interrupted when closing client", e);
Thread.currentThread().interrupt();
} catch (Exception e) {
log.warn("Exception occurred when closing client", e);
} finally {
Expand All @@ -226,7 +214,7 @@ public final void close() {
closeConnection(connection, true);
}

if (executor != null) {
if (executor != null && config.getMaxThreadsPerClient() > 0) {
executor.shutdownNow();
}
} finally {
Expand Down
25 changes: 9 additions & 16 deletions clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import java.util.Objects;
import java.util.ServiceLoader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;

import com.clickhouse.client.config.ClickHouseOption;
import com.clickhouse.client.config.ClickHouseDefaults;
Expand All @@ -23,23 +22,17 @@ public class ClickHouseClientBuilder {
static {
int maxThreads = (int) ClickHouseDefaults.MAX_THREADS.getEffectiveDefaultValue();
int maxRequests = (int) ClickHouseDefaults.MAX_REQUESTS.getEffectiveDefaultValue();
long keepAliveTimeoutMs = (long) ClickHouseDefaults.THREAD_KEEPALIVE_TIMEOUT.getEffectiveDefaultValue();

if (maxThreads <= 0 && maxRequests <= 0) {
// java -XX:+UnlockDiagnosticVMOptions -XX:NativeMemoryTracking=summary
// -XX:+PrintNMTStatistics -version
defaultExecutor = ForkJoinPool.commonPool();
} else {
if (maxThreads <= 0) {
maxThreads = Runtime.getRuntime().availableProcessors();
}

if (maxRequests <= 0) {
maxRequests = 0;
}

defaultExecutor = ClickHouseUtils.newThreadPool(ClickHouseClient.class.getSimpleName(), maxThreads,
maxRequests);
if (maxThreads <= 0) {
maxThreads = Runtime.getRuntime().availableProcessors();
}
if (maxRequests <= 0) {
maxRequests = 0;
}

defaultExecutor = ClickHouseUtils.newThreadPool(ClickHouseClient.class.getSimpleName(), maxThreads,
maxThreads * 2, maxRequests, keepAliveTimeoutMs, false);
}

protected ClickHouseConfig config;
Expand Down
122 changes: 69 additions & 53 deletions clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,69 +36,76 @@ public final class ClickHouseColumn implements Serializable {
private int scale;
private List<ClickHouseColumn> nested;
private List<String> parameters;
private ClickHouseEnum enumConstants;

private int arrayLevel;
private ClickHouseColumn arrayBaseColumn;

private static ClickHouseColumn update(ClickHouseColumn column) {
column.enumConstants = ClickHouseEnum.EMPTY;
int size = column.parameters.size();
switch (column.dataType) {
case Array:
column.arrayLevel = 1;
column.arrayBaseColumn = column.nested.get(0);
while (column.arrayLevel < 255) {
if (column.arrayBaseColumn.dataType == ClickHouseDataType.Array) {
column.arrayLevel++;
column.arrayBaseColumn = column.arrayBaseColumn.nested.get(0);
} else {
break;
case Array:
column.arrayLevel = 1;
column.arrayBaseColumn = column.nested.get(0);
while (column.arrayLevel < 255) {
if (column.arrayBaseColumn.dataType == ClickHouseDataType.Array) {
column.arrayLevel++;
column.arrayBaseColumn = column.arrayBaseColumn.nested.get(0);
} else {
break;
}
}
}
break;
case DateTime:
if (size >= 2) { // same as DateTime64
column.scale = Integer.parseInt(column.parameters.get(0));
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
} else if (size == 1) { // same as DateTime32
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime32:
if (size > 0) {
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime64:
if (size > 0) {
break;
case Enum:
case Enum8:
case Enum16:
column.enumConstants = new ClickHouseEnum(column.parameters);
break;
case DateTime:
if (size >= 2) { // same as DateTime64
column.scale = Integer.parseInt(column.parameters.get(0));
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
} else if (size == 1) { // same as DateTime32
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime32:
if (size > 0) {
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime64:
if (size > 0) {
column.scale = Integer.parseInt(column.parameters.get(0));
}
if (size > 1) {
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
}
break;
case Decimal:
if (size >= 2) {
column.precision = Integer.parseInt(column.parameters.get(0));
column.scale = Integer.parseInt(column.parameters.get(1));
}
break;
case Decimal32:
case Decimal64:
case Decimal128:
case Decimal256:
column.scale = Integer.parseInt(column.parameters.get(0));
}
if (size > 1) {
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
}
break;
case Decimal:
if (size >= 2) {
break;
case FixedString:
column.precision = Integer.parseInt(column.parameters.get(0));
column.scale = Integer.parseInt(column.parameters.get(1));
}
break;
case Decimal32:
case Decimal64:
case Decimal128:
case Decimal256:
column.scale = Integer.parseInt(column.parameters.get(0));
break;
case FixedString:
column.precision = Integer.parseInt(column.parameters.get(0));
break;
default:
break;
break;
default:
break;
}

return column;
Expand Down Expand Up @@ -395,6 +402,11 @@ public boolean isArray() {
return dataType == ClickHouseDataType.Array;
}

public boolean isEnum() {
return dataType == ClickHouseDataType.Enum || dataType == ClickHouseDataType.Enum8
|| dataType == ClickHouseDataType.Enum16;
}

public boolean isMap() {
return dataType == ClickHouseDataType.Map;
}
Expand All @@ -419,6 +431,10 @@ public ClickHouseDataType getDataType() {
return dataType;
}

public ClickHouseEnum getEnumConstants() {
return enumConstants;
}

public String getOriginalTypeName() {
return originalTypeName;
}
Expand Down
3 changes: 2 additions & 1 deletion clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,14 @@ public enum ClickHouseDataType {
UInt64(Long.class, false, true, false, 8, 20, 0, 0, 0, "BIGINT UNSIGNED"),
UInt128(BigInteger.class, false, true, false, 16, 39, 0, 0, 0),
UInt256(BigInteger.class, false, true, false, 32, 78, 0, 0, 0), Int8(Byte.class, false, true, true, 1, 3, 0, 0, 0,
"BOOL", "BOOLEAN", "BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"),
"BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"),
Int16(Short.class, false, true, true, 2, 5, 0, 0, 0, "SMALLINT", "SMALLINT SIGNED"),
Int32(Integer.class, false, true, true, 4, 10, 0, 0, 0, "INT", "INTEGER", "MEDIUMINT", "INT SIGNED",
"INTEGER SIGNED", "MEDIUMINT SIGNED"),
Int64(Long.class, false, true, true, 8, 19, 0, 0, 0, "BIGINT", "BIGINT SIGNED"),
Int128(BigInteger.class, false, true, true, 16, 39, 0, 0, 0),
Int256(BigInteger.class, false, true, true, 32, 77, 0, 0, 0),
Bool(Boolean.class, false, false, true, 1, 3, 0, 0, 0, "BOOLEAN"),
Date(LocalDate.class, false, false, false, 2, 10, 0, 0, 0),
Date32(LocalDate.class, false, false, false, 4, 10, 0, 0, 0),
DateTime(LocalDateTime.class, true, false, false, 0, 29, 0, 0, 9, "TIMESTAMP"),
Expand Down
Loading

0 comments on commit 48a6281

Please sign in to comment.