From 049bc08e9e86f5c8fdb2267213f817c35b2289f4 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 23 Aug 2018 00:12:24 +0300 Subject: [PATCH] Fix mssql: OLE DB provider "STREAM" for linked server "(null)" returned message "Requested conversion is not supported.". (#110) Add param for mssql: stringmaxlength=8000; Fixed fixedstring size parsing and using --- driver/config.cpp | 2 ++ driver/config.h | 2 ++ driver/connection.cpp | 37 +++++++++++++++++++++++++++--------- driver/connection.h | 1 + driver/environment.cpp | 6 +++--- driver/environment.h | 5 +++-- driver/odbc.cpp | 32 +++++++------------------------ driver/result_set.cpp | 7 +++++-- test/CMakeLists.txt | 3 +++ test/mssql.linked.server.sql | 16 ++++++++++++++-- test/test.py | 3 ++- test/test.sh | 8 ++++++++ 12 files changed, 78 insertions(+), 44 deletions(-) diff --git a/driver/config.cpp b/driver/config.cpp index 586b71e3f..19afe5492 100644 --- a/driver/config.cpp +++ b/driver/config.cpp @@ -20,6 +20,7 @@ ConnInfo::ConnInfo() ZERO_FIELD(sslmode); ZERO_FIELD(onlyread); ZERO_FIELD(timeout); + ZERO_FIELD(stringmaxlength); ZERO_FIELD(show_system_tables); ZERO_FIELD(translation_dll); ZERO_FIELD(translation_option); @@ -42,6 +43,7 @@ void getDSNinfo(ConnInfo * ci, bool overwrite) GET_CONFIG(password, INI_PASSWORD, ""); GET_CONFIG(timeout, INI_TIMEOUT, "30"); GET_CONFIG(sslmode, INI_SSLMODE, ""); + GET_CONFIG(stringmaxlength, INI_STRINGMAXLENGTH, "1048575"); #undef GET_CONFIG diff --git a/driver/config.h b/driver/config.h index 84c2d1195..63a31dbe2 100644 --- a/driver/config.h +++ b/driver/config.h @@ -16,6 +16,7 @@ #define INI_READONLY TEXT("ReadOnly") /* Database is read only */ #define INI_TIMEOUT TEXT("Timeout") #define INI_SSLMODE TEXT("SSLMode") +#define INI_STRINGMAXLENGTH TEXT("StringMaxLength") #ifndef WIN32 # define ODBC_INI TEXT(".odbc.ini") @@ -42,6 +43,7 @@ struct ConnInfo MYTCHAR sslmode[16]; MYTCHAR onlyread[SMALL_REGISTRY_LEN]; MYTCHAR timeout[SMALL_REGISTRY_LEN]; + MYTCHAR stringmaxlength[SMALL_REGISTRY_LEN]; MYTCHAR show_system_tables[SMALL_REGISTRY_LEN]; MYTCHAR translation_dll[MEDIUM_REGISTRY_LEN]; MYTCHAR translation_option[SMALL_REGISTRY_LEN]; diff --git a/driver/connection.cpp b/driver/connection.cpp index e6117a6a3..07beebb84 100644 --- a/driver/connection.cpp +++ b/driver/connection.cpp @@ -135,6 +135,14 @@ void Connection::init(const std::string & connection_string) { throw std::runtime_error("Cannot parse timeout."); } } + else if (key_lower == "stringmaxlength") { + int int_val = 0; + if (Poco::NumberParser::tryParse(current_value.toString(), int_val)) + stringmaxlength = int_val; + else { + throw std::runtime_error("Cannot parse stringmaxlength."); + } + } else if (key_lower == "dsn") data_source = current_value.toString(); } @@ -151,20 +159,29 @@ void Connection::loadConfiguration() { getDSNinfo(&ci, true); if (!port && ci.port[0] != 0) { - int int_port = 0; - if (Poco::NumberParser::tryParse(stringFromMYTCHAR(ci.port), int_port)) - port = int_port; - else - throw std::runtime_error(("Cannot parse port number [" + stringFromMYTCHAR(ci.port) + "].").c_str()); + const std::string string = stringFromMYTCHAR(ci.port); + if (!string.empty()) { + int tmp = 0; + if (!Poco::NumberParser::tryParse(string, tmp)) + throw std::runtime_error(("Cannot parse port number [" + string + "].").c_str()); + port = tmp; + } } if (timeout == 0) { - const std::string timeout_string = stringFromMYTCHAR(ci.timeout); - if (!timeout_string.empty()) { - if (!Poco::NumberParser::tryParse(timeout_string, this->timeout)) - throw std::runtime_error("Cannot parse connection timeout value."); + const std::string string = stringFromMYTCHAR(ci.timeout); + if (!string.empty()) { + if (!Poco::NumberParser::tryParse(string, this->timeout)) + throw std::runtime_error("Cannot parse connection timeout value [" + string + "]."); this->connection_timeout = this->timeout; } } + if (stringmaxlength == 0) { + const std::string string = stringFromMYTCHAR(ci.stringmaxlength); + if (!string.empty()) { + if (!Poco::NumberParser::tryParse(string, this->stringmaxlength)) + throw std::runtime_error("Cannot parse stringmaxlength value [" + string + "]."); + } + } if (server.empty()) server = stringFromMYTCHAR(ci.server); @@ -187,6 +204,8 @@ void Connection::setDefaults() { server = "localhost"; if (port == 0) port = (proto == "https" ? 8443 : 8123); + if (stringmaxlength == 0) + stringmaxlength = Environment::string_max_size; if (user.empty()) user = "default"; if (database.empty()) diff --git a/driver/connection.h b/driver/connection.h index 669913141..2646356fb 100644 --- a/driver/connection.h +++ b/driver/connection.h @@ -19,6 +19,7 @@ struct Connection { uint16_t port = 0; int timeout = 0; int connection_timeout = 0; + int32_t stringmaxlength = 0; std::unique_ptr session; DiagnosticRecord diagnostic_record; diff --git a/driver/environment.cpp b/driver/environment.cpp index c529b694d..b95534fd8 100644 --- a/driver/environment.cpp +++ b/driver/environment.cpp @@ -33,11 +33,11 @@ const std::map Environment::types_info = { {"Int64", TypeInfo{"BIGINT", false, SQL_BIGINT, 1 + 19, 8}}, {"Float32", TypeInfo{"REAL", false, SQL_REAL, 7, 4}}, {"Float64", TypeInfo{"DOUBLE", false, SQL_DOUBLE, 15, 8}}, - {"String", TypeInfo{"TEXT", true, SQL_VARCHAR, 0xFFFFFF, (1 << 20)}}, - {"FixedString", TypeInfo{"TEXT", true, SQL_VARCHAR, 0xFFFFFF, (1 << 20)}}, + {"String", TypeInfo{"TEXT", true, SQL_VARCHAR, Environment::string_max_size, Environment::string_max_size}}, + {"FixedString", TypeInfo{"TEXT", true, SQL_VARCHAR, Environment::string_max_size, Environment::string_max_size}}, {"Date", TypeInfo{"DATE", true, SQL_TYPE_DATE, 10, 6}}, {"DateTime", TypeInfo{"TIMESTAMP", true, SQL_TYPE_TIMESTAMP, 19, 16}}, - {"Array", TypeInfo{"TEXT", true, SQL_VARCHAR, 0xFFFFFF, (1 << 20)}}, + {"Array", TypeInfo{"TEXT", true, SQL_VARCHAR, Environment::string_max_size, Environment::string_max_size}}, }; Environment::Environment() { diff --git a/driver/environment.h b/driver/environment.h index 477024006..e780058f9 100644 --- a/driver/environment.h +++ b/driver/environment.h @@ -9,8 +9,8 @@ struct TypeInfo { std::string sql_type_name; bool is_unsigned; SQLSMALLINT sql_type; - size_t column_size; - size_t octet_length; + int32_t column_size; + int32_t octet_length; inline bool IsIntegerType() const { return sql_type == SQL_TINYINT || sql_type == SQL_SMALLINT || sql_type == SQL_INTEGER || sql_type == SQL_BIGINT; @@ -26,6 +26,7 @@ struct Environment { Environment(); ~Environment(); + static const auto string_max_size = 0xFFFFFF; static const std::map types_info; SQLUINTEGER metadata_id = SQL_FALSE; diff --git a/driver/odbc.cpp b/driver/odbc.cpp index 0bd6997c6..9dd73dc93 100644 --- a/driver/odbc.cpp +++ b/driver/odbc.cpp @@ -35,7 +35,7 @@ RETCODE SQL_API FUNCTION_MAYBE_W(SQLConnect)(HDBC connection_handle, SQLTCHAR * password, SQLSMALLINT password_size) { - LOG(__FUNCTION__ << " dsn=" << dsn << " dsn_size=" << dsn_size << " user=" << user << " user_size=" << " password=" << password << " password_size=" << password_size); + //LOG(__FUNCTION__ << " dsn_size=" << dsn_size << " dsn=" << dsn << " user_size=" << user_size << " user=" << user << " password_size=" << password_size << " password=" << password); return doWith(connection_handle, [&](Connection & connection) { @@ -43,7 +43,7 @@ RETCODE SQL_API FUNCTION_MAYBE_W(SQLConnect)(HDBC connection_handle, std::string user_str = stringFromSQLSymbols(user, user_size); std::string password_str = stringFromSQLSymbols(password, password_size); - //LOG(__FUNCTION__ << " dsn="<< dsn_str <<", user="<< user_str << ", pwd="<< password_str); + LOG(__FUNCTION__ << " dsn=" << dsn_str << " user=" << user_str << " pwd=" << password_str); connection.init(dsn_str, 0, user_str, password_str, ""); return SQL_SUCCESS; @@ -209,16 +209,7 @@ RETCODE SQL_API SQLColAttribute(HSTMT statement_handle, break; case SQL_DESC_LENGTH: if (type_info.IsStringType()) - { - if (column_info.fixed_size) - { - num_value = column_info.fixed_size; - } - else - { - num_value = column_info.display_size; - } - } + num_value = std::min(statement.connection.stringmaxlength, column_info.fixed_size ? column_info.fixed_size : column_info.display_size); break; case SQL_DESC_LITERAL_PREFIX: break; @@ -234,25 +225,16 @@ RETCODE SQL_API SQLColAttribute(HSTMT statement_handle, break; case SQL_DESC_OCTET_LENGTH: if (type_info.IsStringType()) - { - if (column_info.fixed_size) - num_value = column_info.fixed_size * SIZEOF_CHAR; - else - num_value = column_info.display_size * SIZEOF_CHAR; - } + num_value = std::min(statement.connection.stringmaxlength, column_info.fixed_size ? column_info.fixed_size : column_info.display_size) * SIZEOF_CHAR; else - { num_value = type_info.octet_length; - } break; case SQL_DESC_PRECISION: num_value = 0; break; case SQL_DESC_NUM_PREC_RADIX: if (type_info.IsIntegerType()) - { num_value = 10; - } break; case SQL_DESC_SCALE: break; @@ -317,7 +299,7 @@ RETCODE SQL_API FUNCTION_MAYBE_W(SQLDescribeCol)(HSTMT statement_handle, if (out_type) *out_type = type_info.sql_type; if (out_column_size) - *out_column_size = type_info.column_size; + *out_column_size = std::min(statement.connection.stringmaxlength, column_info.fixed_size ? column_info.fixed_size : type_info.column_size); if (out_decimal_digits) *out_decimal_digits = 0; if (out_is_nullable) @@ -345,10 +327,10 @@ RETCODE SQL_API impl_SQLGetData(HSTMT statement_handle, size_t column_idx = column_or_param_number - 1; - LOG("column: " << column_idx << ", target_type: " << target_type << ", out_value_max_size: " << out_value_max_size); - const Field & field = statement.current_row.data[column_idx]; + LOG("column: " << column_idx << ", target_type: " << target_type << ", out_value_max_size: " << out_value_max_size << " data=" << field.data); + switch (target_type) { case SQL_C_CHAR: diff --git a/driver/result_set.cpp b/driver/result_set.cpp index 37b306731..65af69366 100644 --- a/driver/result_set.cpp +++ b/driver/result_set.cpp @@ -116,7 +116,10 @@ void assignTypeInfo(const TypeAst & ast, ColumnInfo * info) if (ast.meta == TypeAst::Terminal) { info->type_without_parameters = ast.name; - info->fixed_size = ast.size; + if (ast.elements.size() == 1) + info->fixed_size = ast.elements.front().size; + // info->fixed_size = ast.size; + // LOG("assignTypeInfo: info->type_without_parameters=" << info->type_without_parameters << " info->fixed_size?=" << info->fixed_size << " ast.elements.size()=" << ast.elements.size()); } else if (ast.meta == TypeAst::Nullable) { @@ -164,7 +167,7 @@ void ResultSet::init(Statement * statement_, IResultMutatorPtr mutator_) } } - LOG("Row " << i << " name=" << columns_info[i].name << " type=" << columns_info[i].type << " -> " << columns_info[i].type << " typenoparams=" << columns_info[i].type_without_parameters); + LOG("Row " << i << " name=" << columns_info[i].name << " type=" << columns_info[i].type << " -> " << columns_info[i].type << " typenoparams=" << columns_info[i].type_without_parameters << " fixedsize=" << columns_info[i].fixed_size); } mutator->UpdateColumnInfo(&columns_info); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e1b01783b..a04416414 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,4 +9,7 @@ if (NOT (CMAKE_BUILD_TYPE_UC STREQUAL "TSAN" AND APPLE)) add_test(NAME "test.pl-w" COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/test.pl ${TEST_DSN_W}) add_test(NAME "test.py-w" COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/test.py ${TEST_DSN_W}) endif () + if(COMMAND sqlcmd) # MS SQL server need change server in file: + add_test(NAME "sqlcmd" COMMAND sqlcmd -i ${CMAKE_CURRENT_SOURCE_DIR}/mssql.linked.server.sql) + endif () endif () diff --git a/test/mssql.linked.server.sql b/test/mssql.linked.server.sql index 172b7d1b8..92887939a 100644 --- a/test/mssql.linked.server.sql +++ b/test/mssql.linked.server.sql @@ -1,14 +1,26 @@ --- sqlcmd -S .\MSSQLSERVER -i mssql.linked.server.sql +-- net stop MSSQLSERVER && net start MSSQLSERVER -- sqlcmd -i mssql.linked.server.sql +EXEC master.dbo.sp_dropserver N'clickhouse_link_test'; EXEC master.dbo.sp_addlinkedserver @server = N'clickhouse_link_test' ,@srvproduct=N'Clickhouse' ,@provider=N'MSDASQL' - ,@provstr=N'Driver={ClickHouse Unicode};SERVER=ch1.setun.net;DATABASE=test;' + ,@provstr=N'Driver={ClickHouse Unicode};SERVER=!!!!!!!your.server.name.com!!!!!!!;DATABASE=system;stringmaxlength=8000;' EXEC sp_serveroption 'clickhouse_link_test','rpc','true'; EXEC sp_serveroption 'clickhouse_link_test','rpc out','true'; EXEC('select * from system.numbers limit 10;') at [clickhouse_link_test]; EXEC("select 'Just string'") at [clickhouse_link_test]; EXEC('select name from system.databases;') at [clickhouse_link_test]; +EXEC('select * from system.build_options;') at [clickhouse_link_test]; + + +exec('CREATE TABLE IF NOT EXISTS test.fixedstring ( xx FixedString(100)) ENGINE = Memory;') at [clickhouse_link_test]; +exec(N'INSERT INTO test.fixedstring VALUES (''a''), (''abcdefg''), (''абвгдеёжзийклмнопрстуфх'');') at [clickhouse_link_test]; +--exec('INSERT INTO test.fixedstring VALUES (''a''),(''abcdefg'');') at [clickhouse_link_test]; +exec('select xx as x from test.fixedstring;') at [clickhouse_link_test]; +exec('DROP TABLE test.fixedstring;') at [clickhouse_link_test]; + +exec('SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,-2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,-9223372036854775807,-9223372036854775808,-9223372036854775809,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615,18446744073709551616,18446744073709551617;') at [clickhouse_link_test]; +exec('SELECT *, (CASE WHEN (number == 1) THEN ''o'' WHEN (number == 2) THEN ''two long string'' WHEN (number == 3) THEN ''r'' ELSE ''-'' END) FROM system.numbers LIMIT 5;') at [clickhouse_link_test]; diff --git a/test/test.py b/test/test.py index f216097b9..3d1572123 100755 --- a/test/test.py +++ b/test/test.py @@ -22,7 +22,8 @@ connection.setencoding(encoding='utf-8') def query(q): - print(q + " :") + sys.stdout.write(q) + print(" :") cursor = connection.cursor() cursor.execute(q) rows = cursor.fetchall() diff --git a/test/test.sh b/test/test.sh index f66e571dc..883e36b37 100755 --- a/test/test.sh +++ b/test/test.sh @@ -24,6 +24,7 @@ function q { echo "$*" | $RUNNER $DSN $RUNNER_PARAMS } + q "SELECT * FROM system.build_options;" q "CREATE DATABASE IF NOT EXISTS test;" q "DROP TABLE IF EXISTS test.odbc1;" @@ -133,6 +134,13 @@ q $"SELECT 'абвгдеёжзийклмнопрстуфхцчшщъыьэюяА q $"SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,-2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,-9223372036854775807,-9223372036854775808,-9223372036854775809,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615,18446744073709551616,18446744073709551617" q $"SELECT 2147483647, 2147483648, 2147483647+1, 2147483647+10, 4294967295" + +q "CREATE TABLE IF NOT EXISTS test.fixedstring ( xx FixedString(100)) ENGINE = Memory;" +q "INSERT INTO test.fixedstring VALUES ('a'), ('abcdefg'), ('абвгдеёжзийклмнопрстуфхцч')"; +q "select xx as x from test.fixedstring;" +q "DROP TABLE test.fixedstring;" + + q 'DROP TABLE IF EXISTS test.increment;' q 'CREATE TABLE test.increment (n UInt64) engine Log;'