Fix mssql: OLE DB provider "STREAM" for linked server "(null)" return…

…ed message "Requested conversion is not supported.". (#110) Add param for mssql: stringmaxlength=8000; Fixed fixedstring size parsing and using
ClickHouse · Aug 22, 2018 · 049bc08 · 049bc08
1 parent 8da6769
commit 049bc08
Show file tree

Hide file tree

Showing 12 changed files with 78 additions and 44 deletions.
diff --git a/driver/config.cpp b/driver/config.cpp
@@ -20,6 +20,7 @@ ConnInfo::ConnInfo()
     ZERO_FIELD(sslmode);
     ZERO_FIELD(onlyread);
     ZERO_FIELD(timeout);
+    ZERO_FIELD(stringmaxlength);
     ZERO_FIELD(show_system_tables);
     ZERO_FIELD(translation_dll);
     ZERO_FIELD(translation_option);
@@ -42,6 +43,7 @@ void getDSNinfo(ConnInfo * ci, bool overwrite)
     GET_CONFIG(password, INI_PASSWORD, "");
     GET_CONFIG(timeout, INI_TIMEOUT, "30");
     GET_CONFIG(sslmode, INI_SSLMODE, "");
+    GET_CONFIG(stringmaxlength, INI_STRINGMAXLENGTH, "1048575");
 
 #undef GET_CONFIG
 

diff --git a/driver/config.h b/driver/config.h
@@ -16,6 +16,7 @@
 #define INI_READONLY        TEXT("ReadOnly")    /* Database is read only */
 #define INI_TIMEOUT         TEXT("Timeout")
 #define INI_SSLMODE         TEXT("SSLMode")
+#define INI_STRINGMAXLENGTH TEXT("StringMaxLength")
 
 #ifndef WIN32
 #   define ODBC_INI         TEXT(".odbc.ini")
@@ -42,6 +43,7 @@ struct ConnInfo
     MYTCHAR       sslmode[16];
     MYTCHAR       onlyread[SMALL_REGISTRY_LEN];
     MYTCHAR       timeout[SMALL_REGISTRY_LEN];
+    MYTCHAR       stringmaxlength[SMALL_REGISTRY_LEN];
     MYTCHAR       show_system_tables[SMALL_REGISTRY_LEN];
     MYTCHAR       translation_dll[MEDIUM_REGISTRY_LEN];
     MYTCHAR       translation_option[SMALL_REGISTRY_LEN];

diff --git a/driver/connection.cpp b/driver/connection.cpp
@@ -135,6 +135,14 @@ void Connection::init(const std::string & connection_string) {
                 throw std::runtime_error("Cannot parse timeout.");
             }
         }
+        else if (key_lower == "stringmaxlength") {
+            int int_val = 0;
+            if (Poco::NumberParser::tryParse(current_value.toString(), int_val))
+                stringmaxlength = int_val;
+            else {
+                throw std::runtime_error("Cannot parse stringmaxlength.");
+            }
+        }
         else if (key_lower == "dsn")
             data_source = current_value.toString();
     }
@@ -151,20 +159,29 @@ void Connection::loadConfiguration() {
     getDSNinfo(&ci, true);
 
     if (!port && ci.port[0] != 0) {
-        int int_port = 0;
-        if (Poco::NumberParser::tryParse(stringFromMYTCHAR(ci.port), int_port))
-            port = int_port;
-        else
-            throw std::runtime_error(("Cannot parse port number [" + stringFromMYTCHAR(ci.port) + "].").c_str());
+        const std::string string = stringFromMYTCHAR(ci.port);
+        if (!string.empty()) {
+            int tmp = 0;
+            if (!Poco::NumberParser::tryParse(string, tmp))
+                throw std::runtime_error(("Cannot parse port number [" + string + "].").c_str());
+            port = tmp;
+        }
     }
     if (timeout == 0) {
-        const std::string timeout_string = stringFromMYTCHAR(ci.timeout);
-        if (!timeout_string.empty()) {
-            if (!Poco::NumberParser::tryParse(timeout_string, this->timeout))
-                throw std::runtime_error("Cannot parse connection timeout value.");
+        const std::string string = stringFromMYTCHAR(ci.timeout);
+        if (!string.empty()) {
+            if (!Poco::NumberParser::tryParse(string, this->timeout))
+                throw std::runtime_error("Cannot parse connection timeout value [" + string + "].");
             this->connection_timeout = this->timeout;
         }
     }
+    if (stringmaxlength == 0) {
+        const std::string string = stringFromMYTCHAR(ci.stringmaxlength);
+        if (!string.empty()) {
+            if (!Poco::NumberParser::tryParse(string, this->stringmaxlength))
+                throw std::runtime_error("Cannot parse stringmaxlength value [" + string + "].");
+        }
+    }
 
     if (server.empty())
         server = stringFromMYTCHAR(ci.server);
@@ -187,6 +204,8 @@ void Connection::setDefaults() {
         server = "localhost";
     if (port == 0)
         port = (proto == "https" ? 8443 : 8123);
+    if (stringmaxlength == 0)
+        stringmaxlength = Environment::string_max_size;
     if (user.empty())
         user = "default";
     if (database.empty())

diff --git a/driver/connection.h b/driver/connection.h
@@ -19,6 +19,7 @@ struct Connection {
     uint16_t port = 0;
     int timeout = 0;
     int connection_timeout = 0;
+    int32_t stringmaxlength = 0;
 
     std::unique_ptr<Poco::Net::HTTPClientSession> session;
     DiagnosticRecord diagnostic_record;

diff --git a/driver/environment.cpp b/driver/environment.cpp
@@ -33,11 +33,11 @@ const std::map<std::string, TypeInfo> Environment::types_info = {
     {"Int64", TypeInfo{"BIGINT", false, SQL_BIGINT, 1 + 19, 8}},
     {"Float32", TypeInfo{"REAL", false, SQL_REAL, 7, 4}},
     {"Float64", TypeInfo{"DOUBLE", false, SQL_DOUBLE, 15, 8}},
-    {"String", TypeInfo{"TEXT", true, SQL_VARCHAR, 0xFFFFFF, (1 << 20)}},
-    {"FixedString", TypeInfo{"TEXT", true, SQL_VARCHAR, 0xFFFFFF, (1 << 20)}},
+    {"String", TypeInfo{"TEXT", true, SQL_VARCHAR, Environment::string_max_size, Environment::string_max_size}},
+    {"FixedString", TypeInfo{"TEXT", true, SQL_VARCHAR, Environment::string_max_size, Environment::string_max_size}},
     {"Date", TypeInfo{"DATE", true, SQL_TYPE_DATE, 10, 6}},
     {"DateTime", TypeInfo{"TIMESTAMP", true, SQL_TYPE_TIMESTAMP, 19, 16}},
-    {"Array", TypeInfo{"TEXT", true, SQL_VARCHAR, 0xFFFFFF, (1 << 20)}},
+    {"Array", TypeInfo{"TEXT", true, SQL_VARCHAR, Environment::string_max_size, Environment::string_max_size}},
 };
 
 Environment::Environment() {

diff --git a/driver/environment.h b/driver/environment.h
@@ -9,8 +9,8 @@ struct TypeInfo {
     std::string sql_type_name;
     bool is_unsigned;
     SQLSMALLINT sql_type;
-    size_t column_size;
-    size_t octet_length;
+    int32_t column_size;
+    int32_t octet_length;
 
     inline bool IsIntegerType() const {
         return sql_type == SQL_TINYINT || sql_type == SQL_SMALLINT || sql_type == SQL_INTEGER || sql_type == SQL_BIGINT;
@@ -26,6 +26,7 @@ struct Environment {
     Environment();
     ~Environment();
 
+    static const auto string_max_size = 0xFFFFFF;
     static const std::map<std::string, TypeInfo> types_info;
 
     SQLUINTEGER metadata_id = SQL_FALSE;

diff --git a/driver/odbc.cpp b/driver/odbc.cpp
@@ -35,15 +35,15 @@ RETCODE SQL_API FUNCTION_MAYBE_W(SQLConnect)(HDBC connection_handle,
     SQLTCHAR * password,
     SQLSMALLINT password_size)
 {
-    LOG(__FUNCTION__ << " dsn=" << dsn << " dsn_size=" << dsn_size << " user=" << user << " user_size=" << " password=" << password << " password_size=" << password_size);
+    //LOG(__FUNCTION__ << " dsn_size=" << dsn_size << " dsn=" << dsn << " user_size=" << user_size << " user=" << user << " password_size=" << password_size << " password=" << password);
 
     return doWith<Connection>(connection_handle, [&](Connection & connection) {
 
         std::string dsn_str = stringFromSQLSymbols(dsn, dsn_size);
         std::string user_str = stringFromSQLSymbols(user, user_size);
         std::string password_str = stringFromSQLSymbols(password, password_size);
 
-        //LOG(__FUNCTION__ << " dsn="<< dsn_str <<", user="<< user_str << ", pwd="<< password_str);
+        LOG(__FUNCTION__ << " dsn=" << dsn_str << " user=" << user_str << " pwd=" << password_str);
 
         connection.init(dsn_str, 0, user_str, password_str, "");
         return SQL_SUCCESS;
@@ -209,16 +209,7 @@ RETCODE SQL_API SQLColAttribute(HSTMT statement_handle,
                 break;
             case SQL_DESC_LENGTH:
                 if (type_info.IsStringType())
-                {
-                    if (column_info.fixed_size)
-                    {
-                        num_value = column_info.fixed_size;
-                    }
-                    else
-                    {
-                        num_value = column_info.display_size;
-                    }
-                }
+                    num_value = std::min<int32_t>(statement.connection.stringmaxlength, column_info.fixed_size ? column_info.fixed_size : column_info.display_size);
                 break;
             case SQL_DESC_LITERAL_PREFIX:
                 break;
@@ -234,25 +225,16 @@ RETCODE SQL_API SQLColAttribute(HSTMT statement_handle,
                 break;
             case SQL_DESC_OCTET_LENGTH:
                 if (type_info.IsStringType())
-                {
-                    if (column_info.fixed_size)
-                        num_value = column_info.fixed_size * SIZEOF_CHAR;
-                    else
-                        num_value = column_info.display_size * SIZEOF_CHAR;
-                }
+                    num_value = std::min<int32_t>(statement.connection.stringmaxlength, column_info.fixed_size ? column_info.fixed_size : column_info.display_size) * SIZEOF_CHAR;
                 else
-                {
                     num_value = type_info.octet_length;
-                }
                 break;
             case SQL_DESC_PRECISION:
                 num_value = 0;
                 break;
             case SQL_DESC_NUM_PREC_RADIX:
                 if (type_info.IsIntegerType())
-                {
                     num_value = 10;
-                }
                 break;
             case SQL_DESC_SCALE:
                 break;
@@ -317,7 +299,7 @@ RETCODE SQL_API FUNCTION_MAYBE_W(SQLDescribeCol)(HSTMT statement_handle,
         if (out_type)
             *out_type = type_info.sql_type;
         if (out_column_size)
-            *out_column_size = type_info.column_size;
+            *out_column_size = std::min<int32_t>(statement.connection.stringmaxlength, column_info.fixed_size ? column_info.fixed_size : type_info.column_size);
         if (out_decimal_digits)
             *out_decimal_digits = 0;
         if (out_is_nullable)
@@ -345,10 +327,10 @@ RETCODE SQL_API impl_SQLGetData(HSTMT statement_handle,
 
         size_t column_idx = column_or_param_number - 1;
 
-        LOG("column: " << column_idx << ", target_type: " << target_type << ", out_value_max_size: " << out_value_max_size);
-
         const Field & field = statement.current_row.data[column_idx];
 
+        LOG("column: " << column_idx << ", target_type: " << target_type << ", out_value_max_size: " << out_value_max_size << " data=" << field.data);
+
         switch (target_type)
         {
             case SQL_C_CHAR:

diff --git a/driver/result_set.cpp b/driver/result_set.cpp
@@ -116,7 +116,10 @@ void assignTypeInfo(const TypeAst & ast, ColumnInfo * info)
     if (ast.meta == TypeAst::Terminal)
     {
         info->type_without_parameters = ast.name;
-        info->fixed_size = ast.size;
+        if (ast.elements.size() == 1)
+            info->fixed_size = ast.elements.front().size;
+        //  info->fixed_size = ast.size;
+        // LOG("assignTypeInfo: info->type_without_parameters=" << info->type_without_parameters << " info->fixed_size?=" << info->fixed_size << " ast.elements.size()=" << ast.elements.size());
     }
     else if (ast.meta == TypeAst::Nullable)
     {
@@ -164,7 +167,7 @@ void ResultSet::init(Statement * statement_, IResultMutatorPtr mutator_)
             }
         }
 
-        LOG("Row " << i << " name=" << columns_info[i].name << " type=" << columns_info[i].type << " -> " << columns_info[i].type << " typenoparams=" << columns_info[i].type_without_parameters);
+        LOG("Row " << i << " name=" << columns_info[i].name << " type=" << columns_info[i].type << " -> " << columns_info[i].type << " typenoparams=" << columns_info[i].type_without_parameters << " fixedsize=" << columns_info[i].fixed_size);
     }
 
     mutator->UpdateColumnInfo(&columns_info);

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -9,4 +9,7 @@ if (NOT (CMAKE_BUILD_TYPE_UC STREQUAL "TSAN" AND APPLE))
         add_test(NAME "test.pl-w" COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/test.pl ${TEST_DSN_W})
         add_test(NAME "test.py-w" COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/test.py ${TEST_DSN_W})
     endif ()
+    if(COMMAND sqlcmd) # MS SQL server need change server in file:
+        add_test(NAME "sqlcmd" COMMAND sqlcmd -i ${CMAKE_CURRENT_SOURCE_DIR}/mssql.linked.server.sql)
+    endif ()
 endif ()
diff --git a/test/mssql.linked.server.sql b/test/mssql.linked.server.sql
@@ -1,14 +1,26 @@
--- sqlcmd -S .\MSSQLSERVER -i mssql.linked.server.sql
+-- net stop MSSQLSERVER && net start MSSQLSERVER
 -- sqlcmd -i mssql.linked.server.sql
 
+EXEC master.dbo.sp_dropserver N'clickhouse_link_test';
 EXEC master.dbo.sp_addlinkedserver
         @server = N'clickhouse_link_test'
        ,@srvproduct=N'Clickhouse'
        ,@provider=N'MSDASQL'
-       ,@provstr=N'Driver={ClickHouse Unicode};SERVER=ch1.setun.net;DATABASE=test;'
+       ,@provstr=N'Driver={ClickHouse Unicode};SERVER=!!!!!!!your.server.name.com!!!!!!!;DATABASE=system;stringmaxlength=8000;'
 
 EXEC sp_serveroption 'clickhouse_link_test','rpc','true';
 EXEC sp_serveroption 'clickhouse_link_test','rpc out','true';
 EXEC('select * from system.numbers limit 10;') at [clickhouse_link_test];
 EXEC("select 'Just string'") at [clickhouse_link_test];
 EXEC('select name from system.databases;') at [clickhouse_link_test];
+EXEC('select * from system.build_options;') at [clickhouse_link_test];
+
+
+exec('CREATE TABLE IF NOT EXISTS test.fixedstring ( xx FixedString(100)) ENGINE = Memory;') at [clickhouse_link_test];
+exec(N'INSERT INTO test.fixedstring VALUES (''a''), (''abcdefg''), (''абвгдеёжзийклмнопрстуфх'');') at [clickhouse_link_test];
+--exec('INSERT INTO test.fixedstring VALUES (''a''),(''abcdefg'');') at [clickhouse_link_test];
+exec('select xx as x from test.fixedstring;') at [clickhouse_link_test];
+exec('DROP TABLE test.fixedstring;') at [clickhouse_link_test];
+
+exec('SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,-2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,-9223372036854775807,-9223372036854775808,-9223372036854775809,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615,18446744073709551616,18446744073709551617;') at [clickhouse_link_test];
+exec('SELECT *, (CASE WHEN (number == 1) THEN ''o'' WHEN (number == 2) THEN ''two long string'' WHEN (number == 3) THEN ''r'' ELSE ''-'' END)  FROM system.numbers LIMIT 5;') at [clickhouse_link_test];
diff --git a/test/test.py b/test/test.py
@@ -22,7 +22,8 @@
     connection.setencoding(encoding='utf-8')
 
 def query(q):
-    print(q + " :")
+    sys.stdout.write(q)
+    print(" :")
     cursor = connection.cursor()
     cursor.execute(q)
     rows = cursor.fetchall()

diff --git a/test/test.sh b/test/test.sh
@@ -24,6 +24,7 @@ function q {
     echo "$*" | $RUNNER $DSN $RUNNER_PARAMS
 }
 
+
 q "SELECT * FROM system.build_options;"
 q "CREATE DATABASE IF NOT EXISTS test;"
 q "DROP TABLE IF EXISTS test.odbc1;"
@@ -133,6 +134,13 @@ q $"SELECT 'абвгдеёжзийклмнопрстуфхцчшщъыьэюяА
 q $"SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,-2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,-9223372036854775807,-9223372036854775808,-9223372036854775809,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615,18446744073709551616,18446744073709551617"
 q $"SELECT 2147483647, 2147483648, 2147483647+1, 2147483647+10, 4294967295"
 
+
+q "CREATE TABLE IF NOT EXISTS test.fixedstring ( xx FixedString(100)) ENGINE = Memory;"
+q "INSERT INTO test.fixedstring VALUES ('a'), ('abcdefg'), ('абвгдеёжзийклмнопрстуфхцч')";
+q "select xx as x from test.fixedstring;"
+q "DROP TABLE test.fixedstring;"
+
+
 q 'DROP TABLE IF EXISTS test.increment;'
 q 'CREATE TABLE test.increment (n UInt64) engine Log;'