Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

照着原来的arrow7.0.0写了个能在win下通过的,不过没法添加lz4之类的 #4033

Open
Altina-oz opened this issue May 15, 2024 · 1 comment

Comments

@Altina-oz
Copy link
Contributor

Altina-oz commented May 15, 2024

你在什么场景下需要该功能?

库里原先的apache-arrow是7.0.0 linux版本的,仿照改写了个能在win下16.1.0版本的,不过貌似lz4之类的会挂(读写不压缩的parquet,feather,csv,compute什么的能过),其他的三方库我不确定至少lz4链接会链接不到(static, win11, msvc2022 c++20 release, xmake 2,9.1 dev, 报错见下方)

描述可能的解决方案

package("apache-arrow")
    set_description("Apache arrow")

    add_urls("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-$(version).tar.gz")
    add_versions('16.1.0', '9762d9ecc13d09de2a03f9c625a74db0d645cb012de1e9a10dfed0b4ddc09524')
    
    add_configs("csv",      {description = "CSV reader module", default = true, type = "boolean"})
    add_configs("json",     {description = "JSON reader module", default = true, type = "boolean"})
    add_configs("compute",   {description = "Build all computational kernel functions", default = true, type = "boolean"})
    add_configs("dataset",  {description = "Dataset API, implies the Filesystem API", default = true, type = "boolean"})
    --add_configs("filesystem",      {description = "Filesystem API for accessing local and remote filesystems", default = true, type = "boolean"})
    --add_configs("mimalloc", {description = "Build the Arrow mimalloc-based allocator", default = true, type = "boolean"})
    --add_configs("jemalloc", {description = "Build the Arrow jemalloc-based allocator", default = false, type = "boolean"})
    add_configs("parquet",  {description = "Apache Parquet libraries and Arrow integration", default = true, type = "boolean"})
    -- If true, arrow will look for shared libraries for third party dependency.
    -- The pyarrow python package creates shared library that links in all necessary thirdparty static libraries.
    add_configs("shared_dep", {description = "Use shared library for dependency", default = false, type = "boolean"})

    -- Some libraries are required for build with our default config settings.
    local configdeps = {
        re2 = "re2", utf8proc = "utf8proc",
        -- compression libraries
        brotli = "brotli", bz2 = "bzip2", snappy = "snappy", lz4 = "lz4", zlib = "zlib", zstd = "zstd",
    }
    for config, dep in pairs(configdeps) do
    
        add_configs(config, {description = "Enable " .. dep .. " support.", default = false, type = "boolean"})
    end

    add_deps("cmake", "boost")

    if is_plat("bsd") then
        add_syslinks("pthread", "execinfo")
    elseif is_plat("linux") then
        add_syslinks("pthread")
    elseif is_plat("windows") then 
        add_syslinks("Ole32", "shell32","kernel32","Ws2_32")
    end
    on_load(function (package)

        if package:config("shared") or not is_plat("windows") then
            package:add("links", "arrow", "arrow_bundled_dependencies")
        else 
            package:add("links","arrow_static","arrow_acero_static","arrow_bundled_dependencies")
        
        
        end
        
        if package:config("dataset") then
            if package:config("shared") or not is_plat("windows") then
                package:add("links", "arrow_dataset")
            else 
                package:add("links","arrow_dataset_static")
            end
        end

        for name, dep in pairs(configdeps) do
            if package:config(name) then
                package:add("deps", dep)
            end
        end

        if package:config("parquet")  then
            if package:config("shared") or not is_plat("windows") then
                package:add("links", "parquet")
            else 
                package:add("links","parquet_static")
            end
            
            package:add("deps", "thrift")
        end
        if package:config("json") then
            package:add("deps", "rapidjson")
        end
        
        
    end)

    on_install(function (package)
        local configs = {
            "-DARROW_BUILD_TESTS=OFF",
            "-DARROW_DEPENDENCY_SOURCE=BUNDLED",
            "-DARROW_SIMD_LEVEL=AVX2"
        }
        package:addenv("ARROW_THRIFT_BUILD_VERSION", "0.20.0")
        table.insert(configs, "-DCMAKE_BUILD_TYPE=" .. (package:debug() and "Debug" or "Release"))

        local shared = package:config("shared")
        
        if is_plat("windows") then 
            table.insert(configs, "-DARROW_STATIC -DARROW_FLIGHT_STATIC -DARROW_FLIGHT_SQL_STATIC -DARROW_BUILD_STATIC=" .. (shared and "OFF" or "ON"))
        else
            table.insert(configs, "-DARROW_BUILD_STATIC=" .. (shared and "OFF" or "ON"))
        end
        table.insert(configs, "-DARROW_BUILD_SHARED=" .. (shared and "ON" or "OFF"))
        table.insert(configs, "-DARROW_DEPENDENCY_USE_SHARED=" .. (package:config("shared_dep") and "ON" or "OFF"))

        for config, enabled in pairs(package:configs()) do
            if not package:extraconf("configs", config, "builtin") and configdeps[config] == nil then
                table.insert(configs, "-DARROW_" .. string.upper(config)  .. "=" .. (enabled and "ON" or "OFF"))
            end
        end

        for config, dep in pairs(configdeps) do
            table.insert(configs, "-DARROW_WITH_" .. string.upper(config)  .. "=" .. (package:config(config) and "ON" or "OFF"))
        end

        -- To fix arrow src/arrow/CMakeLists.txt:538, when CMAKE_SYSTEM_NAME set but CMAKE_SYSTEM_PROCESSOR is not causing error.
        table.insert(configs, "-DCMAKE_SYSTEM_PROCESSOR=" .. (package:is_arch("x86_64") and "x86_64" or "x86"))

        os.cd("cpp")
        
        import("package.tools.cmake").install(package, configs)
    end)

    on_test(function (package)
        
        
    end)

以上可以不压缩地读写和运行(至少不依赖三方库的部分)计算模块等

描述你认为的候选方案

arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_compress_default,函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_decompress_safe,函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::Decompress(__int64,unsigned char const *,__int64,unsigned char *)" (?Decompress@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_compressBound,函数 "public: virtual __int64 __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::MaxCompressedLen(__int64,unsigned char const *)" (?MaxCompressedLen@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA_J_JPEBE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_isError,函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_getErrorName,函数 "class arrow::Status __cdecl arrow::util::internal::`anonymous namespace'::LZ4Error(unsigned __int64,char const *)" (?LZ4Error@?A0xa3932215@internal@util@arrow@@YA?AVStatus@4@_KPEBD@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressionLevel_max,函数 "public: virtual int __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::maximum_compression_level(void)const " (?maximum_compression_level@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEBAHXZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressFrameBound,函数 "public: virtual __int64 __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::MaxCompressedLen(__int64,unsigned char const *)" (?MaxCompressedLen@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA_J_JPEBE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressFrame,函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_createCompressionContext,函数 "public: virtual class arrow::Result<class std::shared_ptr<class arrow::util::Compressor> > __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::MakeCompressor(void)" (?MakeCompressor@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@V?$shared_ptr@VCompressor@util@arrow@@@std@@@5@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_freeCompressionContext,函数 "public: virtual __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::~LZ4Compressor(void)" (??1LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressBegin,函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressBound,函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressUpdate,函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_flush,函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::FlushResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Flush(__int64,unsigned char *)" (?Flush@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UFlushResult@Compressor@util@arrow@@@5@_JPEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressEnd,函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::EndResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::End(__int64,unsigned char *)" (?End@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UEndResult@Compressor@util@arrow@@@5@_JPEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_createDecompressionContext,函数 "public: virtual class arrow::Result<class std::shared_ptr<class arrow::util::Decompressor> > __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::MakeDecompressor(void)" (?MakeDecompressor@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@V?$shared_ptr@VDecompressor@util@arrow@@@std@@@5@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_freeDecompressionContext,函数 "public: virtual __cdecl arrow::util::internal::`anonymous namespace'::LZ4Decompressor::~LZ4Decompressor(void)" (??1LZ4Decompressor@?A0xa3932215@internal@util@arrow@@UEAA@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_decompress,函数 "public: virtual class arrow::Result<struct arrow::util::Decompressor::DecompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Decompressor::Decompress(__int64,unsigned char const *,__int64,unsigned char *)" (?Decompress@LZ4Decompressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UDecompressResult@Decompressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该 符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_resetDecompressionContext,函数 "public: virtual class arrow::Status __cdecl arrow::util::internal::`anonymous namespace'::LZ4Decompressor::Reset(void)" (?Reset@LZ4Decompressor@?A0xa3932215@internal@util@arrow@@UEAA?AVStatus@5@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_compress_HC,函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号

其他信息

No response

@Issues-translate-bot
Copy link

Bot detected the issue body's language is not English, translate it automatically.


Title: Based on the original arrow7.0.0, I wrote one that can pass under win, but I can't add lz4 or the like.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Development

No branches or pull requests

2 participants