diff --git a/.gitignore b/.gitignore index 3f02ca7..2c9a88a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.jl.*.cov *.jl.mem Manifest.toml +.vscode/ diff --git a/Project.toml b/Project.toml index 15cac05..909f6fd 100644 --- a/Project.toml +++ b/Project.toml @@ -1,19 +1,22 @@ name = "LDPCStorage" uuid = "d46d874d-5773-4ce9-8adb-568101dc8882" authors = ["Adomas Baliuka "] -version = "0.2.0" +version = "0.3.0" [deps] +DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [compat] julia = "1.6" [extras] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["Test", "Documenter"] diff --git a/README.md b/README.md index b2405fd..859f306 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ [![License](https://img.shields.io/github/license/XQP-Munich/LDPCStorage.jl)](./LICENSE) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5589595.svg)](https://doi.org/10.5281/zenodo.5589595) -*Utility functions for reading and writing files containing low density parity check (LDPC) matrices.* +*Reads and writes file formats for storing sparse matrices containing only zeros and ones. +Intended for use with low density parity check (LDPC) matrices. +Also supports efficient storage for quasi-cyclic LDPC codes.* ## Installation @@ -16,9 +18,9 @@ The package is currently not registered. Install it using the Julia package mana ## Supported File Formats - `alist` (by David MacKay et al., see http://www.inference.org.uk/mackay/codes/alist.html) - `cscmat` (our custom format) DEPRECATED -- `bincsc.json` (Based on compressed sparse columns (CSC). Valid `json`. Replacement for `cscmat`.) -- `qccsc.json` (Based on compressed sparse columns (CSC). Valid `json`. Store exponents of quasi-cyclic LDPC matrices) - +- `bincsc.json` (Based on compressed sparse column (CSC). Valid `json`.) +- `qccsc.json` (Based on compressed sparse column (CSC). Valid `json`. Store exponents of quasi-cyclic LDPC matrices) +- `hpp (C++ header)` CSC of matrix as static data (write-only, reading not supported!) ## How to use ```julia @@ -32,14 +34,21 @@ H = sparse(Int8[ 1 0 0 1 0 0 0 1 0 1 0 1 0 1 ]) -save_to_alist(H, "ldpc.alist") -H_alist = load_alist("ldpc.alist") +save_to_alist("./ldpc.alist", H) +H_alist = load_alist("./ldpc.alist") H == H_alist || warn("Failure") -save_to_bincscjson(H, "ldpc.bincsc.json") -H_csc = load_ldpc_from_json("ldpc.bincsc.json") +save_to_bincscjson("./ldpc.bincsc.json", H) +H_csc = load_ldpc_from_json("./ldpc.bincsc.json") H == H_csc || warn("Failure") + +open("./autogen_ldpc.hpp", "w+") do io + write_cpp_header(io, H) +end ``` +Also available are versions of the other methods accepting an `IO` object: +`write_alist`, `write_bincscjson`, etc. + ## Contributing Contributions, feature requests and suggestions are welcome. Open an issue or contact us directly. diff --git a/src/LDPCStorage.jl b/src/LDPCStorage.jl index 5c81e23..92a16ef 100644 --- a/src/LDPCStorage.jl +++ b/src/LDPCStorage.jl @@ -1,14 +1,25 @@ +""" +$(DocStringExtensions.README) +""" module LDPCStorage +using DocStringExtensions + include("utils.jl") include("alist.jl") -export load_alist, save_to_alist +export save_to_alist, write_alist, load_alist include("cscmat.jl") # this format is deprecated in favour of csc.json # export save_to_cscmat, load_cscmat, load_matrix_from_qc_cscmat_file, CSCMAT_FORMAT_VERSION include("cscjson.jl") -export load_ldpc_from_json, save_to_bincscjson, save_to_qccscjson, CSCJSON_FORMAT_VERSION +export write_bincscjson, save_to_bincscjson +export write_qcscjson, save_to_qccscjson +export load_ldpc_from_json, CSCJSON_FORMAT_VERSION + +# This format stores the LDPC code as static data in a c++ header file. +include("cpp_header_based.jl") +export write_cpp_header end # module diff --git a/src/alist.jl b/src/alist.jl index 302df0e..2f2c871 100644 --- a/src/alist.jl +++ b/src/alist.jl @@ -2,7 +2,11 @@ using SparseArrays using LinearAlgebra -"""Load an LDPC matrix from a text file in alist format.""" +""" +$(SIGNATURES) + +Load an LDPC matrix from a text file in alist format. +""" function load_alist(file_path::AbstractString; check_redundant=false,) if file_extension(file_path) != ".alist" @warn "load_alist called on file with extension '$(file_extension(file_path))', expected '.alist'" @@ -73,15 +77,29 @@ end """ - function save_to_alist(matrix::AbstractArray{Int8,2}, out_file_path::String) +$(SIGNATURES) Save LDPC matrix to file in alist format. For details about the format, see: https://aff3ct.readthedocs.io/en/latest/user/simulation/parameters/codec/ldpc/decoder.html#dec-h-path-image-required-argument http://www.inference.org.uk/mackay/codes/alist.html -todo test this carefully """ -function save_to_alist(matrix::AbstractArray{Int8,2}, out_file_path::String) +function save_to_alist(out_file_path::String, matrix::AbstractArray{Int8,2}) + open(out_file_path, "w+") do file + write_alist(file, matrix) + end + + return nothing +end + +""" +$(SIGNATURES) +Save LDPC matrix to file in alist format. For details about the format, see: +https://aff3ct.readthedocs.io/en/latest/user/simulation/parameters/codec/ldpc/decoder.html#dec-h-path-image-required-argument +http://www.inference.org.uk/mackay/codes/alist.html +""" +function write_alist(io::IO, matrix::AbstractArray{Int8,2}) + # TODO more careful testing (the_M, the_N) = size(matrix) variable_node_degrees = get_variable_node_degrees(matrix) @@ -124,10 +142,8 @@ function save_to_alist(matrix::AbstractArray{Int8,2}, out_file_path::String) # check node '1' append!(lines, get_node_indices(matrix)) - open(out_file_path, "w+") do file - for line in lines - println(file, line) - end + for line in lines + println(io, line) end return nothing diff --git a/src/cpp_header_based.jl b/src/cpp_header_based.jl new file mode 100644 index 0000000..968e33f --- /dev/null +++ b/src/cpp_header_based.jl @@ -0,0 +1,94 @@ +# This script generates a `.hpp` file (C++ header) containing +# an LDPC code stored in compressed sparse column (CSC) format. +# See command line help for how to use it. + +using SparseArrays +using LinearAlgebra + +using Pkg + +const cpp_file_description = """ +// This file was automatically generated using LDPCStorage.jl v$(Pkg.project().version) (https://github.com/XQP-Munich/LDPCStorage.jl). +// A sparse LDPC matrix (containing only zeros and ones) is saved in compressed sparse column (CSC) format. +// Since the matrix (and LDPC code) is known at compile time, there is no need to save it separately in a file. +// This significantly blows up the executable size (the memory would still have to be used when saving the matrix). + +""" + + +""" +$(SIGNATURES) + +Output C++ header storing the sparse binary (containing only zeros and ones) matrix H +in compressed sparse column (CSC) format. + +Note the conversion from Julia's one-based indices to zero-based indices in C++ (also within CSC format). +""" +function write_cpp_header( + io::IO, + H::AbstractArray{Int8, 2} + ; + namespace_name::AbstractString = "AutogenLDPC", + ) + H = dropzeros(H) # remove stored zeros! + _, _, values = findnz(H) + + all(values .== 1) || throw(ArgumentError("Expected matrix containing only zeros and ones.")) + + num_nonzero = length(values) + if log2(num_nonzero) < 16 + colptr_cpp_type = "std::uint16_t" + elseif log2(num_nonzero) < 32 + colptr_cpp_type = "std::uint32_t" + elseif log2(num_nonzero) < 64 + colptr_cpp_type = "std::uint64_t" + else + throw(ArgumentError("Input matrix not sparse? Has $num_nonzero entries...")) + end + + if log2(size(H, 1)) < 16 + row_idx_type = "std::uint16_t" + else + row_idx_type = "std::uint32_t" + end + + print(io, cpp_file_description) + + println(io, """ + #include + #include + + namespace $namespace_name { + + constexpr inline std::size_t M = $(size(H, 1)); + constexpr inline std::size_t N = $(size(H, 2)); + constexpr inline std::size_t num_nz = $num_nonzero; + constexpr inline std::array<$colptr_cpp_type, N + 1> colptr = {""") + + for (i, idx) in enumerate(H.colptr) + print(io, "0x$(string(idx - 1, base=16))") # Convert index to base zero + if i != length(H.colptr) + print(io, ",") + end + if mod(i, 100) == 0 + println(io, "") # for formatting. + end + end + println(io, "\n};\n") + + println(io, "// ------------------------------------------------------- \n") + println(io, "constexpr inline std::array<$row_idx_type, num_nz> row_idx = {") + + for (i, idx) in enumerate(H.rowval) + print(io, "0x$(string(idx - 1, base=16))") # Convert index to base zero + if i != length(H.rowval) + print(io, ",") + end + if mod(i, 100) == 0 + println(io, "") # for formatting. + end + end + println(io, "\n};\n\n") + + println(io, "} // namespace $namespace_name") +end diff --git a/src/cscjson.jl b/src/cscjson.jl index 1b2a1fd..94d31e3 100644 --- a/src/cscjson.jl +++ b/src/cscjson.jl @@ -2,7 +2,7 @@ using SparseArrays using LinearAlgebra using JSON -CSCJSON_FORMAT_VERSION = v"0.3.0" # track version of our custom compressed sparse storage json file format. +const CSCJSON_FORMAT_VERSION = v"0.3.0" # track version of our custom compressed sparse storage json file format. const format_if_nnz_values_omitted = :BINCSCJSON const format_if_nnz_values_stored = :COMPRESSED_SPARSE_COLUMN @@ -12,24 +12,54 @@ const description = "Compressed sparse column storage of a matrix (arrays `colpt "Otherwise, format is expected to be $format_if_nnz_values_stored." +get_metadata() = Dict( + :julia_package_version => "v$(Pkg.project().version)", + :julia_package_url => "https://github.com/XQP-Munich/LDPCStorage.jl", +) + + """ +$(SIGNATURES) + +Helper method to use with files. See `print_bincscjson` for main interface. + +Writes the two arrays `colptr` and `rowval` defining compressed sparse column (CSC) storage of a the into a json file. Errors unless sparse matrix only contains ones and zeros. -writes the two arrays `colptr` and `rowval` defining compressed sparse column (CSC) storage of a the into a json file. -The third array of CSC format, i.e., the nonzero entries, is not needed, since +The third array of CSC format, i.e., the nonzero entries, is not needed, since the matrix is assumed to only contain ones and zeros. """ function save_to_bincscjson( - mat::SparseMatrixCSC, destination_file_path::String + destination_file_path::String, mat::SparseMatrixCSC, ; - comments::AbstractString="", + varargs..., ) - all(x->x==1, mat.nzval) || error( - "The input matrix has nonzero entries besides 1. Note: the matrix should have no stored zeros.") - expected_extension = ".bincsc.json" if !endswith(destination_file_path, expected_extension) @warn "Expected extension '$expected_extension' when writing to '$(destination_file_path)')" end + open(destination_file_path, "w+") do file + print_bincscjson(file, mat; varargs...) + end + + return nothing +end + + +""" +$(SIGNATURES) + +Writes the two arrays `colptr` and `rowval` defining compressed sparse column (CSC) storage of a the into a json file. +Errors unless sparse matrix only contains ones and zeros. +The third array of CSC format, i.e., the nonzero entries, is not needed, since the matrix is assumed to only contain ones and zeros. +""" +function print_bincscjson( + io::IO, mat::SparseMatrixCSC + ; + comments::AbstractString="", + ) + all(x->x==1, mat.nzval) || error( + "The input matrix has nonzero entries besides 1. Note: the matrix should have no stored zeros.") + data = Dict( :CSCJSON_FORMAT_VERSION => string(CSCJSON_FORMAT_VERSION), :description => description, @@ -42,24 +72,32 @@ function save_to_bincscjson( :rowval => mat.rowval .- 1, ) - open(destination_file_path, "w+") do file - JSON.print(file, data) + try + data[:metadata] = get_metadata() + catch e + @warn "Generating metadata failed. Including default. Error:\n $e" + data[:metadata] = "Metadata generation failed." end + JSON.print(io, data) + return nothing end """ +$(SIGNATURES) + +Helper method to use with files. See `print_qccscjson` for main interface. + write the three arrays defining compressed sparse column (CSC) storage of a matrix into a file. This is used to store the exponents of a quasi-cyclic LDPC matrix. The QC expansion factor must be specified. """ function save_to_qccscjson( - mat::SparseMatrixCSC, destination_file_path::String, + destination_file_path::String, mat::SparseMatrixCSC ; - qc_expansion_factor::Integer, - comments::AbstractString="", + varargs... ) expected_extension = ".qccsc.json" @@ -67,6 +105,28 @@ function save_to_qccscjson( @warn "Expected extension '$expected_extension' when writing to '$(destination_file_path)')" end + open(destination_file_path, "w+") do file + print_qccscjson(file, mat; varargs...) + end + + return nothing +end + + +""" +$(SIGNATURES) + +write the three arrays defining compressed sparse column (CSC) storage of a matrix into a file. +This is used to store the exponents of a quasi-cyclic LDPC matrix. +The matrix is assumed to contain quasi-cyclic exponents of an LDPC matrix. +The QC expansion factor must be specified. +""" +function print_qccscjson( + io::IO, mat::SparseMatrixCSC, + ; + qc_expansion_factor::Integer, + comments::AbstractString="", + ) data = Dict( :CSCJSON_FORMAT_VERSION => string(CSCJSON_FORMAT_VERSION), :description => description, @@ -81,14 +141,28 @@ function save_to_qccscjson( :nzval => mat.nzval, ) - open(destination_file_path, "w+") do file - JSON.print(file, data) + try + data[:metadata] = get_metadata() + catch e + @warn "Generating metadata failed. Including default. Error:\n $e" + data[:metadata] = "Metadata generation failed." end + JSON.print(io, data) + return nothing end +""" +$(SIGNATURES) + +Loads LDPC matrix from a json file containing compressed sparse column (CSC) storage for either of +- `qccscjson` (CSC of quasi-cyclic exponents) format +- `bincscjson` (CSC of ) format + +Use option to expand quasi-cyclic exponents and get a sparse binary matrix. +""" function load_ldpc_from_json(file_path::AbstractString; expand_qc_exponents_to_binary=false) data = JSON.parsefile(file_path) diff --git a/src/cscmat.jl b/src/cscmat.jl index 2b1f8fc..09d2b36 100644 --- a/src/cscmat.jl +++ b/src/cscmat.jl @@ -142,6 +142,8 @@ end """ +DEPRECATED! + Convert matrix of exponents for QC LDPC matrix to the actual binary LDPC matrix. The resulting QC-LDPC matrix is a block matrix where each block is either zero, diff --git a/src/utils.jl b/src/utils.jl index 442a97a..c26e1c2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,6 +1,10 @@ using SparseArrays, SHA -"""parse a single line of space separated integers""" +""" +$(SIGNATURES) + +parse a single line of space separated integers +""" space_sep_ints(s::AbstractString; base=10) = parse.(Int, split(s); base) @@ -12,7 +16,10 @@ function file_extension(path::String) end end + """ +$(SIGNATURES) + Returns a 256 bit hash of a sparse matrix. This function should only be used for unit tests!!! """ diff --git a/test/runtests.jl b/test/runtests.jl index dc91d48..4cc3aa6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,8 @@ using LDPCStorage "File Formats" => ["test_alist.jl", "test_cscmat.jl", "test_cscjson.jl", + "test_cpp_header.jl", + "test_readme_doctest.jl", ], ) diff --git a/test/test_alist.jl b/test/test_alist.jl index 2a2a0a4..55e54b4 100644 --- a/test/test_alist.jl +++ b/test/test_alist.jl @@ -10,7 +10,7 @@ using LDPCStorage 1 0 0 1 0 0 0 1 0 1 0 1 0 1 ] file_path = tempname() * "_unit_test.alist" - save_to_alist(H, file_path) + save_to_alist(file_path, H) H_loaded = load_alist(file_path) diff --git a/test/test_cpp_header.jl b/test/test_cpp_header.jl new file mode 100644 index 0000000..9fcef9b --- /dev/null +++ b/test/test_cpp_header.jl @@ -0,0 +1,23 @@ + +function main(args) + +end + +@testset "write c++ header" begin + output_path = tempname() + + H = sparse(Int8[ + 0 0 1 1 0 0 0 0 1 0 0 1 1 0 + 1 9 0 1 1 0 0 0 0 0 1 0 0 1 # 9 will be stored zero + 0 1 0 1 0 1 1 0 1 0 0 1 1 0 + 1 0 0 1 0 0 0 1 0 1 0 1 0 1 + ]) + + H[2,2] = 0 # 9 becomes a stored zero + + open(output_path, "w+") do io + write_cpp_header(io, H) + end + + # TODO check correctness of written C++ header! +end diff --git a/test/test_cscjson.jl b/test/test_cscjson.jl index 81eb244..3ad03a0 100644 --- a/test/test_cscjson.jl +++ b/test/test_cscjson.jl @@ -21,7 +21,7 @@ end target_file = tempname() * ".bincsc.json" - save_to_bincscjson(H, target_file; comments="Some comment") + save_to_bincscjson(target_file, H; comments="Some comment") H_read = load_ldpc_from_json(target_file) @test H_read == H end @@ -31,7 +31,7 @@ end Hqc = load_ldpc_from_json(qccscjson_exampl_file_path) target_file = tempname() * ".qccsc.json" - save_to_qccscjson(Hqc, target_file; comments="Some comment", qc_expansion_factor=32) + save_to_qccscjson(target_file, Hqc; comments="Some comment", qc_expansion_factor=32) H_read = load_ldpc_from_json(target_file) @test H_read == Hqc @@ -78,5 +78,5 @@ end target_file = tempname() * ".bincsc.json" - @test_throws ErrorException save_to_bincscjson(H, target_file; comments="Some comment") + @test_throws ErrorException save_to_bincscjson(target_file, H; comments="Some comment") end diff --git a/test/test_readme_doctest.jl b/test/test_readme_doctest.jl new file mode 100644 index 0000000..3f00f98 --- /dev/null +++ b/test/test_readme_doctest.jl @@ -0,0 +1,56 @@ +const ALL_CODEBLOCKS_IN_README = [ +raw""" +```julia +using SparseArrays +using LDPCStorage + +H = sparse(Int8[ + 0 0 1 1 0 0 0 0 1 0 0 1 1 0 + 1 0 0 1 1 0 0 0 0 0 1 0 0 1 + 0 1 0 1 0 1 1 0 1 0 0 1 1 0 + 1 0 0 1 0 0 0 1 0 1 0 1 0 1 + ]) + +save_to_alist("./ldpc.alist", H) +H_alist = load_alist("./ldpc.alist") +H == H_alist || warn("Failure") + +save_to_bincscjson("./ldpc.bincsc.json", H) +H_csc = load_ldpc_from_json("./ldpc.bincsc.json") +H == H_csc || warn("Failure") + +open("./autogen_ldpc.hpp", "w+") do io + write_cpp_header(io, H) +end +``` +""", +] + + +@testset "README only contains code blocks mentioned here" begin + # if this test fails, enter all codeblocks + + readme_path = "$(pkgdir(LDPCStorage))/README.md" + readme_contents = read(readme_path, String) + + for (i, code_block) in enumerate(ALL_CODEBLOCKS_IN_README) + # check that above code is contained verbatim in README + @test contains(readme_contents, code_block) + end + + # check that README does not contain any other Julia code blocks + @test length(collect(eachmatch(r"```julia", readme_contents))) == length(ALL_CODEBLOCKS_IN_README) +end + + +@testset "codeblocks copied in README run without errors" begin + for (i, code_block) in enumerate(ALL_CODEBLOCKS_IN_README) + @testset "Codeblock $i" begin + # remove the ```julia ... ``` ticks and parse code + parsed_code = Meta.parseall(code_block[9:end-4]) + + # check if it runs without exceptions + eval(parsed_code) + end + end +end