Complete overhaul of interface. Adds support for writing C++ headers.

.gitignore

@@ -2,3 +2,4 @@
 *.jl.*.cov
 *.jl.mem
 Manifest.toml
+.vscode/

Project.toml

@@ -1,19 +1,22 @@
 name = "LDPCStorage"
 uuid = "d46d874d-5773-4ce9-8adb-568101dc8882"
 authors = ["Adomas Baliuka <[email protected]>"]
-version = "0.2.0"
+version = "0.3.0"
 
 [deps]
+DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [compat]
 julia = "1.6"
 
 [extras]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Test", "Documenter"]

README.md

@@ -5,7 +5,9 @@
 [![License](https://img.shields.io/github/license/XQP-Munich/LDPCStorage.jl)](./LICENSE)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5589595.svg)](https://doi.org/10.5281/zenodo.5589595)
 
-*Utility functions for reading and writing files containing low density parity check (LDPC) matrices.*
+*Reads and writes file formats for storing sparse matrices containing only zeros and ones.
+Intended for use with low density parity check (LDPC) matrices.
+Also supports efficient storage for quasi-cyclic LDPC codes.*
 
 ## Installation
 
@@ -16,9 +18,9 @@ The package is currently not registered. Install it using the Julia package mana
 ## Supported File Formats
 - `alist` (by David MacKay et al., see http://www.inference.org.uk/mackay/codes/alist.html)
 - `cscmat` (our custom format) DEPRECATED
-- `bincsc.json` (Based on compressed sparse columns (CSC). Valid `json`. Replacement for `cscmat`.)
-- `qccsc.json` (Based on compressed sparse columns (CSC). Valid `json`. Store exponents of quasi-cyclic LDPC matrices)
-
+- `bincsc.json` (Based on compressed sparse column (CSC). Valid `json`.)
+- `qccsc.json` (Based on compressed sparse column (CSC). Valid `json`. Store exponents of quasi-cyclic LDPC matrices)
+- `hpp (C++ header)` CSC of matrix as static data (write-only, reading not supported!)
 ## How to use
 
 ```julia
@@ -32,14 +34,21 @@ H = sparse(Int8[
         1 0 0 1 0 0 0 1 0 1 0 1 0 1
     ])
 
-save_to_alist(H, "ldpc.alist")
-H_alist = load_alist("ldpc.alist")
+save_to_alist("./ldpc.alist", H)
+H_alist = load_alist("./ldpc.alist")
 H == H_alist || warn("Failure")
 
-save_to_bincscjson(H, "ldpc.bincsc.json")
-H_csc = load_ldpc_from_json("ldpc.bincsc.json")
+save_to_bincscjson("./ldpc.bincsc.json", H)
+H_csc = load_ldpc_from_json("./ldpc.bincsc.json")
 H == H_csc || warn("Failure")
+
+open("./autogen_ldpc.hpp", "w+") do io
+    write_cpp_header(io, H)
+end
 ```
 
+Also available are versions of the other methods accepting an `IO` object: 
+`write_alist`, `write_bincscjson`, etc.
+
 ## Contributing
 Contributions, feature requests and suggestions are welcome. Open an issue or contact us directly.

src/LDPCStorage.jl

@@ -1,14 +1,25 @@
+"""
+$(DocStringExtensions.README)
+"""
 module LDPCStorage
 
+using DocStringExtensions
+
 include("utils.jl")
 
 include("alist.jl")
-export load_alist, save_to_alist
+export save_to_alist, write_alist, load_alist
 
 include("cscmat.jl")  # this format is deprecated in favour of csc.json
 # export save_to_cscmat, load_cscmat, load_matrix_from_qc_cscmat_file, CSCMAT_FORMAT_VERSION
 
 include("cscjson.jl")
-export load_ldpc_from_json, save_to_bincscjson, save_to_qccscjson, CSCJSON_FORMAT_VERSION
+export write_bincscjson, save_to_bincscjson
+export write_qcscjson, save_to_qccscjson
+export load_ldpc_from_json, CSCJSON_FORMAT_VERSION
+
+# This format stores the LDPC code as static data in a c++ header file.
+include("cpp_header_based.jl")
+export write_cpp_header
 
 end # module

src/alist.jl

@@ -2,7 +2,11 @@ using SparseArrays
 using LinearAlgebra
 
 
-"""Load an LDPC matrix from a text file in alist format."""
+"""
+$(SIGNATURES)
+
+Load an LDPC matrix from a text file in alist format.
+"""
 function load_alist(file_path::AbstractString; check_redundant=false,)
     if file_extension(file_path) != ".alist"
         @warn "load_alist called on file with extension '$(file_extension(file_path))', expected '.alist'"
@@ -73,15 +77,29 @@ end
 
 
 """
-    function save_to_alist(matrix::AbstractArray{Int8,2}, out_file_path::String)
+$(SIGNATURES)
 
 Save LDPC matrix to file in alist format. For details about the format, see:
 https://aff3ct.readthedocs.io/en/latest/user/simulation/parameters/codec/ldpc/decoder.html#dec-h-path-image-required-argument
 http://www.inference.org.uk/mackay/codes/alist.html
-todo test this carefully
 """
-function save_to_alist(matrix::AbstractArray{Int8,2}, out_file_path::String)
+function save_to_alist(out_file_path::String, matrix::AbstractArray{Int8,2})
+    open(out_file_path, "w+") do file
+        write_alist(file, matrix)
+    end
+
+    return nothing
+end
+
+"""
+$(SIGNATURES)
 
+Save LDPC matrix to file in alist format. For details about the format, see:
+https://aff3ct.readthedocs.io/en/latest/user/simulation/parameters/codec/ldpc/decoder.html#dec-h-path-image-required-argument
+http://www.inference.org.uk/mackay/codes/alist.html
+"""
+function write_alist(io::IO, matrix::AbstractArray{Int8,2})
+    # TODO more careful testing
     (the_M, the_N) = size(matrix)
 
     variable_node_degrees = get_variable_node_degrees(matrix)
@@ -124,10 +142,8 @@ function save_to_alist(matrix::AbstractArray{Int8,2}, out_file_path::String)
     # check node '1'
     append!(lines, get_node_indices(matrix))
 
-    open(out_file_path, "w+") do file
-        for line in lines
-            println(file, line)
-        end
+    for line in lines
+        println(io, line)
     end
 
     return nothing

src/cpp_header_based.jl

@@ -0,0 +1,94 @@
+# This script generates a `.hpp` file (C++ header) containing
+# an LDPC code stored in compressed sparse column (CSC) format.
+# See command line help for how to use it.
+
+using SparseArrays
+using LinearAlgebra
+
+using Pkg
+
+const cpp_file_description = """
+// This file was automatically generated using LDPCStorage.jl v$(Pkg.project().version) (https://github.com/XQP-Munich/LDPCStorage.jl).
+// A sparse LDPC matrix (containing only zeros and ones) is saved in compressed sparse column (CSC) format.
+// Since the matrix (and LDPC code) is known at compile time, there is no need to save it separately in a file.
+// This significantly blows up the executable size (the memory would still have to be used when saving the matrix).
+
+"""
+
+
+"""
+$(SIGNATURES)
+
+Output C++ header storing the sparse binary (containing only zeros and ones) matrix H
+in compressed sparse column (CSC) format.
+
+Note the conversion from Julia's one-based indices to zero-based indices in C++ (also within CSC format).
+"""
+function write_cpp_header(
+    io::IO,
+    H::AbstractArray{Int8, 2}
+    ;
+    namespace_name::AbstractString = "AutogenLDPC",
+    )
+    H = dropzeros(H)  # remove stored zeros!
+    _, _, values = findnz(H)
+
+    all(values .== 1) || throw(ArgumentError("Expected matrix containing only zeros and ones."))
+
+    num_nonzero = length(values)
+    if log2(num_nonzero) < 16
+        colptr_cpp_type = "std::uint16_t"
+    elseif log2(num_nonzero) < 32
+        colptr_cpp_type = "std::uint32_t"
+    elseif log2(num_nonzero) < 64
+        colptr_cpp_type = "std::uint64_t"
+    else
+        throw(ArgumentError("Input matrix not sparse? Has $num_nonzero entries..."))
+    end
+
+    if log2(size(H, 1)) < 16
+        row_idx_type = "std::uint16_t"
+    else 
+        row_idx_type = "std::uint32_t"
+    end
+
+    print(io, cpp_file_description)
+
+    println(io, """
+    #include <cstdint>
+    #include <array>
+
+    namespace $namespace_name {
+
+    constexpr inline std::size_t M = $(size(H, 1));
+    constexpr inline std::size_t N = $(size(H, 2));
+    constexpr inline std::size_t num_nz = $num_nonzero;
+    constexpr inline std::array<$colptr_cpp_type, N + 1> colptr = {""")
+
+    for (i, idx) in enumerate(H.colptr)
+        print(io, "0x$(string(idx - 1, base=16))")  # Convert index to base zero
+        if i != length(H.colptr)
+            print(io, ",")
+        end
+        if mod(i, 100) == 0
+            println(io, "")  # for formatting.
+        end
+    end
+    println(io, "\n};\n")
+
+    println(io, "// ------------------------------------------------------- \n")
+    println(io, "constexpr inline std::array<$row_idx_type, num_nz> row_idx = {")
+
+    for (i, idx) in enumerate(H.rowval)
+        print(io, "0x$(string(idx - 1, base=16))")  # Convert index to base zero
+        if i != length(H.rowval)
+            print(io, ",")
+        end
+        if mod(i, 100) == 0
+            println(io, "")  # for formatting.
+        end
+    end
+    println(io, "\n};\n\n")
+
+    println(io, "} // namespace $namespace_name")
+end