Merge pull request #10 from sz3/next-release

v0.5.10 release

app/build.gradle

@@ -1,14 +1,15 @@
 apply plugin: 'com.android.application'
 
 android {
-    compileSdkVersion 29
+    compileSdkVersion 32
     buildToolsVersion "29.0.3"
+    ndkVersion "22.1.7171670"
     defaultConfig {
         applicationId "org.cimbar.camerafilecopy"
         minSdkVersion 21
-        targetSdkVersion 29
-        versionCode 4
-        versionName "0.5.9"
+        targetSdkVersion 32
+        versionCode 6
+        versionName "0.5.10"
         testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
         externalNativeBuild {
             cmake {

app/src/cpp/cfc-cpp/jni.cpp

@@ -119,7 +119,7 @@ namespace {
 	{
 		std::stringstream sstop;
 		sstop << "cfc using " << proc.num_threads() << " thread(s). " << proc.color_bits() << "..." << proc.backlog() << "? ";
-		sstop << (MultiThreadedDecoder::bytes / std::max<double>(1, MultiThreadedDecoder::decoded)) << "b v0.5.9";
+		sstop << (MultiThreadedDecoder::bytes / std::max<double>(1, MultiThreadedDecoder::decoded)) << "b v0.5.10a";
 		std::stringstream ssmid;
 		ssmid << "#: " << MultiThreadedDecoder::perfect << " / " << MultiThreadedDecoder::decoded << " / " << MultiThreadedDecoder::scanned << " / " << _calls;
 		std::stringstream ssperf;

app/src/cpp/libcimbar/.github/workflows/ci.yml

@@ -14,6 +14,9 @@ jobs:
           os: ubuntu-18.04
           env: CXX="g++-7" CC="gcc-7"
 
+        - name: "linux gcc9"
+          os: ubuntu-latest
+
         - name: "linux clang"
           os: ubuntu-18.04
           env: CXX="clang++" CC="clang"

app/src/cpp/libcimbar/.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: release
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  package-cimbar:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+
+      - name: Get openCV
+        run: |
+          wget https://github.com/opencv/opencv/archive/refs/tags/4.5.5.zip
+          unzip 4.5.5.zip
+          mv opencv-4.5.5 opencv4
+
+      - name: Run the build process with Docker
+        uses: addnab/docker-run-action@v3
+        with:
+          image: emscripten/emsdk:latest
+          options: -v ${{ github.workspace }}:/usr/src/app
+          shell: bash
+          run: |
+            bash /usr/src/app/package-wasm.sh
+
+      - name: Show results
+        run: ls -l web/
+
+      - name: Create release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: |
+            web/cimbar.asmjs.zip
+            web/cimbar.wasm.tar.gz
+

app/src/cpp/libcimbar/CMakeLists.txt

@@ -8,7 +8,7 @@ endif()
 set(CMAKE_CXX_STANDARD 17)
 
 if(NOT DEFINED OPENCV_LIBS)
-	set(OPENCV_LIBS "opencv_core" "opencv_imgcodecs" "opencv_imgproc" "opencv_photo")
+	set(OPENCV_LIBS "opencv_imgcodecs" "opencv_imgproc" "opencv_photo" "opencv_core")
 endif()
 
 if(NOT DEFINED CPPFILESYSTEM)

app/src/cpp/libcimbar/package-portable-linux.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+## targeting old glibc
+# docker run --mount type=bind,source="$(pwd)",target="/usr/src/app" -it ubuntu:16.04
+
+cd /usr/src/app
+
+# https://gist.github.com/jlblancoc/99521194aba975286c80f93e47966dc5
+apt update
+apt install -y software-properties-common
+add-apt-repository -y ppa:ubuntu-toolchain-r/test
+
+apt update
+apt install -y pkgconf g++-7 python-pip
+apt install -y libgles2-mesa-dev libglfw3-dev
+
+# cmake (via pip)
+python -m pip install cmake
+
+# use gcc7
+update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-7 100
+update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 100
+
+cd opencv4/
+mkdir build-portable/ && cd build-portable/
+/usr/local/bin/cmake .. -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF -DOPENCV_GENERATE_PKGCONFIG=YES -DOPENCV_FORCE_3RDPARTY_BUILD=YES
+make -j5 install
+
+cd /usr/src/app
+mkdir build-portable/ && cd build-portable/
+/usr/local/bin/cmake .. -DBUILD_PORTABLE_LINUX=1
+make -j5 install

app/src/cpp/libcimbar/package-wasm.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+#docker run --mount type=bind,source="$(pwd)",target="/usr/src/app" -it emscripten/emsdk:latest
+
+cd /usr/src/app
+
+apt update
+apt install python -y
+
+cd opencv4/
+mkdir opencv-build-wasm && cd opencv-build-wasm
+python ../platforms/js/build_js.py build_wasm --build_wasm --emscripten_dir=/emsdk/upstream/emscripten
+
+cd /usr/src/app
+mkdir build-wasm && cd build-wasm
+emcmake cmake .. -DUSE_WASM=1 -DOPENCV_DIR=/usr/src/app/opencv4
+make -j5 install
+(cd ../web/ && tar -czvf cimbar.wasm.tar.gz cimbar_js.* index.html main.js)
+
+cd /usr/src/app
+mkdir build-asmjs && cd build-asmjs
+emcmake cmake .. -DUSE_WASM=2 -DOPENCV_DIR=/usr/src/app/opencv4
+make -j5 install
+(cd ../web/ && zip cimbar.asmjs.zip cimbar_js.js index.html main.js)

app/src/cpp/libcimbar/src/exe/cimbar_send/send.cpp

@@ -70,7 +70,7 @@ int main(int argc, char** argv)
 	if (!initialize_GL(window_size, window_size))
 	{
 		std::cerr << "failed to create window :(" << std::endl;
-		return 50;
+		return 70;
 	}
 
 	configure(colorBits, ecc, compressionLevel);

app/src/cpp/libcimbar/src/lib/cimb_translator/CellPositions.h

@@ -1,8 +1,9 @@
 /* This code is subject to the terms of the Mozilla Public License, v.2.0. http://mozilla.org/MPL/2.0/. */
 #pragma once
 
-#include <vector>
+#include <cstddef>
 #include <utility>
+#include <vector>
 
 class CellPositions
 {

app/src/cpp/libcimbar/src/lib/cimb_translator/test/CimbDecoderTest.cpp

@@ -56,7 +56,7 @@ TEST_CASE( "CimbDecoderTest/testPrethresholdDecode", "[unit]" )
 	for (unsigned i = 0; i < 16; ++i)
 	{
 		cv::Mat tile = cimbar::getTile(4, i, true);
-		cv::Mat tenxten(10, 10, tile.type());
+		cv::Mat tenxten(10, 10, tile.type(), cv::Scalar(0, 0, 0));
 		tile.copyTo(tenxten(cv::Rect(cv::Point(1, 1), tile.size())));
 
 		// grayscale and threshold, since that's what average_hash needs

app/src/cpp/libcimbar/src/lib/compression/zstd_compressor.h

@@ -32,7 +32,7 @@ class zstd_compressor : public STREAM
 			size_t compressedBytes = ZSTD_compressCCtx(_cctx, _compBuff.data(), _compBuff.size(), data, writeLen, _compressionLevel);
 			if (ZSTD_isError(compressedBytes))
 			{
-				std::cout << "error? " << ZSTD_getErrorName(compressedBytes) << std::endl;
+				std::cerr << "error? " << ZSTD_getErrorName(compressedBytes) << std::endl;
 				return false;
 			}
 			STREAM::write(_compBuff.data(), compressedBytes);

app/src/cpp/libcimbar/src/lib/encoder/test/DecoderTest.cpp

@@ -56,5 +56,8 @@ TEST_CASE( "DecoderTest/testDecode.Sample", "[unit]" )
 	unsigned bytesDecoded = dec.decode(TestCimbar::getSample("6bit/4_30_f0_627_extract.jpg"), decodedFile);
 	assertEquals( 9300, bytesDecoded );
 
-	assertEquals( "3de927c8aa0221807a2784210160cdc17567eb587bf01233d166900aadf14bf5", get_hash(decodedFile) );
+	if (CV_VERSION_MAJOR == 3)
+		assertEquals( "3de927c8aa0221807a2784210160cdc17567eb587bf01233d166900aadf14bf5", get_hash(decodedFile) );
+	else // # cv4
+		assertEquals( "59ddb2516b4ff5a528aebe538a22b736a6714263a454d20e146e1ffbba36c5ae", get_hash(decodedFile) );
 }

app/src/cpp/libcimbar/src/lib/fountain/FountainMetadata.h

@@ -15,7 +15,7 @@ class FountainMetadata
 	}
 
 public:
-	FountainMetadata(uint64_t id)
+	FountainMetadata(uint32_t id)
 	    : _data(id)
 	{
 	}
@@ -33,7 +33,7 @@ class FountainMetadata
 		to_uint8_arr(encode_id, size, d);
 	}
 
-	unsigned id() const
+	uint32_t id() const
 	{
 		return _data;
 	}
@@ -65,5 +65,5 @@ class FountainMetadata
 	}
 
 protected:
-	uint64_t _data; // might invert this and only generate the uint64_t when we need it
+	uint32_t _data; // might invert this and only generate the uint32_t when we need it
 };

app/src/cpp/libcimbar/src/lib/fountain/fountain_decoder_sink.h

@@ -39,10 +39,10 @@ class fountain_decoder_sink
 		return true;
 	}
 
-	void mark_done(uint64_t id)
+	void mark_done(const FountainMetadata& md)
 	{
-		_done.insert(id);
-		auto it = _streams.find(id);
+		_done.insert(md.id());
+		auto it = _streams.find(stream_slot(md));
 		if (it != _streams.end())
 			_streams.erase(it);
 	}
@@ -60,7 +60,7 @@ class fountain_decoder_sink
 	std::vector<std::string> get_done() const
 	{
 		std::vector<std::string> done;
-		for (uint64_t id : _done)
+		for (uint32_t id : _done)
 			done.push_back( get_filename(FountainMetadata(id)) );
 		return done;
 	}
@@ -77,7 +77,7 @@ class fountain_decoder_sink
 		return progress;
 	}
 
-	bool is_done(uint64_t id) const
+	bool is_done(uint32_t id) const
 	{
 		return _done.find(id) != _done.end();
 	}
@@ -106,7 +106,7 @@ class fountain_decoder_sink
 			return false;
 
 		if (store(md, *finished))
-			mark_done(md.id());
+			mark_done(md);
 		return true;
 	}
 
@@ -122,7 +122,7 @@ class fountain_decoder_sink
 	}
 
 protected:
-	// streams is limited to at most 8 decoders at a time. Current, we just use the lower bits of the encode_id.
+	// streams is limited to at most 8 decoders at a time. Currently, we just use the lower bits of the encode_id.
 	uint8_t stream_slot(const FountainMetadata& md) const
 	{
 		return md.encode_id() & 0x7;
@@ -137,7 +137,10 @@ class fountain_decoder_sink
 	std::string _dataDir;
 	unsigned _chunkSize;
 
+	// maybe instead of unordered_map+set, something where we can "age out" old streams?
+	// e.g. most recent 16/8, or something?
+	// question is what happens to _done/_streams when we wrap for continuous data streaming...
 	std::unordered_map<uint8_t, fountain_decoder_stream> _streams;
-	// track the uint64_t combo of (encode_id,size) to avoid redundant work
-	std::set<uint64_t> _done;
+	// track the uint32_t combo of (encode_id,size) to avoid redundant work
+	std::set<uint32_t> _done;
 };

app/src/cpp/libcimbar/src/lib/image_hash/ahash_result.h

@@ -2,7 +2,7 @@
 #pragma once
 
 #include "bit_extractor.h"
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 #include <array>
 #include <iostream>
 #include <utility>

app/src/cpp/libcimbar/src/lib/image_hash/average_hash.h

@@ -6,7 +6,7 @@
 #include "bit_file/bitmatrix.h"
 #include "cimb_translator/Cell.h"
 
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 #include <opencv2/opencv.hpp>
 
 #include <array>
@@ -51,6 +51,10 @@ namespace image_hash
 				uint64_t mval = (*hax) & 0x101010101ULL;
 				const uint8_t* cv = reinterpret_cast<const uint8_t*>(&mval);
 				uint8_t val = cv[0] << 4 | cv[1] << 3 | cv[2] << 2 | cv[3] << 1 | cv[4];
+				// TODO:
+				/*if (bigEndian)
+					val = cv[7] << 4 | cv[6] << 3 | cv[5] << 2 | cv[4] << 1 | cv[3];*/ // ?
+
 				res |= intx::uint128(val) << bitpos;
 			}
 		}

app/src/cpp/libcimbar/src/lib/image_hash/test/bitExtractorTest.cpp

@@ -2,7 +2,7 @@
 #include "unittest.h"
 
 #include "bit_extractor.h"
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 
 #include <bitset>
 #include <iostream>
@@ -36,7 +36,9 @@ TEST_CASE( "bitExtractorTest/testLargerValue.1", "[unit]" )
 
 TEST_CASE( "bitExtractorTest/testLargerValue.2", "[unit]" )
 {
-	intx::uint128 bits{0xFFBFCFE3FULL, 0xF83C0E030080000ULL};
+	intx::uint128 bits{0xF83C0E030080000ULL, 0xFFBFCFE3FULL};
+	assertEquals( "ffbfcfe3f0f83c0e030080000", intx::hex(bits) );  // sanity check
+
 	bit_extractor<intx::uint128, 100> be(bits);
 	uint64_t res = be.extract(1, 11, 21, 31);
 	assertEquals( 0xfffefcf8, res );

app/src/cpp/libcimbar/src/third_party_lib/PicoSHA2/picosha2.h

@@ -93,7 +93,7 @@ void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last) {
     assert(first + 64 == last);
     static_cast<void>(last);  // for avoiding unused-variable warning
     word_t w[64];
-    std::fill(w, w + 64, 0);
+    std::fill(w, w + 64, word_t(0));
     for (std::size_t i = 0; i < 16; ++i) {
         w[i] = (static_cast<word_t>(mask_8bit(*(first + i * 4))) << 24) |
                (static_cast<word_t>(mask_8bit(*(first + i * 4 + 1))) << 16) |
@@ -185,7 +185,7 @@ class hash256_one_by_one {
 
     void init() {
         buffer_.clear();
-        std::fill(data_length_digits_, data_length_digits_ + 4, 0);
+        std::fill(data_length_digits_, data_length_digits_ + 4, word_t(0));
         std::copy(detail::initial_message_digest,
                   detail::initial_message_digest + 8, h_);
     }
@@ -204,17 +204,17 @@ class hash256_one_by_one {
 
     void finish() {
         byte_t temp[64];
-        std::fill(temp, temp + 64, 0);
+        std::fill(temp, temp + 64, byte_t(0));
         std::size_t remains = buffer_.size();
         std::copy(buffer_.begin(), buffer_.end(), temp);
         temp[remains] = 0x80;
 
         if (remains > 55) {
-            std::fill(temp + remains + 1, temp + 64, 0);
+            std::fill(temp + remains + 1, temp + 64, byte_t(0));
             detail::hash256_block(h_, temp, temp + 64);
-            std::fill(temp, temp + 64 - 4, 0);
+            std::fill(temp, temp + 64 - 4, byte_t(0));
         } else {
-            std::fill(temp + remains + 1, temp + 64 - 4, 0);
+            std::fill(temp + remains + 1, temp + 64 - 4, byte_t(0));
         }
 
         write_data_bit_length(&(temp[56]));

app/src/cpp/libcimbar/src/third_party_lib/concurrentqueue/LICENSE.md

@@ -1,6 +1,7 @@
 This license file applies to everything in this repository except that which
 is explicitly annotated as being written by other authors, i.e. the Boost
 queue (included in the benchmarks for comparison), Intel's TBB library (ditto),
+dlib::pipe (ditto),
 the CDSChecker tool (used for verification), the Relacy model checker (ditto),
 and Jeff Preshing's semaphore implementation (used in the blocking queue) which
 has a zlib license (embedded in lightweightsempahore.h).