Skip to content

Commit

Permalink
For encodes via the cimbar cli, skip over frames that will fail extract
Browse files Browse the repository at this point in the history
Some cimbar frames are bad and will fail extraction due to the data
aligning itself to match the anchor pattern.

This is a design bug/quirk that doesn't matter *too* much for the
on-the-fly "encode data on one screen, decode via camera" use case.
We're constantly generating new frames, after all. But for any offline,
more "QR-code" like usage -- for example printing -- having a bad image
is much more troublesome.

The fix/workaround is to check the generated image after we generate,
and make sure it extracts. If it doesn't, we skip that frame and again.

This is currently only used the cli (and various test code), due to
potential performance and infinite-loop concerns.
  • Loading branch information
sz3 committed Oct 21, 2024
1 parent 75d6cfb commit 1a77fe5
Show file tree
Hide file tree
Showing 6 changed files with 47 additions and 12 deletions.
1 change: 1 addition & 0 deletions src/exe/cimbar/cimbar.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ template <typename FilenameIterable>
int encode(const FilenameIterable& infiles, const std::string& outpath, int ecc, int color_bits, int compression_level, bool legacy_mode, bool no_fountain)
{
Encoder en(ecc, cimbar::Config::symbol_bits(), color_bits);
en.set_encode_id(109);
if (legacy_mode)
en.set_legacy_mode();
for (const string& f : infiles)
Expand Down
8 changes: 8 additions & 0 deletions src/lib/encoder/Encoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include "SimpleEncoder.h"
#include "cimb_translator/Config.h"
#include "extractor/Scanner.h"
#include "serialize/format.h"

#include <opencv2/opencv.hpp>
Expand Down Expand Up @@ -61,6 +62,13 @@ inline unsigned Encoder::encode_fountain(const std::string& filename, const std:
if (!frame)
break;

// some % of generated frames (for the current 8x8 impl)
// will produce random patterns that falsely match as
// corner "anchors" and fail to extract. So:
// if frame fails the scan, skip it.
if (!Scanner::will_it_scan(*frame))
continue;

if (!on_frame(*frame, i))
break;
++i;
Expand Down
1 change: 1 addition & 0 deletions src/lib/encoder/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ add_test(encoder_test encoder_test)
target_link_libraries(encoder_test

cimb_translator
extractor

correct_static
wirehair
Expand Down
2 changes: 1 addition & 1 deletion src/lib/encoder/test/EncoderRoundTripTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ TEST_CASE( "EncoderRoundTripTest/testFountain.Pad", "[unit]" )
Encoder enc(30, 4, 2);
assertEquals( 1, enc.encode_fountain(inputFile, outPrefix) );

uint64_t hash = 0xeecc8800efce8c48;
uint64_t hash = 0xefcc8800efce8c48;
std::string path = fmt::format("{}_0.png", outPrefix);
cv::Mat encodedImg = cv::imread(path);
cv::cvtColor(encodedImg, encodedImg, cv::COLOR_BGR2RGB);
Expand Down
12 changes: 6 additions & 6 deletions src/lib/encoder/test/EncoderTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ TEST_CASE( "EncoderTest/testFountain.4c", "[unit]" )
enc.set_legacy_mode();
assertEquals( 3, enc.encode_fountain(inputFile, outPrefix, 0) );

std::vector<uint64_t> hashes = {0xbb1cc62b662abfe5, 0xf586f6466a5b194, 0x8c2f0f40e6ecb08b};
std::vector<uint64_t> hashes = {0xbb1cc62b662abfe5, 0xf586f6466a5b194, 0x93a3830d042966e1};
for (unsigned i = 0; i < hashes.size(); ++i)
{
DYNAMIC_SECTION( "are we correct? : " << i )
Expand Down Expand Up @@ -93,7 +93,7 @@ TEST_CASE( "EncoderTest/testFountain.Compress", "[unit]" )
Encoder enc(30, 4, 2);
assertEquals( 1, enc.encode_fountain(inputFile, outPrefix) );

uint64_t hash = 0xb36b65402eec434e;
uint64_t hash = 0xa66a666543280e8e;
std::string path = fmt::format("{}_0.png", outPrefix);
cv::Mat img = cv::imread(path);
assertEquals( hash, image_hash::average_hash(img) );
Expand Down Expand Up @@ -131,12 +131,12 @@ TEST_CASE( "EncoderTest/testFountain.Size", "[unit]" )
std::string outPrefix = tempdir.path() / "encoder.fountain";

Encoder enc(30, 4, 2);
assertEquals( 1, enc.encode_fountain(inputFile, outPrefix, 16, 1.6, 1080) );
assertEquals( 1, enc.encode_fountain(inputFile, outPrefix, 16, 1.6, 1024) );

uint64_t hash = 0xbdc232c714226fe6;
uint64_t hash = 0xa66a666543280e8e;
std::string path = fmt::format("{}_0.png", outPrefix);
cv::Mat img = cv::imread(path);
assertEquals( 1080, img.rows );
assertEquals( 1080, img.cols );
assertEquals( 1024, img.rows );
assertEquals( 1024, img.cols );
assertEquals( hash, image_hash::average_hash(img) );
}
35 changes: 30 additions & 5 deletions src/lib/extractor/Scanner.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#pragma once

#include "Anchor.h"
#include "Corners.h"
#include "Point.h"
#include "ScanState.h"

Expand All @@ -10,7 +11,6 @@
#include <iostream>
#include <vector>

class Corners;
class Midpoints;

class Scanner
Expand All @@ -33,6 +33,10 @@ class Scanner

static unsigned nextPowerOfTwoPlusOne(unsigned v); // helper

// external helper method
template <typename MAT>
static bool will_it_scan(const MAT& img);

// rest of public interface
std::vector<Anchor> scan();
std::vector<point<int>> scan_edges(const Corners& corners, Midpoints& mps) const;
Expand Down Expand Up @@ -98,6 +102,27 @@ inline unsigned Scanner::nextPowerOfTwoPlusOne(unsigned v)
return std::max(3U, v + 2);
}

template <typename MAT>
inline bool Scanner::will_it_scan(const MAT& unpadded_img)
{
Scanner scanner(unpadded_img);
std::vector<Anchor> points = scanner.scan();
if (points.size() < 4)
return false;

constexpr int limit = 50;
Corners corners(points);
if (corners.top_left().x() > limit or corners.top_left().y() > limit)
return false;
if (corners.top_right().x() < (unpadded_img.cols - limit) or corners.top_right().y() > limit)
return false;
if (corners.bottom_left().x() > limit or corners.bottom_left().y() < (unpadded_img.rows - limit))
return false;
if (corners.bottom_right().x() < (unpadded_img.cols - limit) or corners.bottom_right().y() < (unpadded_img.rows - limit))
return false;
return true;
}

template <typename MAT, typename MAT2>
inline void Scanner::threshold_fast(const MAT& img, MAT2& out)
{
Expand Down Expand Up @@ -141,10 +166,10 @@ inline void Scanner::preprocess_image(const MAT& img, MAT2& out, bool fast)

template <typename MAT>
inline Scanner::Scanner(const MAT& img, bool fast, bool dark, int skip)
: _dark(dark)
, _skip(skip? skip : std::min(img.rows, img.cols) / 60)
, _mergeCutoff(img.cols / 30)
, _anchorSize(30)
: _dark(dark)
, _skip(skip? skip : std::min(img.rows, img.cols) / 60)
, _mergeCutoff(img.cols / 30)
, _anchorSize(30)
{
_img = preprocess_image(img, fast);
}
Expand Down

0 comments on commit 1a77fe5

Please sign in to comment.