Skip to content

Commit

Permalink
MSVC debug iterator fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@eliaskosunen
eliaskosunen committed Oct 23, 2023
1 parent 3d185d4 commit a82baf3
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 83 deletions.
133 changes: 51 additions & 82 deletions src/scn/impl/algorithms/find_whitespace.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ namespace scn {

namespace impl {
namespace {
bool has_nonascii_char_64(std::string_view source)
template <typename R>
bool has_nonascii_char_64(R source)
{
SCN_EXPECT(source.size() <= 8);
uint64_t word{};
Expand All @@ -34,126 +35,90 @@ namespace scn {
return has_byte_greater(word, 127) != 0;
}

template <typename Cb>
std::string_view::iterator find_impl_ascii(std::string_view sv,
Cb cb)
template <typename R, typename Cb>
auto find_impl_ascii(R sv, Cb cb)
{
return ranges::find_if(sv, cb);
}

template <typename Cb>
std::string_view::iterator find_impl_unicode_invalid(
std::string_view sv,
Cb cb)
auto find_impl_unicode_invalid(std::string_view sv, Cb cb)
{
auto it = sv.begin();
while (it != sv.end()) {
auto res = get_next_code_point(
detail::make_string_view_from_iterators<char>(
it, sv.end()));
auto it = sv.data();
while (it != sv.data() + sv.size()) {
auto tmp = std::string_view{
detail::to_address(it),
static_cast<size_t>(ranges::distance(
it, detail::to_address(sv.end())))};
auto res = get_next_code_point(tmp);
if (cb(res.value)) {
return it;
break;
}
it = res.iterator;
it += ranges::distance(tmp.data(),
detail::to_address(res.iterator));
}
return it;
return sv.begin() + ranges::distance(sv.data(), it);
}

template <typename Cb>
std::string_view::iterator find_impl_unicode_valid(
auto find_impl_unicode_valid(
std::string_view sv,
const std::array<char32_t, 8>& codepoints,
Cb cb)
{
for (size_t i = 0; i < codepoints.size(); ++i) {
if (cb(codepoints[i])) {
return sv.begin() + simdutf::utf8_length_from_utf32(
codepoints.data(), i);
return sv.begin() + static_cast<std::ptrdiff_t>(
simdutf::utf8_length_from_utf32(
codepoints.data(), i));
}
}
return sv.end();
}

std::string_view::iterator find_classic_space_impl(
std::string_view source)
template <typename CuCb, typename CpCb>
std::string_view::iterator
find_classic_impl(std::string_view source, CuCb cu_cb, CpCb cp_cb)
{
auto it = source.begin();
while (it != source.end()) {
auto sv = detail::make_string_view_from_iterators<char>(
it, source.end())
.substr(0, 8);

if (!has_nonascii_char_64(sv)) {
it = find_impl_ascii(
sv, [](char ch) { return is_ascii_space(ch); });
if (it != sv.end()) {
return it;
}
continue;
}
auto it = source.data();
const auto end = source.data() + source.size();

std::array<char32_t, 8> codepoints{};
auto ret = simdutf::convert_utf8_to_utf32(
&*it, sv.size(), codepoints.data());
if (SCN_UNLIKELY(ret == 0)) {
it = find_impl_unicode_invalid(
sv, [](char32_t cp) { return is_cp_space(cp); });
if (it != sv.end()) {
return it;
}
continue;
}

it = find_impl_unicode_valid(
sv, codepoints,
[](char32_t cp) { return is_cp_space(cp); });
if (it != sv.end()) {
return it;
}
}

return source.end();
}

std::string_view::iterator find_classic_nonspace_impl(
std::string_view source)
{
auto it = source.begin();
while (it != source.end()) {
auto sv = detail::make_string_view_from_iterators<char>(
it, source.end())
.substr(0, 8);
while (it != end) {
auto sv =
std::string_view{
it, static_cast<size_t>(ranges::distance(
it, detail::to_address(source.end())))}
.substr(0, 8);

if (!has_nonascii_char_64(sv)) {
it = find_impl_ascii(
sv, [](char ch) { return !is_ascii_space(ch); });
if (it != sv.end()) {
return it;
auto i = find_impl_ascii(sv, cu_cb);
it = detail::to_address(i);
if (i != sv.end()) {
break;
}
continue;
}

std::array<char32_t, 8> codepoints{};
auto ret = simdutf::convert_utf8_to_utf32(
&*it, sv.size(), codepoints.data());
detail::to_address(it), sv.size(), codepoints.data());
if (SCN_UNLIKELY(ret == 0)) {
it = find_impl_unicode_invalid(
sv, [](char32_t cp) { return !is_cp_space(cp); });
if (it != sv.end()) {
return it;
auto i = find_impl_unicode_invalid(sv, cp_cb);
it = detail::to_address(i);
if (i != sv.end()) {
break;
}
continue;
}

it = find_impl_unicode_valid(
sv, codepoints,
[](char32_t cp) { return !is_cp_space(cp); });
if (it != sv.end()) {
return it;
auto i = find_impl_unicode_valid(sv, codepoints, cp_cb);
it = detail::to_address(i);
if (i != sv.end()) {
break;
}
}

return source.end();
return source.begin() + ranges::distance(source.data(), it);
}

bool is_decimal_digit(char ch) SCN_NOEXCEPT
Expand Down Expand Up @@ -208,13 +173,17 @@ namespace scn {
std::string_view::iterator find_classic_space_narrow_fast(
std::string_view source)
{
return find_classic_space_impl(source);
return find_classic_impl(
source, [](char ch) { return is_ascii_space(ch); },
[](char32_t cp) { return is_cp_space(cp); });
}

std::string_view::iterator find_classic_nonspace_narrow_fast(
std::string_view source)
{
return find_classic_nonspace_impl(source);
return find_classic_impl(
source, [](char ch) { return !is_ascii_space(ch); },
[](char32_t cp) { return !is_cp_space(cp); });
}

std::string_view::iterator find_nondecimal_digit_narrow_fast(
Expand Down
3 changes: 3 additions & 0 deletions src/scn/impl/algorithms/read.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,9 @@ namespace scn {
read_matching_code_unit(Range&& range, detail::char_t<Range> ch)
{
auto it = read_code_unit(range);
if (SCN_UNLIKELY(!it)) {
return unexpected(it.error());
}

if (SCN_UNLIKELY(*ranges::begin(range) !=
static_cast<detail::char_t<Range>>(ch))) {
Expand Down
2 changes: 1 addition & 1 deletion src/scn/impl/unicode/unicode.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ namespace scn {
sizeof(DestCharT) == 2) {
return simdutf::convert_utf8_to_utf16_with_errors(
src.data(), src.size(),
reinterpret_cast<char32_t*>(dst.data()));
reinterpret_cast<char16_t*>(dst.data()));
}
else if constexpr (sizeof(SourceCharT) == 1 &&
sizeof(DestCharT) == 4) {
Expand Down

0 comments on commit a82baf3

Please sign in to comment.