drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2025-12-21 13:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix TokenizeIterator on Windows

Browse Source 
Signed-off-by: Osyotr <Osyotr@users.noreply.github.com>
This commit is contained in:
Osyotr
2025-07-27 13:32:44 +03:00
committed by Konstantin Pastbin
parent dfd7af92fa
commit b5b333c832
1 changed files with 15 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
libs/base/string_utils.hpp
View File

@@ -150,25 +150,30 @@ inline std::string DebugPrint(UniString const & s)
return ToUtf8(s); return ToUtf8(s);
} }
template <typename DelimFn, typename Iter>
class TokenizeIteratorBase class TokenizeIteratorBase
{ {
template <typename...>
static constexpr bool is_utf8cpp_iterator = false;
template <typename... Args>
static constexpr bool is_utf8cpp_iterator<utf8::unchecked::iterator<Args...>> = true;
public: public:
using difference_type = std::ptrdiff_t; using difference_type = std::ptrdiff_t;
using iterator_category = std::input_iterator_tag; using iterator_category = std::input_iterator_tag;
// Hack to get buffer pointer from any iterator.
// Deliberately made non-static to simplify the call like this->ToCharPtr.
char const * ToCharPtr(char const * p) const { return p; }
template <class T> template <class T>
auto ToCharPtr(T const & i) const static std::string_view ToStringView(T first, T last)
{ {
return ToCharPtr(i.base()); if constexpr (is_utf8cpp_iterator<T>)
return TokenizeIteratorBase::ToStringView(first.base(), last.base());
else
return std::string_view(first, last);
} }
}; };
template <typename DelimFn, typename Iter, bool KeepEmptyTokens = false> template <typename DelimFn, typename Iter, bool KeepEmptyTokens = false>
class TokenizeIterator : public TokenizeIteratorBase<DelimFn, Iter> class TokenizeIterator : public TokenizeIteratorBase
{ {
public: public:
template <class InIterT> template <class InIterT>
@@ -185,8 +190,7 @@ public:
{ {
ASSERT(m_start != m_finish, ("Dereferencing of empty iterator.")); ASSERT(m_start != m_finish, ("Dereferencing of empty iterator."));
auto const baseI = this->ToCharPtr(m_start); return this->ToStringView(m_start, m_end);
return std::string_view(baseI, std::distance(baseI, this->ToCharPtr(m_end)));
} }
UniString GetUniString() const UniString GetUniString() const
@@ -246,7 +250,7 @@ private:
/// Used in ParseCSVRow for the generator routine. /// Used in ParseCSVRow for the generator routine.
template <typename DelimFn, typename Iter> template <typename DelimFn, typename Iter>
class TokenizeIterator<DelimFn, Iter, true /* KeepEmptyTokens */> : public TokenizeIteratorBase<DelimFn, Iter> class TokenizeIterator<DelimFn, Iter, true /* KeepEmptyTokens */> : public TokenizeIteratorBase
{ {
public: public:
template <class InIterT> template <class InIterT>
@@ -265,8 +269,7 @@ public:
{ {
ASSERT(!m_finished, ("Dereferencing of empty iterator.")); ASSERT(!m_finished, ("Dereferencing of empty iterator."));
auto const baseI = this->ToCharPtr(m_start); return this->ToStringView(m_start, m_end);
return std::string_view(baseI, std::distance(baseI, this->ToCharPtr(m_end)));
} }
explicit operator bool() const { return !m_finished; } explicit operator bool() const { return !m_finished; }