diff --git a/generator/maxspeeds_parser.cpp b/generator/maxspeeds_parser.cpp index 072777117..94a9ae3a0 100644 --- a/generator/maxspeeds_parser.cpp +++ b/generator/maxspeeds_parser.cpp @@ -173,7 +173,7 @@ bool ParseMaxspeedTag(std::string const & maxspeedValue, routing::SpeedInUnits & speedStr += maxspeedValue[i]; } - while (i < maxspeedValue.size() && isspace(maxspeedValue[i])) + while (i < maxspeedValue.size() && strings::IsASCIISpace(maxspeedValue[i])) ++i; if (maxspeedValue.size() == i || maxspeedValue.substr(i).starts_with("kmh")) diff --git a/libs/base/string_utils.cpp b/libs/base/string_utils.cpp index b6369f034..24ccc63a1 100644 --- a/libs/base/string_utils.cpp +++ b/libs/base/string_utils.cpp @@ -11,6 +11,7 @@ #include #include +#include namespace strings { @@ -227,15 +228,15 @@ void AsciiToUpper(std::string & s) void Trim(std::string & s) { - boost::trim_if(s, ::isspace); + boost::trim_if(s, IsASCIISpace); } void Trim(std::string_view & sv) { - auto const beg = std::find_if(sv.cbegin(), sv.cend(), [](auto c) { return !std::isspace(c); }); + auto const beg = std::find_if(sv.cbegin(), sv.cend(), [](auto c) { return !IsASCIISpace(c); }); if (beg != sv.end()) { - auto const end = std::find_if(sv.crbegin(), sv.crend(), [](auto c) { return !std::isspace(c); }).base(); + auto const end = std::find_if(sv.crbegin(), sv.crend(), [](auto c) { return !IsASCIISpace(c); }).base(); sv = std::string_view(sv.data() + std::distance(sv.begin(), beg), std::distance(beg, end)); } else @@ -355,11 +356,6 @@ bool IsASCIIDigit(UniChar c) return c >= '0' && c <= '9'; } -bool IsASCIISpace(UniChar c) -{ - return c == ' ' || c == '\f' || c == '\n' || c == '\r' || c == '\t' || c == '\v'; -} - bool IsASCIILatin(UniChar c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); diff --git a/libs/base/string_utils.hpp b/libs/base/string_utils.hpp index 6bb77b17d..3bcd603f5 100644 --- a/libs/base/string_utils.hpp +++ b/libs/base/string_utils.hpp @@ -105,9 +105,7 @@ size_t CountNormLowerSymbols(UniString const & s, UniString const & lowStr); void AsciiToLower(std::string & s); void AsciiToUpper(std::string & s); -// All triming functions return a reference on an input string. -// They do in-place trimming. In general, it does not work for any unicode whitespace except -// ASCII U+0020 one. +// All trimming functions do in-place trimming. Only ASCII whitespaces are trimmed. void Trim(std::string & s); void Trim(std::string_view & sv); /// Remove any characters that contain in "anyOf" on left and right side of string s @@ -156,7 +154,23 @@ inline bool IsASCIINumeric(char const * s) { return IsASCIINumeric(std::string_view(s)); } -bool IsASCIISpace(UniChar c); + +// std::isspace is locale-dependent and fails for trailing UTF-8 characters. +template +inline constexpr bool IsASCIISpace(T c) +{ + switch (c) + { + case ' ': + case '\f': + case '\n': + case '\r': + case '\t': + case '\v': return true; + default: return false; + } +} + bool IsASCIILatin(UniChar c); inline std::string DebugPrint(UniString const & s) diff --git a/libs/indexer/road_shields_parser.cpp b/libs/indexer/road_shields_parser.cpp index 660fd10d2..d9a7e2d9d 100644 --- a/libs/indexer/road_shields_parser.cpp +++ b/libs/indexer/road_shields_parser.cpp @@ -240,7 +240,7 @@ public: { std::string shieldText(rawText); - std::erase_if(shieldText, [](char c) { return c == '-' || ::isspace(c); }); + std::erase_if(shieldText, [](char c) { return c == '-' || strings::IsASCIISpace(c); }); if (shieldText.size() <= 2) return RoadShield(RoadShieldType::Default, rawText); diff --git a/libs/platform/measurement_utils.cpp b/libs/platform/measurement_utils.cpp index 25f155259..94000e157 100644 --- a/libs/platform/measurement_utils.cpp +++ b/libs/platform/measurement_utils.cpp @@ -279,7 +279,7 @@ bool OSMDistanceToMeters(std::string const & osmRawValue, double & outMeters) case ';': return false; } - while (*stop && isspace(*stop)) + while (*stop && strings::IsASCIISpace(*stop)) ++stop; // Default units - meters. diff --git a/libs/routing/turns.cpp b/libs/routing/turns.cpp index 012512555..21b82a0ab 100644 --- a/libs/routing/turns.cpp +++ b/libs/routing/turns.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace routing @@ -312,7 +313,7 @@ bool ParseLanes(string lanesString, vector & lanes) return false; lanes.clear(); strings::AsciiToLower(lanesString); - base::EraseIf(lanesString, [](char c) { return isspace(c); }); + base::EraseIf(lanesString, strings::IsASCIISpace); vector SplitLanesStrings; SingleLaneInfo lane; diff --git a/libs/search/processor.cpp b/libs/search/processor.cpp index b8665219d..b0273f097 100644 --- a/libs/search/processor.cpp +++ b/libs/search/processor.cpp @@ -86,7 +86,7 @@ void RemoveStopWordsIfNeeded(QueryTokens & tokens, strings::UniString & prefix) void TrimLeadingSpaces(string & s) { - while (!s.empty() && isspace(s.front())) + while (!s.empty() && strings::IsASCIISpace(s.front())) s = s.substr(1); }