#include "routing/routing_quality/routing_quality_tool/utils.hpp" #include "routing/routing_quality/routing_quality_tool/benchmark_stat.hpp" #include "routing/routing_quality/routing_quality_tool/error_type_counter.hpp" #include "routing/routing_quality/api/api.hpp" #include "routing/routing_quality/waypoints.hpp" #include "routing/routes_builder/routes_builder.hpp" #include "platform/platform.hpp" #include "base/exception.hpp" #include "base/file_name_utils.hpp" #include "base/logging.hpp" #include #include #include #include #include #include DEFINE_string(mapsme_old_results, "", "Path to directory with previous mapsme router results."); DEFINE_string(mapsme_results, "", "Path to directory with mapsme router results."); DEFINE_string(api_results, "", "Path to directory with api router results."); DEFINE_string(save_results, "", "The directory where results of tool will be saved."); DEFINE_double(kml_percent, 0.0, "The percent of routes for which kml file will be generated." "With kml files you can make screenshots with a desktop app"); DEFINE_bool(benchmark_stat, false, "Dump statistics about route time building."); namespace { // Shows distribution of simularity in comparison mode. static std::string const kPythonDistribution = "show_distribution.py"; static std::string const kPythonBarDistributionError = "show_errors_distribution.py"; double constexpr kBadETADiffPercent = std::numeric_limits::max(); void PrintHelpAndExit() { std::stringstream usage; usage << std::boolalpha; usage << R"(Description: This tool takes two paths to directory with routes, that were dumped by routes_builder_tool and calculate some metrics. Here possible options of execution: --mapsme_results and --api_results are required to compare mapsme and api. or --mapsme_results and --mapsme_old_results are required to compare mapsme and old mapsme version. or --mapsme_results and --benchmark_stat are required to calculate benchmark statistic of mapsme. or --mapsme_results and --mapsme_old_results and --benchmark_stat are required to calculate different statistics of comparison mapsme and old mapsme version. --save_results is required for saving results in this directory. --kml_percent may be used in non-benchamrk mode for dumping kmls and visual comparison different routes. )"; auto const addStringInfo = [&usage](auto const & arg, auto const & argValue) { using T = decltype(argValue); usage << "\n\t" << arg << " is"; if (argValue == T{}) usage << " not set."; else usage << " set to: " << argValue; }; addStringInfo("--mapsme_results", FLAGS_mapsme_results); addStringInfo("--api_results", FLAGS_api_results); addStringInfo("--mapsme_old_results", FLAGS_mapsme_old_results); addStringInfo("--benchmark_stat", FLAGS_benchmark_stat); addStringInfo("--save_results", FLAGS_save_results); addStringInfo("--kml_percent", FLAGS_kml_percent); usage << "\n\nType --help for usage.\n"; std::cout << usage.str(); exit(0); } bool HasHelpFlags(int argc, char ** argv) { for (int i = 0; i < argc; ++i) { auto const value = std::string(argv[i]); if (value == "--help" || value == "-h") return true; } return false; } } // namespace using namespace routing; using namespace routes_builder; using namespace routing_quality; using namespace routing_quality_tool; void PrintResults(std::vector && results, RoutesSaver & routesSaver) { double sumSimilarity = 0.0; double sumETADiffPercent = 0.0; double goodETANumber = 0.0; for (auto const & result : results) { sumSimilarity += result.m_similarity; if (result.m_etaDiffPercent != kBadETADiffPercent) { sumETADiffPercent += result.m_etaDiffPercent; goodETANumber += 1; } } LOG(LINFO, ("Matched routes:", results.size())); LOG(LINFO, ("Average similarity:", sumSimilarity / results.size())); LOG(LINFO, ("Average eta difference by fullmathed routes:", sumETADiffPercent / goodETANumber, "%")); auto const pythonScriptPath = base::JoinPath(FLAGS_save_results, kPythonDistribution); std::vector values; values.reserve(results.size()); for (auto const & result : results) values.emplace_back(result.m_similarity); CreatePythonScriptForDistribution(pythonScriptPath, "Simularity distribution", values); SimilarityCounter similarityCounter(routesSaver); std::sort(results.begin(), results.end()); for (auto const & result : results) similarityCounter.Push(result); similarityCounter.CreateKmlFiles(FLAGS_kml_percent, results); } bool IsMapsmeVsApi() { return !FLAGS_mapsme_results.empty() && !FLAGS_api_results.empty(); } bool IsMapsmeVsMapsme() { return !FLAGS_mapsme_results.empty() && !FLAGS_mapsme_old_results.empty() && !FLAGS_benchmark_stat; } bool IsMapsmeBenchmarkStat() { return !FLAGS_mapsme_results.empty() && FLAGS_benchmark_stat && FLAGS_mapsme_old_results.empty(); } bool IsMapsmeVsMapsmeBenchmarkStat() { return !FLAGS_mapsme_results.empty() && FLAGS_benchmark_stat && !FLAGS_mapsme_old_results.empty(); } void CheckDirExistence(std::string const & dir) { CHECK(Platform::IsDirectory(dir), ("Can not find directory:", dir)); } template void RunComparison(std::vector> && mapsmeResults, std::vector> && anotherResults) { ErrorTypeCounter mapsmeErrorCounter; ErrorTypeCounter anotherErrorCounter; ComparisonType type = IsMapsmeVsApi() ? ComparisonType::MapsmeVsApi : ComparisonType::MapsmeVsMapsme; RoutesSaver routesSaver(FLAGS_save_results, type); std::vector results; size_t apiErrors = 0; for (size_t i = 0; i < mapsmeResults.size(); ++i) { auto const & mapsmeResult = mapsmeResults[i].first; auto const & mapsmeFile = mapsmeResults[i].second; std::pair anotherResultPair; if (!FindAnotherResponse(mapsmeResult, anotherResults, anotherResultPair)) { LOG(LDEBUG, ("Can not find pair for:", i)); continue; } auto const & anotherResult = anotherResultPair.first; auto const & anotherFile = anotherResultPair.second; auto const & startLatLon = mercator::ToLatLon(anotherResult.GetStartPoint()); auto const & finishLatLon = mercator::ToLatLon(anotherResult.GetFinishPoint()); mapsmeErrorCounter.PushError(mapsmeResult.m_code); anotherErrorCounter.PushError(anotherResult.m_code); if (!mapsmeResult.IsCodeOK() && anotherResult.IsCodeOK()) { routesSaver.PushError(mapsmeResult.m_code, startLatLon, finishLatLon); continue; } if (anotherResult.GetRoutes().empty() || !anotherResult.IsCodeOK()) { routesSaver.PushRivalError(startLatLon, finishLatLon); ++apiErrors; continue; } auto maxSimilarity = std::numeric_limits::min(); double etaDiff = kBadETADiffPercent; auto const & mapsmeRoute = mapsmeResult.GetRoutes().back(); for (auto const & route : anotherResult.GetRoutes()) { auto const similarity = metrics::CompareByNumberOfMatchedWaypoints(mapsmeRoute.m_followedPolyline, route.GetWaypoints()); if (maxSimilarity < similarity) { maxSimilarity = similarity; if (maxSimilarity == 1.0) { etaDiff = 100.0 * std::abs(route.GetETA() - mapsmeRoute.GetETA()) / std::max(route.GetETA(), mapsmeRoute.GetETA()); } } } results.emplace_back(mapsmeFile, anotherFile, startLatLon, finishLatLon, maxSimilarity, etaDiff); } std::string const anotherSourceName = IsMapsmeVsMapsme() ? "old mapsme," : "api,"; LOG(LINFO, (apiErrors, "routes can not build via", anotherSourceName, "but mapsme do built them.")); PrintResults(std::move(results), routesSaver); std::vector errorLabels; std::vector> errorsCount; FillLabelsAndErrorTypeDistribution(errorLabels, errorsCount, mapsmeErrorCounter, anotherErrorCounter); auto const pythonScriptPath = base::JoinPath(FLAGS_save_results, kPythonBarDistributionError); CreatePythonBarByMap(pythonScriptPath, errorLabels, errorsCount, {"mapsme", IsMapsmeVsMapsme() ? "old mapsme" : "api"} /* legends */, "Type of errors" /* xlabel */, "Number of errors" /* ylabel */); } void CheckArgs() { bool const modeIsChosen = IsMapsmeVsApi() || IsMapsmeVsMapsme() || IsMapsmeBenchmarkStat() || IsMapsmeVsMapsmeBenchmarkStat(); if (!modeIsChosen) PrintHelpAndExit(); CHECK(!FLAGS_save_results.empty(), ("\n\n\t--save_results is required. Tool will save results there.", "\n\nType --help for usage.")); } int Main(int argc, char ** argv) { if (HasHelpFlags(argc, argv)) PrintHelpAndExit(); gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); CheckArgs(); if (Platform::IsFileExistsByFullPath(FLAGS_save_results)) CheckDirExistence(FLAGS_save_results); else CHECK_EQUAL(Platform::MkDir(FLAGS_save_results), Platform::EError::ERR_OK,()); CheckDirExistence(FLAGS_mapsme_results); CHECK(0.0 <= FLAGS_kml_percent && FLAGS_kml_percent <= 100.0, ("--kml_percent should be in interval: [0.0, 100.0].")); LOG(LINFO, ("Start loading mapsme results.")); auto mapsmeResults = LoadResults(FLAGS_mapsme_results); LOG(LINFO, ("Receive:", mapsmeResults.size(), "routes from --mapsme_results.")); if (IsMapsmeVsApi()) { LOG(LINFO, ("Start loading api results.")); auto apiResults = LoadResults(FLAGS_api_results); LOG(LINFO, ("Receive:", apiResults.size(), "routes from --api_results.")); RunComparison(std::move(mapsmeResults), std::move(apiResults)); } else if (IsMapsmeVsMapsmeBenchmarkStat()) { LOG(LINFO, ("Benchmark different mapsme versions. Start loading old mapsme results.")); auto oldMapsmeResults = LoadResults(FLAGS_mapsme_old_results); LOG(LINFO, ("Receive:", oldMapsmeResults.size(), "routes from --mapsme_old_results.")); RunBenchmarkComparison(std::move(mapsmeResults), std::move(oldMapsmeResults), FLAGS_save_results); } else if (IsMapsmeVsMapsme()) { LOG(LINFO, ("Start loading another mapsme results.")); auto oldMapsmeResults = LoadResults(FLAGS_mapsme_old_results); LOG(LINFO, ("Receive:", oldMapsmeResults.size(), "routes from --mapsme_old_results.")); RunComparison(std::move(mapsmeResults), std::move(oldMapsmeResults)); } else if (IsMapsmeBenchmarkStat()) { LOG(LINFO, ("Running in benchmark stat mode.")); RunBenchmarkStat(mapsmeResults, FLAGS_save_results); } else { UNREACHABLE(); } return 0; } int main(int argc, char ** argv) { try { return Main(argc, argv); } catch (RootException const & e) { LOG(LCRITICAL, ("Core exception:", e.Msg())); } catch (std::exception const & e) { LOG(LCRITICAL, ("Std exception:", e.what())); } catch (...) { LOG(LCRITICAL, ("Unknown exception.")); } LOG(LINFO, ("Done.")); return 0; }