This commit is contained in:
Viktor Govako
2023-11-10 01:05:06 -03:00
committed by Konstantin Pastbin
parent 3b46dd1dee
commit 128b0f3e2b
10 changed files with 105 additions and 187 deletions

View File

@@ -13,7 +13,6 @@
#include "coding/varint.hpp"
#include "geometry/mercator.hpp"
#include "geometry/parametrized_segment.hpp"
#include "geometry/simplification.hpp"
#include "base/assert.hpp"
@@ -24,20 +23,18 @@
#include <cmath>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <mutex>
#include <vector>
#include "base/assert.hpp"
#include "base/string_utils.hpp"
#include "defines.hpp"
namespace borders
{
namespace
{
template <class ToDo>
void ForEachCountry(std::string const & baseDir, ToDo && toDo)
{
@@ -49,11 +46,11 @@ void ForEachCountry(std::string const & baseDir, ToDo && toDo)
Platform::GetFilesByExt(bordersDir, BORDERS_EXTENSION, files);
for (std::string file : files)
{
std::vector<m2::RegionD> polygons;
PolygonsList polygons;
if (LoadBorders(bordersDir + file, polygons))
{
base::GetNameWithoutExt(file);
toDo(file, polygons);
toDo(std::move(file), std::move(polygons));
}
}
}
@@ -65,7 +62,7 @@ public:
{
}
void operator()(std::string const & name, std::vector<m2::RegionD> const & borders)
void operator()(std::string name, PolygonsList && borders)
{
// use index in vector as tag
auto w = m_writer.GetWriter(strings::to_string(m_polys.size()));
@@ -105,7 +102,7 @@ private:
std::vector<storage::CountryDef> m_polys;
};
bool ReadPolygon(std::istream & stream, m2::RegionD & region, std::string const & filename)
bool ReadPolygon(std::istream & stream, Polygon & poly, std::string const & filename)
{
std::string line, name;
double lon, lat;
@@ -130,7 +127,7 @@ bool ReadPolygon(std::istream & stream, m2::RegionD & region, std::string const
iss >> lon >> lat;
CHECK(!iss.fail(), ("Incorrect data in", filename));
region.AddPoint(mercator::FromLatLon(lat, lon));
poly.AddPoint(mercator::FromLatLon(lat, lon));
}
// drop inner rings
@@ -146,7 +143,7 @@ bool CountryPolygons::Contains(m2::PointD const & point) const
});
}
bool LoadBorders(std::string const & borderFile, std::vector<m2::RegionD> & outBorders)
bool LoadBorders(std::string const & borderFile, PolygonsList & outBorders)
{
std::ifstream stream(borderFile);
std::string line;
@@ -156,12 +153,12 @@ bool LoadBorders(std::string const & borderFile, std::vector<m2::RegionD> & outB
return false;
}
m2::RegionD currentPolygon;
Polygon currentPolygon;
while (ReadPolygon(stream, currentPolygon, borderFile))
{
CHECK(currentPolygon.IsValid(), ("Invalid region in", borderFile));
outBorders.emplace_back(std::move(currentPolygon));
currentPolygon = m2::RegionD();
currentPolygon = {};
}
CHECK(!outBorders.empty(), ("No borders were loaded from", borderFile));
@@ -178,7 +175,7 @@ bool GetBordersRect(std::string const & baseDir, std::string const & country,
return false;
}
std::vector<m2::RegionD> borders;
PolygonsList borders;
CHECK(LoadBorders(bordersFile, borders), ());
bordersRect.MakeEmpty();
for (auto const & border : borders)
@@ -192,13 +189,16 @@ CountryPolygonsCollection LoadCountriesList(std::string const & baseDir)
LOG(LINFO, ("Loading countries in", BORDERS_DIR, "folder in", baseDir));
CountryPolygonsCollection countryPolygonsCollection;
ForEachCountry(baseDir, [&](auto const & name, auto const & borders)
ForEachCountry(baseDir, [&](std::string name, PolygonsList && borders)
{
PolygonsTree polygons;
for (m2::RegionD const & border : borders)
polygons.Add(border, border.GetRect());
for (Polygon & border : borders)
{
auto const rect = border.GetRect();
polygons.Add(std::move(border), rect);
}
countryPolygonsCollection.Add(CountryPolygons(name, polygons));
countryPolygonsCollection.Add(CountryPolygons(std::move(name), std::move(polygons)));
});
LOG(LINFO, ("Countries loaded:", countryPolygonsCollection.GetSize()));
@@ -214,7 +214,7 @@ void GeneratePackedBorders(std::string const & baseDir)
}
void DumpBorderToPolyFile(std::string const & targetDir, storage::CountryId const & mwmName,
std::vector<m2::RegionD> const & polygons)
PolygonsList const & polygons)
{
CHECK(!polygons.empty(), ());
@@ -222,7 +222,8 @@ void DumpBorderToPolyFile(std::string const & targetDir, storage::CountryId cons
std::ofstream poly(filePath);
CHECK(poly.good(), ());
poly << std::setprecision(20) << std::fixed;
// Used to have fixed precicion with 6 digits. And Alaska has 4 digits after comma :) Strange, but as is.
poly << std::setprecision(6) << std::fixed;
poly << mwmName << std::endl;
size_t polygonId = 1;

View File

@@ -1,7 +1,5 @@
#pragma once
#include "generator/feature_builder.hpp"
#include "storage/storage_defines.hpp"
#include "coding/geometry_coding.hpp"
@@ -11,12 +9,9 @@
#include "geometry/region2d.hpp"
#include "geometry/tree4d.hpp"
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#define BORDERS_DIR "borders/"
@@ -48,9 +43,8 @@ class CountryPolygons
{
public:
CountryPolygons() = default;
explicit CountryPolygons(std::string const & name, PolygonsTree const & regions)
: m_name(name)
, m_polygons(regions)
explicit CountryPolygons(std::string && name, PolygonsTree && regions)
: m_name(std::move(name)), m_polygons(std::move(regions))
{
}
@@ -97,6 +91,8 @@ public:
private:
std::string m_name;
/// @todo Is it an overkill to store Tree4D for each country's polygon?
PolygonsTree m_polygons;
};
@@ -105,11 +101,15 @@ class CountryPolygonsCollection
public:
CountryPolygonsCollection() = default;
void Add(CountryPolygons const & countryPolygons)
void Add(CountryPolygons && countryPolygons)
{
auto const it = m_countryPolygonsMap.emplace(countryPolygons.GetName(), countryPolygons);
countryPolygons.ForEachPolygon([&](auto const & polygon) {
m_regionsTree.Add(it.first->second, polygon.GetRect());
auto const res = m_countryPolygonsMap.emplace(countryPolygons.GetName(), std::move(countryPolygons));
CHECK(res.second, ());
auto const & inserted = res.first->second;
inserted.ForEachPolygon([&inserted, this](Polygon const & polygon)
{
m_regionsTree.Add(inserted, polygon.GetRect());
});
}
@@ -119,9 +119,10 @@ public:
void ForEachCountryInRect(m2::RectD const & rect, ToDo && toDo) const
{
std::unordered_set<CountryPolygons const *> uniq;
m_regionsTree.ForEachInRect(rect, [&](auto const & countryPolygons) {
if (uniq.emplace(&countryPolygons.get()).second)
toDo(countryPolygons);
m_regionsTree.ForEachInRect(rect, [&](CountryPolygons const & cp)
{
if (uniq.insert(&cp).second)
toDo(cp);
});
}
@@ -142,8 +143,10 @@ private:
std::unordered_map<std::string, CountryPolygons> m_countryPolygonsMap;
};
using PolygonsList = std::vector<Polygon>;
/// @return false if borderFile can't be opened
bool LoadBorders(std::string const & borderFile, std::vector<m2::RegionD> & outBorders);
bool LoadBorders(std::string const & borderFile, PolygonsList & outBorders);
bool GetBordersRect(std::string const & baseDir, std::string const & country,
m2::RectD & bordersRect);
@@ -153,10 +156,10 @@ bool LoadCountriesList(std::string const & baseDir, CountryPolygonsCollection &
void GeneratePackedBorders(std::string const & baseDir);
template <typename Source>
std::vector<m2::RegionD> ReadPolygonsOfOneBorder(Source & src)
PolygonsList ReadPolygonsOfOneBorder(Source & src)
{
auto const count = ReadVarUint<uint32_t>(src);
std::vector<m2::RegionD> result(count);
PolygonsList result(count);
for (size_t i = 0; i < count; ++i)
{
std::vector<m2::PointD> points;
@@ -168,7 +171,7 @@ std::vector<m2::RegionD> ReadPolygonsOfOneBorder(Source & src)
}
void DumpBorderToPolyFile(std::string const & filePath, storage::CountryId const & mwmName,
std::vector<m2::RegionD> const & polygons);
PolygonsList const & polygons);
void UnpackBorders(std::string const & baseDir, std::string const & targetDir);
CountryPolygonsCollection const & GetOrCreateCountryPolygonsTree(std::string const & baseDir);

View File

@@ -157,6 +157,7 @@ public:
}
Container const & Data() const { return m_points; }
Container & MutableData() { return m_points; }
template <typename EqualFn>
static bool IsIntersect(Coord const & x11, Coord const & y11, Coord const & x12,

View File

@@ -18,7 +18,7 @@
#include "base/thread_pool_computational.hpp"
#include <algorithm>
#include <future>
#include <iostream>
#include <tuple>
namespace
@@ -133,22 +133,16 @@ void SwapIfNeeded(size_t & a, size_t & b)
namespace poly_borders
{
// BordersData::Processor --------------------------------------------------------------------------
void BordersData::Processor::operator()(size_t borderId)
{
if (ShouldLog(borderId, m_data.m_bordersPolygons.size()))
LOG(LINFO, ("Marking:", borderId + 1, "/", m_data.m_bordersPolygons.size()));
auto const & polygon = m_data.m_bordersPolygons[borderId];
for (size_t pointId = 0; pointId < polygon.m_points.size(); ++pointId)
m_data.MarkPoint(borderId, pointId);
}
// BordersData -------------------------------------------------------------------------------------
void BordersData::Init(std::string const & bordersDir)
{
LOG(LINFO, ("Borders path:", bordersDir));
// key - coordinates
// value - {border idx, point idx}
std::unordered_map<int64_t, std::vector<std::pair<size_t, size_t>>> index;
std::vector<std::string> files;
Platform::GetFilesByExt(bordersDir, kBorderExtension, files);
@@ -159,11 +153,14 @@ void BordersData::Init(std::string const & bordersDir)
auto const fullPath = base::JoinPath(bordersDir, file);
size_t polygonId = 1;
std::vector<m2::RegionD> borders;
borders::PolygonsList borders;
borders::LoadBorders(fullPath, borders);
for (auto const & region : borders)
for (auto & region : borders)
{
Polygon polygon;
auto & points = region.MutableData();
m_duplicatedPointsCount += RemoveDuplicatingPointImpl(points);
CHECK_GREATER(points.size(), 1, (fullPath));
// Some mwms have several polygons. For example, for Japan_Kanto_Tokyo that has 2 polygons we
// will write 2 files:
// Japan_Kanto_Tokyo.poly1
@@ -172,35 +169,27 @@ void BordersData::Init(std::string const & bordersDir)
m_indexToPolyFileName[prevIndex] = fileCopy;
m_polyFileNameToIndex[fileCopy] = prevIndex++;
for (auto const & point : region.Data())
polygon.m_points.emplace_back(point);
polygon.m_rect = region.GetRect();
size_t const borderIdx = m_bordersPolygons.size();
for (size_t i = 0; i < points.size(); ++i)
index[PointToInt64Obsolete(points[i], kPointCoordBits)].emplace_back(borderIdx, i);
++polygonId;
m_bordersPolygons.emplace_back(std::move(polygon));
m_bordersPolygons.emplace_back(region.GetRect(), points);
}
}
m_duplicatedPointsCount += RemoveDuplicatePoints();
LOG(LINFO, ("Removed:", m_duplicatedPointsCount, "from input data."));
}
void BordersData::MarkPoints()
{
size_t const threadsNumber = std::thread::hardware_concurrency();
LOG(LINFO, ("Start marking points, threads number:", threadsNumber));
base::ComputationalThreadPool threadPool(threadsNumber);
std::vector<std::future<void>> tasks;
for (size_t i = 0; i < m_bordersPolygons.size(); ++i)
for (auto const & [_, v] : index)
{
Processor processor(*this);
tasks.emplace_back(threadPool.Submit(processor, i));
for (size_t i = 0; i < v.size() - 1; ++i)
for (size_t j = i + 1; j < v.size(); ++j)
{
m_bordersPolygons[v[i].first].m_points[v[i].second].AddLink(v[j].first, v[j].second);
m_bordersPolygons[v[j].first].m_points[v[j].second].AddLink(v[i].first, v[i].second);
}
}
for (auto & task : tasks)
task.wait();
LOG(LINFO, ("Removed:", m_duplicatedPointsCount, "from input data."));
}
void BordersData::DumpPolyFiles(std::string const & targetDir)
@@ -236,70 +225,11 @@ void BordersData::DumpPolyFiles(std::string const & targetDir)
size_t BordersData::RemoveDuplicatePoints()
{
size_t count = 0;
auto const pointsAreEqual = [](auto const & p1, auto const & p2) {
return base::AlmostEqualAbs(p1.m_point, p2.m_point, kEqualityEpsilon);
};
for (auto & polygon : m_bordersPolygons)
{
auto & points = polygon.m_points;
auto const last = std::unique(points.begin(), points.end(), pointsAreEqual);
count += std::distance(last, points.end());
points.erase(last, points.end());
if (polygon.m_points.begin() == polygon.m_points.end())
continue;
while (points.size() > 1 && pointsAreEqual(points.front(), points.back()))
{
++count;
points.pop_back();
}
}
count += RemoveDuplicatingPointImpl(polygon.m_points);
return count;
}
void BordersData::MarkPoint(size_t curBorderId, size_t curPointId)
{
MarkedPoint & curMarkedPoint = m_bordersPolygons[curBorderId].m_points[curPointId];
for (size_t anotherBorderId = 0; anotherBorderId < m_bordersPolygons.size(); ++anotherBorderId)
{
if (curBorderId == anotherBorderId)
continue;
if (curMarkedPoint.m_marked)
return;
Polygon & anotherPolygon = m_bordersPolygons[anotherBorderId];
if (!anotherPolygon.m_rect.IsPointInside(curMarkedPoint.m_point))
continue;
for (size_t anotherPointId = 0; anotherPointId < anotherPolygon.m_points.size(); ++anotherPointId)
{
auto & anotherMarkedPoint = anotherPolygon.m_points[anotherPointId];
if (base::AlmostEqualAbs(anotherMarkedPoint.m_point, curMarkedPoint.m_point, kEqualityEpsilon))
{
anotherMarkedPoint.m_marked = true;
curMarkedPoint.m_marked = true;
// Save info that border with id: |anotherBorderId| has the same point with id:
// |anotherPointId|.
curMarkedPoint.AddLink(anotherBorderId, anotherPointId);
// And vice versa.
anotherMarkedPoint.AddLink(curBorderId, curPointId);
return;
}
}
}
}
void BordersData::PrintDiff()
{
using Info = std::tuple<double, std::string, size_t, size_t>;

View File

@@ -14,13 +14,10 @@ namespace poly_borders
class BordersData
{
public:
inline static double const kEqualityEpsilon = 1e-20;
inline static double const kEqualityEpsilon = 1.0E-7;
inline static std::string const kBorderExtension = ".poly";
void Init(std::string const & bordersDir);
/// \brief Runs |MarkPoint(borderId, pointId)| for each borderId and its pointId. Look to
/// |MarkPoint| for more details.
void MarkPoints();
void RemoveEmptySpaceBetweenBorders();
@@ -29,23 +26,29 @@ public:
void PrintDiff();
private:
struct Processor
{
explicit Processor(BordersData & data) : m_data(data) {}
void operator()(size_t borderId);
BordersData & m_data;
};
/// \brief Some polygons can have sequentially same points - duplicates. This method removes such
/// points and leaves only unique.
size_t RemoveDuplicatePoints();
/// \brief Finds point on other polygons equal to points passed as in the argument. If such point
/// is found, the method will create a link of the form "some border (with id = anotherBorderId)
/// has the same point (with id = anotherPointId)".
// If point belongs to more than 2 polygons, the link will be created for an arbitrary pair.
void MarkPoint(size_t curBorderId, size_t curPointId);
template <class PointsT> static size_t RemoveDuplicatingPointImpl(PointsT & points)
{
auto const equalFn = [](auto const & p1, auto const & p2)
{
return p1.EqualDxDy(p2, kEqualityEpsilon);
};
auto const last = std::unique(points.begin(), points.end(), equalFn);
size_t count = std::distance(last, points.end());
points.erase(last, points.end());
while (points.size() > 1 && equalFn(points.front(), points.back()))
{
++count;
points.pop_back();
}
return count;
}
/// \brief Checks whether we can replace points from segment: [curLeftPointId, curRightPointId]
/// of |curBorderId| to points from another border in order to get rid of empty space

View File

@@ -21,7 +21,6 @@ bool ReplaceData::operator<(ReplaceData const & rhs) const
// MarkedPoint -------------------------------------------------------------------------------------
void MarkedPoint::AddLink(size_t borderId, size_t pointId)
{
std::lock_guard<std::mutex> lock(*m_mutex);
m_links.emplace(borderId, pointId);
}

View File

@@ -1,40 +1,19 @@
#pragma once
#include "geometry/rect2d.hpp"
#include "geometry/point2d.hpp"
#include "base/assert.hpp"
#include "base/non_intersecting_intervals.hpp"
#include <atomic>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>
namespace poly_borders
{
struct AtomicBoolWrapper
{
AtomicBoolWrapper() { m_value = false; }
AtomicBoolWrapper(bool value) { m_value = value; }
AtomicBoolWrapper(std::atomic<bool> const & rhs) { m_value = rhs.load(); }
AtomicBoolWrapper(AtomicBoolWrapper const & rhs) { m_value = rhs.m_value.load(); }
AtomicBoolWrapper operator=(AtomicBoolWrapper const & rhs)
{
m_value = rhs.m_value.load();
return *this;
}
explicit operator bool() const { return m_value.load(); }
std::atomic<bool> m_value;
};
struct Link
{
@@ -77,23 +56,32 @@ struct ReplaceData
struct MarkedPoint
{
MarkedPoint() = default;
explicit MarkedPoint(m2::PointD const & point) : m_point(point) {}
MarkedPoint(m2::PointD const & point) : m_point(point) {}
void AddLink(size_t borderId, size_t pointId);
std::optional<Link> GetLink(size_t curBorderId) const;
bool EqualDxDy(MarkedPoint const & p, double eps) const
{
return m_point.EqualDxDy(p.m_point, eps);
}
m2::PointD m_point;
AtomicBoolWrapper m_marked;
std::set<Link> m_links;
std::unique_ptr<std::mutex> m_mutex = std::make_unique<std::mutex>();
};
struct Polygon
{
Polygon() = default;
Polygon(m2::RectD const & rect, std::vector<m2::PointD> const & points) : m_rect(rect)
{
m_points.assign(points.begin(), points.end());
}
Polygon(m2::RectD const & rect, std::vector<MarkedPoint> && points)
: m_rect(rect), m_points(std::move(points)) {}
: m_rect(rect), m_points(std::move(points))
{
}
Polygon(Polygon &&) = default;
Polygon & operator=(Polygon &&) noexcept = default;

View File

@@ -27,12 +27,12 @@ static string const kTestDir = "borders_poly_dir";
void TestMarked(Polygon const & polygon, size_t i)
{
TEST(polygon.m_points[i].m_marked, (i, "th point point must be marked."));
TEST(!polygon.m_points[i].m_links.empty(), (i, "th point point must be marked."));
}
void TestNotMarked(Polygon const & polygon, size_t i)
{
TEST(!polygon.m_points[i].m_marked, (i, "th point must not be marked."));
TEST(polygon.m_points[i].m_links.empty(), (i, "th point must not be marked."));
}
void CheckByMask(Polygon const & polygons, vector<bool> markedMask)
@@ -77,7 +77,6 @@ UNIT_TEST(PolyBordersPostprocessor_MarkPoints_1)
BordersData bordersData;
bordersData.Init(bordersDir);
bordersData.MarkPoints();
auto const & bordersPolygon1 = bordersData.GetBordersPolygonByName("First" + BordersData::kBorderExtension + "1");
CheckByMask(bordersPolygon1, markedMask1[0]);
@@ -113,7 +112,6 @@ UNIT_TEST(PolyBordersPostprocessor_MarkPoints_2)
BordersData bordersData;
bordersData.Init(bordersDir);
bordersData.MarkPoints();
auto const & bordersPolygon1 = bordersData.GetBordersPolygonByName("First" + BordersData::kBorderExtension + "1");
CheckByMask(bordersPolygon1, markedMask1[0]);
@@ -174,7 +172,6 @@ UNIT_TEST(PolyBordersPostprocessor_MarkPoints_3)
BordersData bordersData;
bordersData.Init(bordersDir);
bordersData.MarkPoints();
auto const & bordersPolygon1 = bordersData.GetBordersPolygonByName("First" + BordersData::kBorderExtension + "1");
CheckByMask(bordersPolygon1, markedMask1[0]);
@@ -221,7 +218,6 @@ UNIT_TEST(PolyBordersPostprocessor_MarkPoints_4)
BordersData bordersData;
bordersData.Init(bordersDir);
bordersData.MarkPoints();
auto const & firstBordersPolygon1 = bordersData.GetBordersPolygonByName("First" + BordersData::kBorderExtension + "1");
CheckByMask(firstBordersPolygon1, markedMask1[0]);

View File

@@ -27,7 +27,6 @@ auto constexpr kSmallPointShift = m2::PointD(kSmallShift, kSmallShift);
void Process(BordersData & bordersData, string const & bordersDir)
{
bordersData.Init(bordersDir);
bordersData.MarkPoints();
bordersData.RemoveEmptySpaceBetweenBorders();
}

View File

@@ -42,9 +42,7 @@ int main(int argc, char ** argv)
{
BordersData data;
data.Init(FLAGS_borders_path);
data.MarkPoints();
data.RemoveEmptySpaceBetweenBorders();
data.MarkPoints();
data.PrintDiff();
data.DumpPolyFiles(FLAGS_output_path);
}