drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2025-12-19 13:03:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

[generator] Improve logging, add comments

Browse Source 
Signed-off-by: Konstantin Pastbin <konstantin.pastbin@gmail.com>
This commit is contained in:
Konstantin Pastbin
2025-08-07 18:32:25 +07:00
committed by Konstantin Pastbin
parent d7f73f4b9b
commit 5de22c6de8
5 changed files with 20 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
generator/intermediate_data.cpp
View File

@@ -114,8 +114,10 @@ public:
uint64_t const fileSize = m_fileReader.Size(); uint64_t const fileSize = m_fileReader.Size();
CHECK_EQUAL(fileSize % sizeof(LatLon), 0, ("Node's coordinates file is broken")); CHECK_EQUAL(fileSize % sizeof(LatLon), 0, ("Node's coordinates file is broken"));
LOG(LINFO, ("Start reading nodes storage from", name));
m_data.resize(fileSize / sizeof(LatLon)); m_data.resize(fileSize / sizeof(LatLon));
m_fileReader.Read(0, m_data.data(), fileSize); m_fileReader.Read(0, m_data.data(), fileSize);
LOG(LINFO, ("Finished reading nodes storage"));
} }
// PointStorageReaderInterface overrides: // PointStorageReaderInterface overrides:
@@ -404,9 +406,9 @@ IntermediateDataObjectsCache::GetOrCreatePointStorageReader(
std::lock_guard lock(m_mutex); std::lock_guard lock(m_mutex);
LOG(LINFO, ("Creating nodes storage reader:", strType, name));
auto res = m_objects.try_emplace(key, type, name); auto res = m_objects.try_emplace(key, type, name);
if (res.second)
LOG(LINFO, ("Created nodes reader:", strType, name));
return res.first->second; return res.first->second;
} }

9
generator/raw_generator.cpp
View File

@@ -162,6 +162,7 @@ bool RawGenerator::Execute()
m_queue.reset(); m_queue.reset();
m_intermediateDataObjectsCache.Clear(); m_intermediateDataObjectsCache.Clear();
LOG(LINFO, ("Start final processing..."));
while (!m_finalProcessors.empty()) while (!m_finalProcessors.empty())
{ {
auto const finalProcessor = m_finalProcessors.top(); auto const finalProcessor = m_finalProcessors.top();
@@ -244,7 +245,13 @@ bool RawGenerator::GenerateFilteredFeatures()
} }
CHECK(sourceProcessor, ()); CHECK(sourceProcessor, ());
// Create translators threads.
// Each thread may contain separate translators for countries and World
// They process chunks of source data and pass features to a chain of processors.
// The last processor writes to a "processed" queue.
TranslatorsPool translators(m_translators, m_threadsCount); TranslatorsPool translators(m_translators, m_threadsCount);
// The writer thread pops from the "processed" queue and writes to per-country files.
RawGeneratorWriter rawGeneratorWriter(m_queue, m_genInfo.m_tmpDir); RawGeneratorWriter rawGeneratorWriter(m_queue, m_genInfo.m_tmpDir);
rawGeneratorWriter.Run(); rawGeneratorWriter.Run();
@@ -267,7 +274,7 @@ bool RawGenerator::GenerateFilteredFeatures()
} while (!isEnd); } while (!isEnd);
LOG(LINFO, ("Input was processed.")); LOG(LINFO, ("OSM source input was processed."));
if (!translators.Finish()) if (!translators.Finish())
return false; return false;

2
generator/translator.cpp
View File

@@ -63,7 +63,9 @@ void Translator::Emit(OsmElement const & src)
void Translator::Finish() void Translator::Finish()
{ {
LOG(LINFO, ("Finishing collectors..."));
m_collector->Finish(); m_collector->Finish();
LOG(LINFO, ("Finishing processors..."));
m_processor->Finish(); m_processor->Finish();
} }

2
tools/python/maps_generator/generator/settings.py
View File

@@ -92,7 +92,7 @@ OSM_TOOLS_PATH = os.path.join(_WORK_PATH, "osmctools")
# Generator tool section: # Generator tool section:
USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data") USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data")
NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map" NODE_STORAGE = "map"
# Stages section: # Stages section:
NEED_PLANET_UPDATE = False NEED_PLANET_UPDATE = False

5
tools/python/maps_generator/var/etc/map_generator.ini.default
View File

@@ -21,6 +21,11 @@ MAIN_OUT_PATH: ${Developer:OMIM_PATH}/../maps_build
USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data
# Features stage only parallelism level. Set to 0 for auto detection. # Features stage only parallelism level. Set to 0 for auto detection.
THREADS_COUNT_FEATURES_STAGE: 0 THREADS_COUNT_FEATURES_STAGE: 0
# How to store all nodes with their coords.
# "map" (default) - fast, suitable to generate a few countries, but is not suitable for the whole planet
# "mem" - fastest, best for the whole planet generation, needs ~100GB memory (as of 2025)
# "raw" - read from a mmapped file, slow, but uses the least memory
NODE_STORAGE: mem
[Osm tools] [Osm tools]