diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp index c8b600bc0..8e3032117 100644 --- a/generator/intermediate_data.cpp +++ b/generator/intermediate_data.cpp @@ -114,8 +114,10 @@ public: uint64_t const fileSize = m_fileReader.Size(); CHECK_EQUAL(fileSize % sizeof(LatLon), 0, ("Node's coordinates file is broken")); + LOG(LINFO, ("Start reading nodes storage from", name)); m_data.resize(fileSize / sizeof(LatLon)); m_fileReader.Read(0, m_data.data(), fileSize); + LOG(LINFO, ("Finished reading nodes storage")); } // PointStorageReaderInterface overrides: @@ -404,9 +406,9 @@ IntermediateDataObjectsCache::GetOrCreatePointStorageReader( std::lock_guard lock(m_mutex); + LOG(LINFO, ("Creating nodes storage reader:", strType, name)); auto res = m_objects.try_emplace(key, type, name); - if (res.second) - LOG(LINFO, ("Created nodes reader:", strType, name)); + return res.first->second; } diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index 29426a3ee..32da41033 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -162,6 +162,7 @@ bool RawGenerator::Execute() m_queue.reset(); m_intermediateDataObjectsCache.Clear(); + LOG(LINFO, ("Start final processing...")); while (!m_finalProcessors.empty()) { auto const finalProcessor = m_finalProcessors.top(); @@ -244,7 +245,13 @@ bool RawGenerator::GenerateFilteredFeatures() } CHECK(sourceProcessor, ()); + // Create translators threads. + // Each thread may contain separate translators for countries and World + // They process chunks of source data and pass features to a chain of processors. + // The last processor writes to a "processed" queue. TranslatorsPool translators(m_translators, m_threadsCount); + + // The writer thread pops from the "processed" queue and writes to per-country files. RawGeneratorWriter rawGeneratorWriter(m_queue, m_genInfo.m_tmpDir); rawGeneratorWriter.Run(); @@ -267,7 +274,7 @@ bool RawGenerator::GenerateFilteredFeatures() } while (!isEnd); - LOG(LINFO, ("Input was processed.")); + LOG(LINFO, ("OSM source input was processed.")); if (!translators.Finish()) return false; diff --git a/generator/translator.cpp b/generator/translator.cpp index 6b36d088a..662b27f9b 100644 --- a/generator/translator.cpp +++ b/generator/translator.cpp @@ -63,7 +63,9 @@ void Translator::Emit(OsmElement const & src) void Translator::Finish() { + LOG(LINFO, ("Finishing collectors...")); m_collector->Finish(); + LOG(LINFO, ("Finishing processors...")); m_processor->Finish(); } diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index d5ecf443b..01ba96f67 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -92,7 +92,7 @@ OSM_TOOLS_PATH = os.path.join(_WORK_PATH, "osmctools") # Generator tool section: USER_RESOURCE_PATH = os.path.join(OMIM_PATH, "data") -NODE_STORAGE = "mem" if total_virtual_memory() / 10 ** 9 >= 64 else "map" +NODE_STORAGE = "map" # Stages section: NEED_PLANET_UPDATE = False diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index 3af13c82d..230171874 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -21,6 +21,11 @@ MAIN_OUT_PATH: ${Developer:OMIM_PATH}/../maps_build USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data # Features stage only parallelism level. Set to 0 for auto detection. THREADS_COUNT_FEATURES_STAGE: 0 +# How to store all nodes with their coords. +# "map" (default) - fast, suitable to generate a few countries, but is not suitable for the whole planet +# "mem" - fastest, best for the whole planet generation, needs ~100GB memory (as of 2025) +# "raw" - read from a mmapped file, slow, but uses the least memory +NODE_STORAGE: mem [Osm tools]