From 2c9a85d5a38ed91745b3dc02ea307fccacf6cf19 Mon Sep 17 00:00:00 2001 From: zyphlar Date: Mon, 12 May 2025 06:37:44 +0000 Subject: [PATCH] Map generation mostly working, but slow and not doing full planet with add-ins yet Signed-off-by: zyphlar --- .../var/etc/map_generator.ini.prod | 23 +++++++++++-------- tools/unix/Dockerfile | 21 ++++++++++++++--- tools/unix/docker_maps_generator.sh | 10 ++++---- 3 files changed, 38 insertions(+), 16 deletions(-) mode change 100644 => 100755 tools/unix/Dockerfile diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.prod b/tools/python/maps_generator/var/etc/map_generator.ini.prod index c4e7cb44d..23422d673 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.prod +++ b/tools/python/maps_generator/var/etc/map_generator.ini.prod @@ -2,7 +2,8 @@ # Path to the `organicmaps` source code repository: OMIM_PATH: ~/OM/organicmaps # A path with the generator_tool binary: -BUILD_PATH: ${Developer:OMIM_PATH}/../omim-build-release +BUILD_PATH: /root/OM/omim-build-relwithdebinfo +#${Developer:OMIM_PATH}/../omim-build-release [Main] @@ -21,16 +22,18 @@ MAIN_OUT_PATH: ${Developer:OMIM_PATH}/../maps_build USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data # Features stage only parallelism level. Set to 0 for auto detection. THREADS_COUNT_FEATURES_STAGE: 0 +# Do not change it. This is determined automatically. +NODE_STORAGE: mem [Osm tools] # Path to osmctools binaries: -OSM_TOOLS_PATH: ${Developer:OMIM_PATH}/../osmctools +OSM_TOOLS_PATH: /usr/bin/ +#${Developer:OMIM_PATH}/../osmctools # If the binaries are not found neither in the configured path nor system-wide, # then the tools are built from the sources: OSM_TOOLS_SRC_PATH: ${Developer:OMIM_PATH}/tools/osmctools - [Logging] # maps_generator's general (python output only) log file path and name. # More detailed logs that include output of the `generator_tool` binary @@ -62,7 +65,8 @@ PLANET_COASTS_URL: file:///home/planet/ NEED_BUILD_WORLD_ROADS: true # Subway file location, see docs/SUBWAY_GENERATION.md if you want to generate your own file. # Comment to disable subway layer generation. -SUBWAY_URL: file:///home/planet/subway/beta.json + +#SUBWAY_URL: file:///home/planet/subway/beta.json # Location of the EXPERIMENTAL GTFS-extracted public transport transit files: # TRANSIT_URL: @@ -74,13 +78,14 @@ SUBWAY_URL: file:///home/planet/subway/beta.json # POPULARITY_URL: # FOOD_URL: # FOOD_TRANSLATIONS_URL: -SRTM_PATH: /home/planet/SRTM-patched-europe/ -ISOLINES_PATH: /home/planet/planet/isolines/ -ADDRESSES_PATH: /home/planet/planet/tiger/ + +#SRTM_PATH: /home/planet/SRTM-patched-europe/ +#ISOLINES_PATH: /home/planet/planet/isolines/ +#ADDRESSES_PATH: /home/planet/planet/tiger/ # Local path (not url!) to .csv files. -UK_POSTCODES_URL: /home/planet/postcodes/gb-postcode-data/gb_postcodes.csv -US_POSTCODES_URL: /home/planet/postcodes/us-postcodes/uszips.csv +#UK_POSTCODES_URL: /home/planet/postcodes/gb-postcode-data/gb_postcodes.csv +#US_POSTCODES_URL: /home/planet/postcodes/us-postcodes/uszips.csv [Stages] diff --git a/tools/unix/Dockerfile b/tools/unix/Dockerfile old mode 100644 new mode 100755 index 83f1c35a8..d19ec8b92 --- a/tools/unix/Dockerfile +++ b/tools/unix/Dockerfile @@ -1,10 +1,24 @@ -# Check out this whole repo with submodules and shallow +# Check out this whole repo with submodules and shallow: + +# cd /path/to/4tb +# git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps.git +# cd /path/to/4tb/comaps +# git checkout docker_maps_generator + +#TODO: shaders_complier ? + +# Get static data: +# cd /path/to/4tb/comaps/data/ +# wget World.mwm +# wget WorldCoasts.mwm +#TODO: isolines, postcodes, subways, wiki + # Build with: docker build . -t maps_generator # Edit as appropriate and run with: # docker run \ # -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \ -# -v /path/to/organicmaps:/root/OM/organicmaps -v /tmp/osm-planet:/home/planet \ -# -v /tmp/osm-maps:/root/OM/maps_build -it maps_generator +# -v /path/to/comaps:/root/OM/organicmaps -v /path/to/4tb/osm-planet:/home/planet \ +# -v /path/to/4tb/osm-maps:/root/OM/maps_build -it maps_generator FROM ubuntu:noble @@ -15,6 +29,7 @@ RUN apt-get update -qq \ && apt-get install -y --no-install-recommends \ curl \ osmctools \ + s3cmd \ sshpass \ vim \ wget \ diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh index 7bfe26500..095da9546 100755 --- a/tools/unix/docker_maps_generator.sh +++ b/tools/unix/docker_maps_generator.sh @@ -97,18 +97,20 @@ else fi # (currently unused:) ~/OM/organicmaps/tools/unix/update_planet.sh planet.o5m + echo "<$(date +%T)> Generating maps..." cd ~/OM/organicmaps/tools/python -#/tmp/venv/bin/python -m maps_generator # do not use --production except for Kayak/recommendation/popularity/food data +/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" +# do not use --production except for Kayak/recommendation/popularity/food data #/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production -/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs" +#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs" +#/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs" shopt -s nullglob mwmfiles=( ~/OM/maps_build/*/*/*.mwm ) if (( ${#mwmfiles[@]} )); then echo "<$(date +%T)> Uploading maps..." - # maps.zyphon.com:www < No MWM files, not uploading maps." fi -echo "<$(date +%T)> NOT Removing intermediate data..." +echo "<$(date +%T)> Temporarily NOT Removing intermediate data..." #rm -rf ~/OM/maps_build/*/intermediate_data # rm -rf ~/OM/