Map generation mostly working, but slow and not doing full planet with add-ins yet

Signed-off-by: zyphlar <zyphlar@gmail.com>

tools/python/maps_generator/var/etc/map_generator.ini.prod

@@ -2,7 +2,8 @@
 # Path to the `organicmaps` source code repository:
 OMIM_PATH: ~/OM/organicmaps
 # A path with the generator_tool binary:
-BUILD_PATH: ${Developer:OMIM_PATH}/../omim-build-release
+BUILD_PATH: /root/OM/omim-build-relwithdebinfo
+#${Developer:OMIM_PATH}/../omim-build-release
 
 
 [Main]
@@ -21,16 +22,18 @@ MAIN_OUT_PATH: ${Developer:OMIM_PATH}/../maps_build
 USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data
 # Features stage only parallelism level. Set to 0 for auto detection.
 THREADS_COUNT_FEATURES_STAGE: 0
+# Do not change it. This is determined automatically.
+NODE_STORAGE: mem
 
 
 [Osm tools]
 # Path to osmctools binaries:
-OSM_TOOLS_PATH: ${Developer:OMIM_PATH}/../osmctools
+OSM_TOOLS_PATH: /usr/bin/
+#${Developer:OMIM_PATH}/../osmctools
 # If the binaries are not found neither in the configured path nor system-wide,
 # then the tools are built from the sources:
 OSM_TOOLS_SRC_PATH: ${Developer:OMIM_PATH}/tools/osmctools
 
-
 [Logging]
 # maps_generator's general (python output only) log file path and name.
 # More detailed logs that include output of the `generator_tool` binary
@@ -62,7 +65,8 @@ PLANET_COASTS_URL: file:///home/planet/
 NEED_BUILD_WORLD_ROADS: true
 # Subway file location, see docs/SUBWAY_GENERATION.md if you want to generate your own file.
 # Comment to disable subway layer generation.
-SUBWAY_URL: file:///home/planet/subway/beta.json
+
+#SUBWAY_URL: file:///home/planet/subway/beta.json
 
 # Location of the EXPERIMENTAL GTFS-extracted public transport transit files:
 # TRANSIT_URL:
@@ -74,13 +78,14 @@ SUBWAY_URL: file:///home/planet/subway/beta.json
 # POPULARITY_URL:
 # FOOD_URL:
 # FOOD_TRANSLATIONS_URL:
-SRTM_PATH: /home/planet/SRTM-patched-europe/
+
-ISOLINES_PATH: /home/planet/planet/isolines/
+#SRTM_PATH: /home/planet/SRTM-patched-europe/
-ADDRESSES_PATH: /home/planet/planet/tiger/
+#ISOLINES_PATH: /home/planet/planet/isolines/
+#ADDRESSES_PATH: /home/planet/planet/tiger/
 
 # Local path (not url!) to .csv files.
-UK_POSTCODES_URL: /home/planet/postcodes/gb-postcode-data/gb_postcodes.csv
+#UK_POSTCODES_URL: /home/planet/postcodes/gb-postcode-data/gb_postcodes.csv
-US_POSTCODES_URL: /home/planet/postcodes/us-postcodes/uszips.csv
+#US_POSTCODES_URL: /home/planet/postcodes/us-postcodes/uszips.csv
 
 
 [Stages]

tools/unix/Dockerfile

@@ -1,10 +1,24 @@
-# Check out this whole repo with submodules and shallow
+# Check out this whole repo with submodules and shallow:
+
+# cd /path/to/4tb
+# git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps.git
+# cd /path/to/4tb/comaps
+# git checkout docker_maps_generator
+
+#TODO: shaders_complier ?
+
+# Get static data:
+# cd /path/to/4tb/comaps/data/
+# wget World.mwm
+# wget WorldCoasts.mwm
+#TODO: isolines, postcodes, subways, wiki
+
 # Build with: docker build . -t maps_generator
 # Edit as appropriate and run with:
 #  docker run \
 #  -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \
-#  -v /path/to/organicmaps:/root/OM/organicmaps -v /tmp/osm-planet:/home/planet \
+#  -v /path/to/comaps:/root/OM/organicmaps -v /path/to/4tb/osm-planet:/home/planet \
-#  -v /tmp/osm-maps:/root/OM/maps_build -it maps_generator
+#  -v /path/to/4tb/osm-maps:/root/OM/maps_build -it maps_generator
 
 FROM ubuntu:noble
 
@@ -15,6 +29,7 @@ RUN apt-get update -qq \
     && apt-get install -y --no-install-recommends \
     curl \
     osmctools \
+    s3cmd \
     sshpass \
     vim \
     wget \

tools/unix/docker_maps_generator.sh

@@ -97,18 +97,20 @@ else
 fi
 # (currently unused:) ~/OM/organicmaps/tools/unix/update_planet.sh planet.o5m
 
+
 echo "<$(date +%T)> Generating maps..."
 cd ~/OM/organicmaps/tools/python
-#/tmp/venv/bin/python -m maps_generator # do not use --production except for Kayak/recommendation/popularity/food data
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs"
+# do not use --production except for Kayak/recommendation/popularity/food data
 #/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production
-/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs"
+#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs"
+#/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs"
 
 shopt -s nullglob
 mwmfiles=( ~/OM/maps_build/*/*/*.mwm )
 
 if (( ${#mwmfiles[@]} )); then
   echo "<$(date +%T)> Uploading maps..."
-  # maps.zyphon.com:www <<EOF
   # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt
   #sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH <<EOF
   #put ~/OM/maps_build/generation.log
@@ -124,7 +126,7 @@ else
   echo "<$(date +%T)> No MWM files, not uploading maps."
 fi
 
-echo "<$(date +%T)> NOT Removing intermediate data..."
+echo "<$(date +%T)> Temporarily NOT Removing intermediate data..."
 #rm -rf ~/OM/maps_build/*/intermediate_data
 # rm -rf ~/OM/