Add files to automatically generate maps with Docker

Signed-off-by: zyphlar <zyphlar@gmail.com>
2026-01-04 11:53:47 +00:00 · 2025-05-01 22:47:34 -07:00
parent ca5cc46f2b
commit cf4e400052
9 changed files with 348 additions and 12 deletions
--- a/tools/unix/Dockerfile
+++ b/tools/unix/Dockerfile
@@ -0,0 +1,54 @@
+# See run-docker.sh for cloning, building, and running the maps generator Docker routine
+FROM ubuntu:noble
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC 
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+    curl \
+    osmctools \
+    s3cmd \
+    sshpass \
+    vim \
+    wget \
+    build-essential \
+    clang \
+    cmake \
+    ninja-build \
+    python3 \
+    python3-pip \
+    python3.12-venv \
+    qt6-base-dev \
+    qt6-positioning-dev \
+    libc++-dev \
+    libfreetype-dev \
+    libglvnd-dev \
+    libgl1-mesa-dev \
+    libharfbuzz-dev \
+    libicu-dev \
+    libqt6svg6-dev \
+    libqt6positioning6-plugins \
+    libqt6positioning6 \
+    libsqlite3-dev \
+    libxrandr-dev \
+    libxinerama-dev \
+    libxcursor-dev \
+    libxi-dev \
+    zlib1g-dev \
+    && rm -rf /var/cache/apt/* /var/lib/apt/lists/*;
+
+RUN mkdir /root/OM
+# When running, bind mount the repo here
+RUN mkdir /root/OM/organicmaps
+# And a volume to store the large output/temp files here
+RUN mkdir /root/OM/maps_build
+# And a volume to store >10gb files for the planet here
+RUN mkdir /home/planet
+
+WORKDIR /root/OM/organicmaps
+
+# For debugging
+#CMD /bin/bash 
+
+CMD /root/OM/organicmaps/tools/unix/docker_maps_generator.sh
--- a/tools/unix/docker_maps_generator.sh
+++ b/tools/unix/docker_maps_generator.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+#Volumes/paths for downloads:
+#home/planet/planet/planet.o5m
+#home/planet/planet/planet.o5m.md5
+#PLANET_COASTS_URL:file:///home/planet/planet/
+  #home/planet/planet/latest_coasts.geom and latest_coasts.rawgeom
+#SUBWAY_URL: file:///home/planet/subway/beta.json
+  #home/planet/subway/beta.json
+#HOTELS_URL:/home/planet/planet/kayak/
+  #home/planet/planet/kayak/
+#SRTM_PATH:/home/planet/SRTM-patched-europe/
+#ISOLINES_PATH:/home/planet/planet/isolines/
+#ADDRESSES_PATH:/home/planet/planet/tiger/
+#UK_POSTCODES_URL:/home/planet/postcodes/gb-postcode-data/gb_postcodes.csv
+#US_POSTCODES_URL:/home/planet/postcodes/us-postcodes/uszips.csv
+
+echo "<$(date +%T)> Starting..."
+
+# Prepare paths
+#
+# Already created by Dockerfile:
+#   /root/OM
+#   /root/OM/organicmaps
+#   /root/OM/maps_build
+#   /home/planet
+#
+mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these
+chmod -R 777 /root/.config
+mkdir -p ~/OM/maps_build
+mkdir -p ~/OM/omim-build-release
+mkdir -p ~/OM/osmctools
+mkdir -p /home/planet/planet/isolines/
+mkdir -p /home/planet/planet/kayak/
+mkdir -p /home/planet/planet/tiger/
+mkdir -p /home/planet/postcodes/gb-postcode-data/
+mkdir -p /home/planet/postcodes/us-postcodes/
+mkdir -p /home/planet/SRTM-patched-europe/
+mkdir -p /home/planet/subway
+
+echo "Writing S3 config..."
+echo "[default]" > ~/.s3cfg
+echo "access_key = $S3_KEY_ID" >> ~/.s3cfg
+echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg
+echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg
+echo "host_bucket = \$(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg
+# S3_BUCKET is used during upload
+
+echo "Wrote:"
+cat ~/.s3cfg
+
+echo "<$(date +%T)> Running ./configure.sh ..."
+cd ~/OM/organicmaps
+./configure.sh
+
+echo "<$(date +%T)> Compiling tools..."
+cd ~/OM/organicmaps
+./tools/unix/build_omim.sh -R generator_tool
+./tools/unix/build_omim.sh -R world_roads_builder_tool
+./tools/unix/build_omim.sh -R mwm_diff_tool
+cd tools/python/maps_generator
+python3 -m venv /tmp/venv
+/tmp/venv/bin/pip3 install -r requirements_dev.txt
+
+echo "<$(date +%T)> Copying map generator INI..."
+cp var/etc/map_generator.ini.prod var/etc/map_generator.ini
+
+#TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources"
+#echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..."
+#cd ~/OM/organicmaps/tools/osmctools
+#gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate
+#gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert
+
+# May be unnecessary when running world
+# /tmp/venv/bin/python -m maps_generator --coasts
+# save to /path/to/coasts WorldCoasts.geom as latest_coasts.geom and WorldCoasts.rawgeom latest_coasts.rawgeom
+# (from https://github.com/mapsme/omim/issues/11994)
+
+cd /home/planet/planet
+if [ ! -f planet-latest.osm.pbf ]; then
+  echo "<$(date +%T)> Downloading planet-latest.osm.pbf..."
+  curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf
+  echo "<$(date +%T)> Downloading planet-latest.osm.pbf.md5..."
+  curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf.md5
+else
+  echo "<$(date +%T)> planet-latest.osm.pbf exists, not downloading..."
+fi
+
+#curl -OL https://download.geofabrik.de/north-america/us-west-latest.osm.pbf
+#curl -OL https://download.geofabrik.de/north-america/us-west-latest.osm.pbf.md5
+# (rename us-west-latest to planet-latest and edit the md5 file accordingly)
+if [ ! -f planet.o5m ]; then
+  echo "<$(date +%T)> Converting planet-latest.osm.pbf to planet.o5m..."
+  ~/OM/osmctools/osmconvert planet-latest.osm.pbf -o=planet.o5m
+else
+  echo "<$(date +%T)> planet.o5m exists, not converting..."
+fi
+# (currently unused:) ~/OM/organicmaps/tools/unix/update_planet.sh planet.o5m
+
+
+echo "<$(date +%T)> Generating maps..."
+cd ~/OM/organicmaps/tools/python
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs"
+#/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" --continue
+
+# do not use --production except for Kayak/recommendation/popularity/food data
+#/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production
+#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs"
+#/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs"
+
+shopt -s nullglob
+mwmfiles=( ~/OM/maps_build/*/*/*.mwm )
+
+if (( ${#mwmfiles[@]} )); then
+  echo "<$(date +%T)> Uploading maps..."
+  # TODO: upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app)
+  # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt
+  #sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH <<EOF
+  #put ~/OM/maps_build/20*/2*/countries.txt
+  #put ~/OM/maps_build/20*/2*/World.mwm
+  #put ~/OM/maps_build/20*/2*/WorldCoasts.mwm
+  #exit
+  #EOF
+  
+  # TODO: upload all files via rclone to Cloudflare (R2)
+  #s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/"
+  #s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive
+  #s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive
+else
+  echo "<$(date +%T)> No MWM files, not uploading maps."
+fi
+
+echo "<$(date +%T)> Temporarily NOT Removing intermediate data..."
+#rm -rf ~/OM/maps_build/*/intermediate_data
+# rm -rf ~/OM/
+
+echo "<$(date +%T)> DONE"
+
--- a/tools/unix/maps/generate_subways.sh
+++ b/tools/unix/maps/generate_subways.sh
@@ -20,9 +20,9 @@ export SKIP_PLANET_UPDATE="1"
 # If unavailable then replace with a local file.
 # TODO: keep the downloaded csv file from the latest run.
 #export CITIES_INFO_URL=""
-export TMPDIR="$BUILD_PATH/subways"
+export TMPDIR="$BUILD_PATH/subway"
 # The output file, which needs post-processing by transit_graph_generator.py
-export MAPSME="$SUBWAYS_PATH/subways.json"
+export MAPSME="$SUBWAYS_PATH/subway.json"

 # Produce additional files needed for https://cdn.organicmaps.app/subway/
 export HTML_DIR="$SUBWAYS_VALIDATOR_PATH"
@@ -36,7 +36,7 @@ export DUMP_CITY_LIST="$SUBWAYS_VALIDATOR_PATH/cities.txt"
 cp -r "$SUBWAYS_REPO_PATH"/render/* "$SUBWAYS_VALIDATOR_PATH/"

 TRANSIT_TOOL_PATH="$REPO_PATH/tools/python/transit"
-SUBWAYS_GRAPH_FILE="$SUBWAYS_PATH/subways.transit.json"
+SUBWAYS_GRAPH_FILE="$SUBWAYS_PATH/subway.transit.json"

 activate_venv_at_path "$TRANSIT_TOOL_PATH"
 "$PYTHON" "$TRANSIT_TOOL_PATH/transit_graph_generator.py" "$MAPSME" "$SUBWAYS_GRAPH_FILE" 2>&1 | tee -a "$SUBWAYS_LOG"
--- a/tools/unix/maps/settings_default.sh
+++ b/tools/unix/maps/settings_default.sh
@@ -20,6 +20,6 @@ PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
 # Subways

 SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-$CODE_PATH/subways}"
-SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subways}"
-SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subways.log}"
+SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}"
+SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}"
 SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"
--- a/tools/unix/update_planet.sh
+++ b/tools/unix/update_planet.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+OSMUPDATE=~/osmctools/osmupdate
+# osmconvert should be accessible in PATH.
+PATH="$(dirname "$OSMUPDATE"):$PATH"
+
+# Pass pbf or o5m file as a parameter
+OLD="$1"
+NEW="${1/.pbf/.new.pbf}"
+NEW="${NEW/.o5m/.new.o5m}"
+
+"$OSMUPDATE" -v --drop-authors --drop-version --hash-memory=512000 "$OLD" "$NEW"
+# Uncomment to replace old planet.
+mv "$NEW" "$OLD"
+#md5sum -b "$OLD" > "$OLD.md5"
+echo "Successfully updated $OLD"