[generator] Automate maps generation with Docker and CI/CD

Co-authored-by: Konstantin Pastbin <konstantin.pastbin@gmail.com>
Signed-off-by: zyphlar <zyphlar@gmail.com>

tools/unix/maps/Dockerfile

@@ -0,0 +1,68 @@
+# See run-docker.sh for cloning, building, and running the maps generator Docker routine
+FROM debian:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cargo \
+    clang \
+    cmake \
+    curl \
+    git \
+    jq \
+    libc++-dev \
+    libfreetype-dev \
+    libgeos-dev \
+    libgl1-mesa-dev \
+    libglvnd-dev \
+    libharfbuzz-dev \
+    libicu-dev \
+    libqt6positioning6 \
+    libqt6positioning6-plugins \
+    libqt6svg6-dev \
+    libsqlite3-dev \
+    libxcursor-dev \
+    libxi-dev \
+    libxinerama-dev \
+    libxrandr-dev \
+    ninja-build \
+    nodejs \
+    openssh-client \
+    osmctools \
+    osmium-tool \
+    pyosmium \
+    python3 \
+    python3-pip \
+    python3-venv \
+    python-is-python3 \
+    python3-dev \
+    qt6-base-dev \
+    qt6-positioning-dev \
+    rclone \
+    rustc \
+    sshpass \
+    vim-tiny \
+    wget \
+    wget2 \
+    zlib1g-dev \
+    && rm -rf /var/cache/apt/* /var/lib/apt/lists/*;
+
+# Allow pip to install system-wide in the container
+RUN pip3 install "protobuf<4" --break-system-packages
+
+# Mount a volume to store the large input, output, and temp files here
+RUN mkdir /mnt/4tbexternal
+# And a volume to store >10gb files for the planet output here
+RUN mkdir /home/planet
+
+WORKDIR ~
+
+# The actual map generation process will run something like this,
+# but ~/comaps isn't mounted nor provided in the docker image.
+#CMD ~/comaps/tools/unix/maps/docker_maps_generator.sh
+
+CMD /bin/bash

tools/unix/maps/docker_maps_generator.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "<$(date +%T)> Starting..."
+
+# Prepare paths
+# Most other paths in /mnt/4tbexternal or /home/planet are already created by Dockerfile or CI/CD.
+#
+mkdir -p /root/.config/CoMaps # Odd mkdir permission errors in generator_tool in Docker without these
+chmod -R 777 /root/.config
+mkdir -p /home/planet/postcodes/gb-postcode-data/
+mkdir -p /home/planet/postcodes/us-postcodes/
+mkdir -p /home/planet/SRTM-patched-europe/
+mkdir -p /home/planet/subway
+
+echo "<$(date +%T)> Running ./configure.sh ..."
+cd ~/comaps
+export SKIP_MAP_DOWNLOAD=1 SKIP_GENERATE_SYMBOLS=1
+./configure.sh
+
+echo "<$(date +%T)> Compiling tools..."
+cd ~/comaps
+./tools/unix/build_omim.sh -p ~ -R generator_tool
+./tools/unix/build_omim.sh -p ~ -R world_roads_builder_tool
+./tools/unix/build_omim.sh -p ~ -R mwm_diff_tool
+cd tools/python/maps_generator
+python3 -m venv /tmp/venv
+/tmp/venv/bin/pip3 install -r requirements_dev.txt
+
+echo "<$(date +%T)> Copying map generator INI..."
+cp var/etc/map_generator.ini.prod var/etc/map_generator.ini
+
+
+cd ~/comaps/tools/python
+if [ $MWMCONTINUE -gt 0 ]; then
+
+echo "<$(date +%T)> Continuing from preexisting generator run..."
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" --continue
+
+else
+
+if [[ -n $MWMCOUNTRIES ]]; then
+
+echo "<$(date +%T)> Generating only specific maps [$MWMCOUNTRIES]..."
+/tmp/venv/bin/python -m maps_generator --countries=$MWMCOUNTRIES --skip="MwmDiffs"
+
+else
+
+echo "<$(date +%T)> Generating maps..."
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs"
+
+fi
+fi
+
+echo "<$(date +%T)> DONE"

tools/unix/maps/generate_subways.sh

@@ -20,9 +20,9 @@ export SKIP_PLANET_UPDATE="1"
 # If unavailable then replace with a local file.
 # TODO: keep the downloaded csv file from the latest run.
 #export CITIES_INFO_URL=""
-export TMPDIR="$BUILD_PATH/subways"
+export TMPDIR="$BUILD_PATH/subway"
 # The output file, which needs post-processing by transit_graph_generator.py
-export MAPSME="$SUBWAYS_PATH/subways.json"
+export MAPSME="$SUBWAYS_PATH/subway.json"
 
 # Produce additional files needed for https://cdn.organicmaps.app/subway/
 export HTML_DIR="$SUBWAYS_VALIDATOR_PATH"
@@ -30,7 +30,12 @@ export DUMP="$SUBWAYS_VALIDATOR_PATH"
 export GEOJSON="$SUBWAYS_VALIDATOR_PATH"
 export DUMP_CITY_LIST="$SUBWAYS_VALIDATOR_PATH/cities.txt"
 
-"$SUBWAYS_REPO_PATH/scripts/process_subways.sh" 2>&1 | tee "$SUBWAYS_LOG"
+# cd to subways repo so relative paths work in the script
+PREVDIR=$(pwd)
+cd "$SUBWAYS_REPO_PATH"
+echo "Running process_subways.sh:"
+./scripts/process_subways.sh 2>&1 | tee "$SUBWAYS_LOG"
+cd "$PREVDIR"
 
 # Make render.html available for map visualization on the web
 cp -r "$SUBWAYS_REPO_PATH"/render/* "$SUBWAYS_VALIDATOR_PATH/"

tools/unix/maps/run-docker.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Run the maps generator via Docker manually without CI
+# See .forgejo/workflows/map-generator.yml for steps to run before the main mapgen process,
+# e.g. clone the repos, get/update planet file, generate isolines etc.
+
+# To build the docker container:
+# cd /mnt/4tbexternal/comaps/tools/unix/maps
+# docker build . -t maps_generator
+#
+# To push for ci/cd, tag for codeberg:
+#   docker login codeberg.org
+#   docker tag maps_generator codeberg.org/comaps/maps_generator:latest
+#   docker push codeberg.org/comaps/maps_generator:latest
+# You can also tag and push the image Id for posterity: codeberg.org/comaps/maps_generator:1234abcd
+# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first.
+# Or use docker via apt instead of snap.
+
+# We assume that the following will be cloned into the container itself at runtime:
+# ~/comaps (comaps main app repo)
+# ~/subways (repo for processing OSM subway/transit info)
+# ~/wikiparser (repo for processing Wikipedia data)
+
+# We also assume a number of files/folders/repos are pre-set-up before mounting via volumes below:
+# /mnt/4tbexternal (base folder for directory traversal)
+# /mnt/4tbexternal/osm-maps (folder for holding generated map data output)
+# /home/planet (folder for holding required input dumps)
+
+docker run \
+  --ulimit nofile=262144:262144 \
+  -v /mnt/4tbexternal/:/mnt/4tbexternal/ \
+  -v /mnt/4tbexternal/osm-planet:/home/planet \
+  -it codeberg.org/comaps/maps_generator:latest \
+  ~/comaps/tools/unix/maps/docker_maps_generator.sh

tools/unix/maps/settings.sh.prod

@@ -0,0 +1,26 @@
+## NOTE: edit the settings.sh file to customize/override the defaults.
+
+# Absolutize & normalize paths.
+REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}"
+
+#TODO: is base path used?
+BASE_PATH="${BASE_PATH:-$REPO_PATH/..}"
+# Source map data and processed outputs e.g. wiki articles
+DATA_PATH="${DATA_PATH:-/home/planet}"
+# Temporary files
+BUILD_PATH="${BUILD_PATH:-$DATA_PATH/build}"
+# Other code repositories, e.g. subways, wikiparser..
+CODE_PATH="${CODE_PATH:-~}"
+
+# OSM planet source files
+
+PLANET_PATH="${PLANET_PATH:-$DATA_PATH/planet}"
+PLANET_PBF="${PLANET_PBF:-$PLANET_PATH/planet-latest.osm.pbf}"
+PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
+
+# Subways
+
+SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-/root/subways}"
+SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}"
+SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}"
+SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"

tools/unix/maps/settings_default.sh

@@ -20,6 +20,6 @@ PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
 # Subways
 
 SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-$CODE_PATH/subways}"
-SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subways}"
-SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subways.log}"
-SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"
+SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}"
+SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}"
+SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"

tools/unix/maps/upload_to_cdn.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Upload new maps version to all CDN nodes (in parallel).
+# Upload new maps version to all CDN nodes (in parallel) and remove old versions.
 
 # Use following commands for deleting older maps:
 #
@@ -8,15 +8,15 @@
 # First list all maps versions on the server
 #   sudo rclone lsd ru1:comaps-maps/maps
 # Delete the old version
-#   sudo rclone purge -v ru1:comaps-maps/maps/250713
+#   sudo rclone purge -v ru1:comaps-maps/maps/250713/
 #
 # fi1 - max 3 versions
 #   sudo rclone lsd fi1:/var/www/html/maps
-#   sudo rclone purge -v fi1:/var/www/html/maps/250713
+#   sudo rclone purge -v fi1:/var/www/html/maps/250713/
 #
 # de1 - max 6 versions
 #    sudo rclone lsd de1:/var/www/html/comaps-cdn/maps
-#    sudo rclone purge -v de1:/var/www/html/comaps-cdn/maps/250713
+#    sudo rclone purge -v de1:/var/www/html/comaps-cdn/maps/250713/
 #
 # us2 - all versions, don't delete
 #    sudo rclone lsd us2:comaps-map-files/maps
@@ -39,6 +39,54 @@ DIR=$(dirname $1)/$MAPS
 
 echo "Uploading maps folder $DIR to $MAPS"
 
+# Remove old versions before uploading new ones
+echo "Checking for old versions to remove..."
+
+# ru1 - keep max 3 versions
+echo "Cleaning ru1 (keeping 3 newest versions)..."
+OLD_VERSIONS_RU1=$(rclone lsd ru1:comaps-maps/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_RU1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting ru1:comaps-maps/maps/$version/"
+    rclone purge -v ru1:comaps-maps/maps/$version/
+  fi
+done
+
+# fi1 - keep max 3 versions
+echo "Cleaning fi1 (keeping 3 newest versions)..."
+OLD_VERSIONS_FI1=$(rclone lsd fi1:/var/www/html/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_FI1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting fi1:/var/www/html/maps/$version/"
+    rclone purge -v fi1:/var/www/html/maps/$version/
+  fi
+done
+
+# de1 - keep max 6 versions
+echo "Cleaning de1 (keeping 6 newest versions)..."
+OLD_VERSIONS_DE1=$(rclone lsd de1:/var/www/html/comaps-cdn/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_DE1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting de1:/var/www/html/comaps-cdn/maps/$version/"
+    rclone purge -v de1:/var/www/html/comaps-cdn/maps/$version/
+  fi
+done
+
+# fr1 - keep max 6 versions
+echo "Cleaning fr1 (keeping 6 newest versions)..."
+OLD_VERSIONS_FR1=$(rclone lsd fr1:/data/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_FR1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting fr1:/data/maps/$version/"
+    rclone purge -v fr1:/data/maps/$version/
+  fi
+done
+
+# us2 - keep all versions (no cleanup)
+echo "Skipping us2 cleanup (keeping all versions)"
+
+echo "Old version cleanup complete"
+
 echo "Uploading to us2"
 # An explicit mwm/txt filter is used to skip temp files when run for an unfinished generation
 rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS &
@@ -52,9 +100,29 @@ rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS &
 echo "Uploading to de1"
 rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS &
 
+echo "Uploading to fr1"
+rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS &
+
 # us1 is not used for maps atm
 # rclone lsd us1:/home/dh_zzxxrk/cdn-us-1.comaps.app/maps
 
 wait
 
+echo "Running once more without parallelization to output status:"
+
+echo "us2 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS
+
+echo "ru1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS
+
+echo "fi1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS
+
+echo "de1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS
+
+echo "fr1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS
+
 echo "Upload complete"