lower thread count for less OOM

Signed-off-by: zyphlar <zyphlar@gmail.com>

.forgejo/workflows/map-generator.yml

@@ -9,6 +9,7 @@ on:
         type: choice
         options:
           - all
+          - clone-repos
           - copy-coasts
           - planet
           - wiki
@@ -16,30 +17,95 @@ on:
           - subways
           - tiger
           - maps
+          - upload
+      reclone:
+        description: 'Force a re-clone of all git repos?'
+        required: false
+        default: 'no'
+        type: choice
+        options:
+          - 'no'
+          - force
+
+## RCLONE_CONF is multi-line text containing keys and credentials for us2,ru1,fi1,de1 servers
 
 env:
+  RCLONE_CONF: ${{ secrets.RCLONE_CONF }}
   WIKIMEDIA_USERNAME: ${{ secrets.WIKIMEDIA_USERNAME }}
   WIKIMEDIA_PASSWORD: ${{ secrets.WIKIMEDIA_PASSWORD }}
-  S3_KEY_ID: ${{ secrets.S3_KEY_ID }}
-  S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
-  S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-  S3_BUCKET: ${{ secrets.S3_BUCKET }}
-  SFTP_USER: ${{ secrets.SFTP_USER }}
-  SFTP_PASSWORD: ${{ secrets.SFTP_PASSWORD }}
-  SFTP_HOST: ${{ secrets.SFTP_HOST }}
-  SFTP_PATH: ${{ secrets.SFTP_PATH }}
+  ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }}
+  ZULIP_API_KEY: ${{ secrets.ZULIP_API_KEY }}
   DEBIAN_FRONTEND: noninteractive
   TZ: Etc/UTC 
 
 jobs:
+  clone-repos:
+    if: inputs.jobs == 'clone-repos' || inputs.jobs == 'all'
+    name: Clone Git Repos
+    runs-on: mapfilemaker
+    container:
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
+      volumes:
+        - /mnt/4tbexternal:/mnt/4tbexternal
+    concurrency:
+      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Clone or update main repo if necessary
+        shell: bash
+        run: |
+          ls -al /mnt
+          ls -al /mnt/4tbexternal
+          if [[ '${{ inputs.reclone }}' == 'force' ]]; then
+          rm -rf /mnt/4tbexternal/comaps-init
+          fi
+
+          if [ ! -d /mnt/4tbexternal/comaps-init ]; then
+          cd /mnt/4tbexternal
+          git clone --recurse-submodules --shallow-submodules -b rebase-generator-pastk-wb251027 --single-branch https://codeberg.org/comaps/comaps.git comaps-init
+          else
+          cd /mnt/4tbexternal/comaps-init && git pull origin rebase-generator-pastk-wb251027
+          fi
+      - name: Clone or update wikiparser repo if necessary
+        shell: bash
+        run: |
+          if [[ '${{ inputs.reclone }}' == 'force' ]]; then
+          rm -rf /mnt/4tbexternal/wikiparser
+          fi
+
+          if [ ! -d /mnt/4tbexternal/wikiparser ]; then
+          cd /mnt/4tbexternal
+          git clone https://codeberg.org/comaps/wikiparser.git
+          else
+          cd /mnt/4tbexternal/wikiparser && git pull origin main
+          fi
+      - name: Clone or update subways repo if necessary
+        shell: bash
+        run: |
+          if [[ '${{ inputs.reclone }}' == 'force' ]]; then
+          rm -rf /mnt/4tbexternal/subways
+          fi
+
+          if [ ! -d /mnt/4tbexternal/subways ]; then
+          cd /mnt/4tbexternal
+          git clone https://codeberg.org/comaps/subways.git
+          else
+          cd /mnt/4tbexternal/subways && git pull origin master
+          fi
+
   copy-coasts:
     if: inputs.jobs == 'copy-coasts' || inputs.jobs == 'all'
     name: Copy Previously Generated Coasts
     runs-on: mapfilemaker
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
@@ -47,9 +113,9 @@ jobs:
       - name: Copy Coasts
         shell: bash
         run: |
-          if [ -f /media/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.geom ]; then
-          cp /media/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.geom /media/4tbexternal/osm-planet/latest_coasts.geom
-          cp /media/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.rawgeom /media/4tbexternal/osm-planet/latest_coasts.rawgeom
+          if [ -f /root/OM/osm-maps/*/intermediate_data/WorldCoasts.geom ]; then
+          cp /root/OM/osm-maps/*/intermediate_data/WorldCoasts.geom /home/planet/latest_coasts.geom
+          cp /root/OM/osm-maps/*/intermediate_data/WorldCoasts.rawgeom /home/planet/latest_coasts.rawgeom
           fi
 
   update-planet:
@@ -57,86 +123,93 @@ jobs:
     name: Update Planet
     runs-on: mapfilemaker
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Install dependencies
-        shell: bash
-        run: |
-          apt-get update -y
-          apt-get install -y pyosmium osmium-tool python3-venv python3-pip wget2
-          rm -f /usr/lib/python*/EXTERNALLY-MANAGED
-          pip3 install "protobuf<4"
       - name: Download Planet File if Absent
         shell: bash
         run: |
-          if [ ! -d /media/4tbexternal/osm-planet/planet/ ]; then
-          mkdir -p /media/4tbexternal/osm-planet/planet/
+          if [ ! -d /home/planet/planet/ ]; then
+          mkdir -p /home/planet/planet/
           fi
-          if [ ! -f /media/4tbexternal/osm-planet/planet/planet-latest.osm.pbf ]; then
-          cd /media/4tbexternal/osm-planet/planet/
-          wget2 --verbose --progress=bar --continue --debug https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf
+          if [ ! -f /home/planet/planet/planet-latest.osm.pbf ]; then
+          cd /home/planet/planet/
+          wget2 --verbose --progress=bar --continue https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf
           fi
       - name: Update Planet
         shell: bash
         run: |
-          cd /media/4tbexternal/osm-planet/planet/
+          cd /home/planet/planet/
           pyosmium-up-to-date planet-latest.osm.pbf -o planet-latest-new.osm.pbf -vv --size 16384
           mv planet-latest-new.osm.pbf planet-latest.osm.pbf
       - name: Converting planet-latest.osm.pbf to planet.o5m
-        run: /root/OM/osmctools/osmconvert planet-latest.osm.pbf -o=planet.o5m
+        run: |
+          echo "Starting..."
+          cd /home/planet/planet/
+          osmconvert planet-latest.osm.pbf -o=planet.o5m
+          echo "Done."
+      - name: Notify Zulip
+        run: |
+          curl -X POST https://comaps.zulipchat.com/api/v1/messages \
+            -u $ZULIP_BOT_EMAIL:$ZULIP_API_KEY \
+            --data-urlencode type=stream \
+            --data-urlencode 'to="DevOps"' \
+            --data-urlencode topic=codeberg-bot \
+            --data-urlencode 'content=Planet update is done!'
 
   wiki-update:
     if: inputs.jobs == 'wiki' || inputs.jobs == 'all'
     name: Update Wikipedia
     runs-on: mapfilemaker
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Install dependencies
-        shell: bash
-        run: |
-          apt-get update -y
-          apt-get install -y jq curl wget2 rustc cargo git ca-certificates
-      - name: Clone wikiparser if necessary
-        shell: bash
-        run: |
-          if [ ! -d /media/4tbexternal/wikiparser ]; then
-          cd /media/4tbexternal
-          git clone https://codeberg.org/comaps/wikiparser.git
-          fi
       - name: Check for planet file
         shell: bash
         run: |
-          if [ ! -f /media/4tbexternal/osm-planet/planet/planet-latest.osm.pbf ]; then
-          echo "ERROR: No file at /media/4tbexternal/osm-planet/planet/planet-latest.osm.pbf"
-          ls -al /media/4tbexternal/
-          ls -al /media/4tbexternal/osm-planet/
-          ls -al /media/4tbexternal/osm-planet/planet/
+          if [ ! -f /home/planet/planet/planet-latest.osm.pbf ]; then
+          echo "ERROR: No file at /home/planet/planet/planet-latest.osm.pbf"
+          ls -al /home/planet/
+          ls -al /home/planet/planet/
           exit 1
           fi
       - name: Update Wikipedia from Enterprise API
         shell: bash
         run: |
-          mkdir -p /media/4tbexternal/osm-planet/wikipedia/dumps
-          mkdir -p /media/4tbexternal/osm-planet/wikipedia/build
-          cd /media/4tbexternal/wikiparser
+          #todo: curl in download.sh can fail when rate limited and even save error messages to the output. need to validate.
+          #downloading all languages can also trigger rate limits or fail as well. needs work.
+          #also: a failure to download means a failure to build, and could result in no wiki descriptions etc.
+          #also-also: do we want to remove old wiki data in planet between builds?
+          mkdir -p /home/planet/wikipedia/dumps
+          mkdir -p /home/planet/wikipedia/build
+          cd /root/OM/wikiparser
           ls -al
           echo "Downloading ..."
-          ./download.sh /media/4tbexternal/osm-planet/wikipedia/dumps
+          ./download.sh /home/planet/wikipedia/dumps
+          ls -al /home/planet/wikipedia/dumps/*
           echo "Running ..."
-          ./run.sh /media/4tbexternal/osm-planet/wikipedia/build \
-          /media/4tbexternal/osm-planet/planet/planet-latest.osm.pbf \
-          /media/4tbexternal/osm-planet/wikipedia/dumps/latest/*.tar.gz
+          ./run.sh /home/planet/wikipedia/build \
+          /home/planet/planet/planet-latest.osm.pbf \
+          /home/planet/wikipedia/dumps/latest/*.tar.gz
           echo "DONE"
 
   update-isolines:
@@ -144,74 +217,34 @@ jobs:
     name: Update Isolines
     runs-on: mapfilemaker
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Install dependencies
-        shell: bash
-        run: |
-          apt-get update -qq \
-          && apt-get install -y --no-install-recommends \
-          curl \
-          osmctools \
-          rclone \
-          git \
-          ca-certificates \
-          openssh-client \
-          sshpass \
-          vim \
-          wget \
-          build-essential \
-          clang \
-          cmake \
-          python3 \
-          python3-pip \
-          python3.12-venv \
-          qt6-base-dev \
-          qt6-positioning-dev \
-          libc++-dev \
-          libfreetype-dev \
-          libglvnd-dev \
-          libgl1-mesa-dev \
-          libharfbuzz-dev \
-          libicu-dev \
-          libqt6svg6-dev \
-          libqt6positioning6-plugins \
-          libqt6positioning6 \
-          libsqlite3-dev \
-          libxrandr-dev \
-          libxinerama-dev \
-          libxcursor-dev \
-          libxi-dev \
-          zlib1g-dev
-          rm -f /usr/lib/python*/EXTERNALLY-MANAGED
-          pip3 install "protobuf<4"
-      - name: Clone main repo if necessary
-        shell: bash
-        run: |
-          if [ ! -d /media/4tbexternal/comaps-init ]; then
-          apt-get update -qq && apt-get install -y --no-install-recommends git
-          cd /media/4tbexternal
-          git clone --recurse-submodules --shallow-submodules -b rebase-generator-pastk-wb251014 --single-branch https://codeberg.org/comaps/comaps.git comaps-init
-          fi
+      # TODO: we only need to update these if our SRTM or countries change
+      # TODO: after update, verify that sizable files exist: /home/planet/isolines/*.isolines
       - name: Update Isolines
         shell: bash
         run: |
-          cd /media/4tbexternal/comaps-init/
+          cd /root/OM/comaps-init/
           ./tools/unix/build_omim.sh -R topography_generator_tool
-          rm -rf ../osm-planet/isolines/
-          mkdir ../osm-planet/isolines/
+          rm -rf /home/planet/isolines/
+          mkdir /home/planet/isolines/
           ../omim-build-relwithdebinfo/topography_generator_tool \
           --profiles_path=./data/conf/isolines/isolines-profiles.json \
           --countries_to_generate_path=./data/conf/isolines/countries-to-generate.json \
-          --tiles_isolines_out_dir=../osm-planet/isolines/tmp-tiles/ \
-          --countries_isolines_out_dir=../osm-planet/isolines/ \
+          --tiles_isolines_out_dir=/home/planet/isolines/tmp-tiles/ \
+          --countries_isolines_out_dir=/home/planet/isolines/ \
           --data_dir=./data/ \
-          --srtm_path=../osm-planet/SRTM-patched-europe/ \
+          --srtm_path=/home/planet/SRTM-patched-europe/ \
           --threads=22
 
   update-subways:
@@ -219,37 +252,22 @@ jobs:
     name: Update Subways
     runs-on: mapfilemaker
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Install dependencies
-        shell: bash
-        run: |
-          apt-get update -qq && apt-get install -y --no-install-recommends curl osmctools osmium-tool python3-venv ca-certificates git python3-pip
-          rm -f /usr/lib/python*/EXTERNALLY-MANAGED
-          pip3 install "protobuf<4"
-      - name: Clone subways if necessary
-        shell: bash
-        run: |
-          if [ ! -d /media/4tbexternal/subways ]; then
-          cd /media/4tbexternal
-          git clone https://codeberg.org/comaps/subways.git
-          fi
-      - name: Clone main repo if necessary
-        shell: bash
-        run: |
-          if [ ! -d /media/4tbexternal/comaps-init ]; then
-          cd /media/4tbexternal
-          git clone --recurse-submodules --shallow-submodules -b rebase-generator-pastk-wb251014 --single-branch https://codeberg.org/comaps/comaps.git comaps-init
-          fi
       - name: Update Subways
         shell: bash
         run: |
-          cd /media/4tbexternal/comaps-init/
+          cd /root/OM/comaps-init/
           cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh
           ./tools/unix/maps/generate_subways.sh
 
@@ -258,125 +276,127 @@ jobs:
     name: Update TIGER
     runs-on: mapfilemaker
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Install dependencies
-        shell: bash
-        run: |
-          apt-get update -qq && apt-get install -y --no-install-recommends \
-          build-essential \
-          clang \
-          cmake \
-          ninja-build \
-          ca-certificates \
-          git \
-          wget2
-      - name: Clone main repo if necessary
-        shell: bash
-        run: |
-          if [ ! -d /media/4tbexternal/comaps-init ]; then
-          cd /media/4tbexternal
-          git clone --recurse-submodules --shallow-submodules -b rebase-generator-pastk-wb251014 --single-branch https://codeberg.org/comaps/comaps.git comaps-init
-          fi
       - name: Build address_parser
         shell: bash
         run: |
-          cd /media/4tbexternal/comaps-init
+          cd /root/OM/comaps-init
           rm -rf ../omim-build-relwithdebinfo/CMakeCache.txt
           rm -rf ../omim-build-relwithdebinfo/CMakeFiles
           ./tools/unix/build_omim.sh -R address_parser_tool
       - name: Update TIGER from Nominatim
         shell: bash
         run: |
-          cd /media/4tbexternal/osm-planet/
+          # TODO: maybe remove old osm-planet/tiger first?
+          cd /home/planet/
+          mkdir -p tiger
           wget2 https://nominatim.org/data/tiger-nominatim-preprocessed-latest.csv.tar.gz
-          tar -xOzf tiger-nominatim-preprocessed-latest.csv.tar.gz | /media/4tbexternal/omim-build-relwithdebinfo/address_parser_tool --output_path=./tiger
+          cd /root/OM/comaps-init
+          tar -xOzf /home/planet/tiger-nominatim-preprocessed-latest.csv.tar.gz | /root/OM/omim-build-relwithdebinfo/address_parser_tool --output_path=/home/planet/tiger
 
   generate-maps:
     if: inputs.jobs == 'maps' || inputs.jobs == 'all'
     name: Generate Maps
     runs-on: mapfilemaker
+    timeout-minutes: 40320
     container:
-      image: ubuntu:latest
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
       volumes:
-        - /media/4tbexternal:/media/4tbexternal
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
       options: --ulimit nofile=262144:262144
     concurrency:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Install dependencies
-        shell: bash
-        run: |
-          apt-get update -qq \
-          && apt-get install -y --no-install-recommends \
-          curl \
-          osmctools \
-          rclone \
-          git \
-          ca-certificates \
-          openssh-client \
-          sshpass \
-          vim \
-          wget \
-          build-essential \
-          clang \
-          cmake \
-          ninja-build \
-          python3 \
-          python3-pip \
-          python3.12-venv \
-          qt6-base-dev \
-          qt6-positioning-dev \
-          libc++-dev \
-          libfreetype-dev \
-          libglvnd-dev \
-          libgl1-mesa-dev \
-          libharfbuzz-dev \
-          libicu-dev \
-          libqt6svg6-dev \
-          libqt6positioning6-plugins \
-          libqt6positioning6 \
-          libsqlite3-dev \
-          libxrandr-dev \
-          libxinerama-dev \
-          libxcursor-dev \
-          libxi-dev \
-          zlib1g-dev
-      - name: Clone repo if necessary
-        shell: bash
-        run: |
-          if [ ! -d /media/4tbexternal/comaps-init ]; then
-          cd /media/4tbexternal
-          git clone --recurse-submodules --shallow-submodules -b rebase-generator-pastk-wb251014 --single-branch https://codeberg.org/comaps/comaps.git comaps-init
-          fi
       - name: Make output folders if necessary
         shell: bash
         run: |
-          if [ ! -d /media/4tbexternal/osm-maps ]; then
-          mkdir -p /media/4tbexternal/osm-maps
+          if [ ! -d /root/OM/osm-maps ]; then
+          mkdir -p /root/OM/osm-maps
           fi
       - name: Get SRTM if necessary
         shell: bash
         run: |
-          if [ ! -d /media/4tbexternal/osm-planet/SRTM-patched-europe/ ]; then
+          if [ ! -d /home/planet/SRTM-patched-europe/ ]; then
           echo "ERROR: NO SRTM"
           exit 1
           fi
-      - name: Symlink paths for repo scripts
-        shell: bash
-        run: |
-          mkdir -p /root/OM
-          ln -s /media/4tbexternal/comaps-init /root/OM/organicmaps
-          ln -s /media/4tbexternal/osm-planet /home/planet
-          ln -s /media/4tbexternal/osm-maps /root/OM/maps_build
       - name: Run docker_maps_generator.sh
         shell: bash
         run: |
-          cd /root/OM/organicmaps
-          ./tools/unix/docker_maps_generator.sh
+          cd /root/OM/comaps-init
+          ./tools/unix/docker_maps_generator.sh
+      - name: Notify Zulip
+        run: |
+          curl -X POST https://comaps.zulipchat.com/api/v1/messages \
+            -u $ZULIP_BOT_EMAIL:$ZULIP_API_KEY \
+            --data-urlencode type=stream \
+            --data-urlencode 'to="DevOps"' \
+            --data-urlencode topic=codeberg-bot \
+            --data-urlencode 'content=Generator is done!'
+
+  upload-maps:
+    if: inputs.jobs == 'upload' || inputs.jobs == 'all'
+    name: Upload Maps
+    runs-on: mapfilemaker
+    container:
+      image: codeberg.org/comaps/maps_generator:96d2a38aa23d
+      volumes:
+        - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
+        - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
+        - /mnt/4tbexternal/osm-planet:/home/planet
+        - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
+        - /mnt/4tbexternal/subways:/root/OM/subways
+        - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
+    concurrency:
+      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Write config file
+        run: |
+          mkdir -p ~/.config/rclone/
+          echo "${{ secrets.RCLONE_CONF }}" > ~/.config/rclone/rclone.conf
+          head -n3 ~/.config/rclone/rclone.conf
+      - name: Upload map files to CDNs
+        shell: bash
+        run: |
+          shopt -s nullglob
+          buildfolder=$(find /root/OM/osm-maps/ -mindepth 1 -maxdepth 1 -iname "2*" -type d | sort -n -r | head -1 | cut -d/ -f5)
+          builddate=$(find /root/OM/osm-maps/*/ -mindepth 1 -maxdepth 1 -iname "2*" -type d | sort -n -r | head -1 | cut -d/ -f6)
+          mwmfiles=( /root/OM/osm-maps/$buildfolder/$builddate/*.mwm )
+
+          if (( ${#mwmfiles[@]} )); then
+            echo "<$(date +%T)> Uploading maps from $buildfolder/$builddate..."
+            cd /root/OM/comaps-init/tools/unix/maps
+            ./upload_to_cdn.sh /root/OM/osm-maps/$buildfolder/$builddate
+          else
+            echo "<$(date +%T)> No MWM files in /root/OM/osm-maps/$buildfolder/$builddate/*.mwm, not uploading maps."
+            echo "<$(date +%T)> Found top level: $(ls -alt /root/OM/osm-maps/*)"
+            echo "<$(date +%T)> Found second level: $(ls -alt /root/OM/osm-maps/$buildfolder/*)"
+          fi
+      - name: Notify Zulip
+        run: |
+          curl -X POST https://comaps.zulipchat.com/api/v1/messages \
+            -u $ZULIP_BOT_EMAIL:$ZULIP_API_KEY \
+            --data-urlencode type=stream \
+            --data-urlencode 'to="DevOps"' \
+            --data-urlencode topic=codeberg-bot \
+            --data-urlencode 'content=Upload is done!'
+
+          

.gitignore

@@ -9,6 +9,7 @@ Makefile.Release
 object_script.*.Debug
 object_script.*.Release
 compile_commands.json
+*.local.*
 
 stxxl.errlog
 stxxl.log

data/categories-strings/en.json/localize.json

@@ -28,7 +28,7 @@
 "shop-bakery|shop-pastry|@category_eat|@category_food|@shop": "3Bakery|3Pastry|Cake|3Cakes|U+1F35E",
 "shop|@shop": "",
 "shop-cannabis|@shop": "Cannabis",
-"shop-cosmetics|@category_shopping|@shop": "4Cosmetics|4Beauty Care|makeup",
+"shop-cosmetics|@category_shopping|@shop": "4Cosmetics|4Beauty Care|makeup|make-up",
 "shop-convenience|@category_food|@shop": "4Convenience|mini-mart|mini-market|superette",
 "shop-deli|@category_food|@shop": "4Delicatessen",
 "shop-farm|@category_food|@shop": "Farm food",

data/categories-strings/es.json/localize.json

@@ -225,11 +225,11 @@
 "tourism-alpine_hut|@category_hotel": "4Cabaña|cabaña alpina|hotel de montaña",
 "shop-hairdresser": "4Peluquería",
 "aeroway-aerodrome": "3Aeropuerto|3avión|aeródromo",
-"leisure-stadium": "5Estadio",
-"leisure-playground|@category_children": "Patio de recreo|4Parque infantil|Corral|Corralito",
-"leisure-sports_centre|leisure-sports_centre-sport-american_football|leisure-sports_centre-sport-archery|leisure-sports_centre-sport-athletics|leisure-sports_centre-sport-australian_football|leisure-sports_centre-sport-badminton|leisure-sports_centre-sport-baseball|leisure-sports_centre-sport-basketball|leisure-sports_centre-sport-beachvolleyball|leisure-sports_centre-sport-bowls|leisure-sports_centre-sport-climbing|leisure-sports_centre-sport-cricket|leisure-sports_centre-sport-curling|leisure-sports_centre-sport-equestrian|leisure-sports_centre-sport-field_hockey|leisure-sports_centre-sport-futsal|leisure-sports_centre-sport-golf|leisure-sports_centre-sport-gymnastics|leisure-sports_centre-sport-handball|leisure-sports_centre-sport-ice_hockey|leisure-sports_centre-sport-multi|leisure-sports_centre-sport-padel|leisure-sports_centre-sport-pelota|leisure-sports_centre-sport-scuba_diving|leisure-sports_centre-sport-shooting|leisure-sports_centre-sport-skateboard|leisure-sports_centre-sport-skiing|leisure-sports_centre-sport-soccer|leisure-sports_centre-sport-table_tennis|leisure-sports_centre-sport-tennis|leisure-sports_centre-sport-volleyball|leisure-sports_centre-sport-yoga": "Polideportivo|Centro deportivo|deporte",
-"leisure-golf_course": "Campo de golf|Golf",
-"leisure-miniature_golf": "5Minigolf|Mini golf",
+"leisure-stadium": "5Estadio|4deporte",
+"leisure-playground|@category_children": "Patio de recreo|Parque infantil",
+"leisure-sports_centre|leisure-sports_centre-sport-american_football|leisure-sports_centre-sport-archery|leisure-sports_centre-sport-athletics|leisure-sports_centre-sport-australian_football|leisure-sports_centre-sport-badminton|leisure-sports_centre-sport-baseball|leisure-sports_centre-sport-basketball|leisure-sports_centre-sport-beachvolleyball|leisure-sports_centre-sport-bowls|leisure-sports_centre-sport-climbing|leisure-sports_centre-sport-cricket|leisure-sports_centre-sport-curling|leisure-sports_centre-sport-equestrian|leisure-sports_centre-sport-field_hockey|leisure-sports_centre-sport-futsal|leisure-sports_centre-sport-golf|leisure-sports_centre-sport-gymnastics|leisure-sports_centre-sport-handball|leisure-sports_centre-sport-ice_hockey|leisure-sports_centre-sport-multi|leisure-sports_centre-sport-padel|leisure-sports_centre-sport-pelota|leisure-sports_centre-sport-scuba_diving|leisure-sports_centre-sport-shooting|leisure-sports_centre-sport-skateboard|leisure-sports_centre-sport-skiing|leisure-sports_centre-sport-soccer|leisure-sports_centre-sport-table_tennis|leisure-sports_centre-sport-tennis|leisure-sports_centre-sport-volleyball|leisure-sports_centre-sport-yoga": "Polideportivo|deporte",
+"leisure-golf_course": "Campo de golf",
+"leisure-miniature_golf": "Minigolf",
 "leisure-escape_game": "Escape room|Sala de escape|Cuarto de escape",
 "leisure-hackerspace": "Espacio hacker|Espacio para hackers|Hacklab|Hackerspace",
 "leisure-pitch": "pista|deporte|cancha|campo",

data/categories-strings/fr.json/localize.json

@@ -224,7 +224,7 @@
 "leisure-stadium": "4Stade|4sport",
 "leisure-playground|@category_children": "Terrain de jeu|aire de jeu",
 "leisure-sports_centre|leisure-sports_centre-sport-american_football|leisure-sports_centre-sport-archery|leisure-sports_centre-sport-athletics|leisure-sports_centre-sport-australian_football|leisure-sports_centre-sport-badminton|leisure-sports_centre-sport-baseball|leisure-sports_centre-sport-basketball|leisure-sports_centre-sport-beachvolleyball|leisure-sports_centre-sport-bowls|leisure-sports_centre-sport-climbing|leisure-sports_centre-sport-cricket|leisure-sports_centre-sport-curling|leisure-sports_centre-sport-equestrian|leisure-sports_centre-sport-field_hockey|leisure-sports_centre-sport-futsal|leisure-sports_centre-sport-golf|leisure-sports_centre-sport-gymnastics|leisure-sports_centre-sport-handball|leisure-sports_centre-sport-ice_hockey|leisure-sports_centre-sport-multi|leisure-sports_centre-sport-padel|leisure-sports_centre-sport-pelota|leisure-sports_centre-sport-scuba_diving|leisure-sports_centre-sport-shooting|leisure-sports_centre-sport-skateboard|leisure-sports_centre-sport-skiing|leisure-sports_centre-sport-soccer|leisure-sports_centre-sport-table_tennis|leisure-sports_centre-sport-tennis|leisure-sports_centre-sport-volleyball|leisure-sports_centre-sport-yoga": "Centre sportif|sport",
-"leisure-golf_course": "Terrain de golf|Parcours de golf",
+"leisure-golf_course": "Terrain de golf",
 "leisure-miniature_golf": "Minigolf",
 "leisure-escape_game": "Salle d'évasion",
 "leisure-pitch": "Terrain de sport|sport",

data/categories-strings/hu.json/localize.json

@@ -210,7 +210,7 @@
 "leisure-stadium": "4Stadion|4sport|U+26BD|U+26BE|U+1F3BE|U+1F3C0|U+1F3C8|U+1F4AA|U+1F3C9|aréna|olimpiai stadion",
 "leisure-playground|@category_children": "Játszótér",
 "leisure-sports_centre|leisure-sports_centre-sport-american_football|leisure-sports_centre-sport-archery|leisure-sports_centre-sport-athletics|leisure-sports_centre-sport-australian_football|leisure-sports_centre-sport-badminton|leisure-sports_centre-sport-baseball|leisure-sports_centre-sport-basketball|leisure-sports_centre-sport-beachvolleyball|leisure-sports_centre-sport-bowls|leisure-sports_centre-sport-climbing|leisure-sports_centre-sport-cricket|leisure-sports_centre-sport-curling|leisure-sports_centre-sport-equestrian|leisure-sports_centre-sport-field_hockey|leisure-sports_centre-sport-futsal|leisure-sports_centre-sport-golf|leisure-sports_centre-sport-gymnastics|leisure-sports_centre-sport-handball|leisure-sports_centre-sport-ice_hockey|leisure-sports_centre-sport-multi|leisure-sports_centre-sport-padel|leisure-sports_centre-sport-pelota|leisure-sports_centre-sport-scuba_diving|leisure-sports_centre-sport-shooting|leisure-sports_centre-sport-skateboard|leisure-sports_centre-sport-skiing|leisure-sports_centre-sport-soccer|leisure-sports_centre-sport-table_tennis|leisure-sports_centre-sport-tennis|leisure-sports_centre-sport-volleyball|leisure-sports_centre-sport-yoga": "4Sportközpont|sport",
-"leisure-golf_course": "Golfpálya|U+26F3",
+"leisure-golf_course": "Golfpálya",
 "leisure-miniature_golf": "Minigolf",
 "leisure-escape_game": "Szabadulószoba",
 "leisure-pitch": "4Sportpálya|sport|U+26BD|U+26BE|U+1F3BE|U+1F3C0|U+1F3C8|U+1F3C9|U+1F3C3",

data/styles/default/dark/colors.mapcss

@@ -231,14 +231,14 @@
 @subway_label: #9E9E9E;
 @indigenous_label: #6A512F;
 @shop_label: #6B425C;
-@industry_label: #574469;
+@industry_label: #000000;
 @food_label: #8C491C;
-@entertainment_label: #8C311B;
+@entertainment_label: #000000;
 @culture_label: #6E4426;
 @hotel_label: #614A43;
-@sport_label: #2B5C4B;
-@nature_label: #3C612E;
-@water_label: #0A6074;
+@sport_label: #000000;
+@nature_label: #000000;
+@water_label: #000000;
 @neutral_label: #51585E;
 @healthcare_label: #983E44;
 @public_transport_label: #2F6499;

data/styles/default/include/Icons.mapcss

@@ -44,7 +44,7 @@ node|z18-[office],
 {icon-image: office-m.svg;}
 
 node|z18-[craft],
-{icon-image: craft-m.svg;}
+{text-color: @industry_label;text-halo-radius: 1;text-halo-opacity: 0.9;text-halo-color: @label_halo_light;icon-image: craft-m.svg;}
 
 node|z16-[shop],
 {icon-min-distance: 10;}
@@ -441,15 +441,17 @@ node|z18-[historic=stone],
 
 node|z12-14[historic=fort],
 node|z12-14[historic=castle],
-{icon-image: remains-s.svg;text-offset: 1;icon-min-distance: 12;}
+{icon-image: castle-s.svg;text-offset: 1;icon-min-distance: 12;}
 node|z15[historic=fort],
 node|z15[historic=castle],
-{icon-image: remains-m.svg;icon-min-distance: 5;}
+{icon-image: castle-m.svg;icon-min-distance: 5;}
 node|z16[historic=fort],
 node|z16[historic=castle],
-{icon-image: remains-m.svg;text-offset: 1;}
+{icon-image: castle-m.svg;text-offset: 1;}
 node|z17-[historic=fort],
 node|z17-[historic=castle],
+{icon-image: castle-m.svg;}
+node|z17-[historic=ruins][name],
 {icon-image: remains-m.svg;}
 
 node|z12-14[historic=castle][castle_type=castrum],
@@ -1308,7 +1310,7 @@ node|z15-[landuse=industrial][industrial=mine],
 {icon-image: mine-m.svg;text-offset: 1;font-size: 10;}
 
 node|z16-[leisure=water_park],
-{icon-image: swimming-m.svg;icon-min-distance: 30;}
+{icon-image: swimming-centre-m.svg;icon-min-distance: 30;}
 node|z17-[leisure=water_park],
 {icon-min-distance: 20;}
 
@@ -2269,4 +2271,239 @@ node|z18-[amenity=parking_space][parking_space=disabled],
 
 /* 8. Colored Labels */
 
-@import("../../default/include/Icons_Label_Colors.mapcss");
+node[amenity=car_rental],
+node[amenity=car_sharing],
+node[amenity=internet_cafe],
+node[amenity=marketplace],
+node[amenity=motorcycle_rental],
+node[shop=alcohol],
+node[shop=antiques],
+node[shop=art],
+node[shop=auction],
+node[shop=bakery],
+node[shop=beauty],
+node[shop=beverages],
+node[shop=bicycle],
+node[shop=books],
+node[shop=butcher],
+node[shop=camera],
+node[shop=car],
+node[shop=caravan],
+node[shop=charity],
+node[shop=cheese],
+node[shop=chemist],
+node[shop=chocolate],
+node[shop=clothes],
+node[shop=collector],
+node[shop=computer],
+node[shop=confectionery],
+node[shop=convenience],
+node[shop=copyshop],
+node[shop=cosmetics],
+node[shop=deli],
+node[shop=department_store],
+node[shop=doityourself],
+node[shop=dry_cleaning],
+node[shop=dry_cleaning],
+node[shop=electronics],
+node[shop=erotic],
+node[shop=fabric],
+node[shop=farm],
+node[shop=florist],
+node[shop=furniture],
+node[shop=garden_centre],
+node[shop=gift],
+node[shop=greengrocer],
+node[shop=grocery],
+node[shop=hairdresser],
+node[shop=hardware],
+node[shop=health_food],
+node[shop=hearing_aids],
+node[shop=houseware],
+node[shop=interior_decoration],
+node[shop=jewelry],
+node[shop=kiosk],
+node[shop=kitchen],
+node[shop=laundry],
+node[shop=laundry],
+node[shop=lighting],
+node[shop=mall],
+node[shop=mobile_phone],
+node[shop=motorcycle],
+node[shop=music],
+node[shop=musical_instrument],
+node[shop=optician],
+node[shop=outdoor],
+node[shop=pasta],
+node[shop=pastry],
+node[shop=pawnbroker],
+node[shop=pet],
+node[shop=photo],
+node[shop=rental],
+node[shop=rental][rental=bike],
+node[shop=seafood],
+node[shop=second_hand],
+node[shop=sewing],
+node[shop=shoes],
+node[shop=sports],
+node[shop=stationery],
+node[shop=supermarket],
+node[shop=ticket],
+node[shop=tobacco],
+node[shop=toys],
+node[shop=variety_store],
+node[shop=video],
+node[shop=video_games],
+node[shop=wine],
+node[shop],
+{text-color: @shop_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[amenity=car_wash],
+node[amenity=fuel],
+node[amenity=studio],
+node[amenity=vehicle_inspection],
+node[craft=beekeeper],
+node[craft=blacksmith],
+node[craft=carpenter],
+node[craft=caterer],
+node[craft=electrician],
+node[craft=electronics_repair],
+node[craft=grinding_mill],
+node[craft=handicraft],
+node[craft=hvac],
+node[craft=key_cutter],
+node[craft=locksmith],
+node[craft=metal_construction],
+node[craft=painter],
+node[craft=plumber],
+node[craft=sawmill],
+node[craft],
+node[landuse=industrial][industrial=mine],
+node[landuse=quarry],
+node[man_made=works],
+node[office],
+node[shop=agrarian],
+node[shop=car_parts],
+node[shop=car_repair],
+node[shop=car_repair][service=tyres],
+node[shop=funeral_directors],
+node[shop=gas],
+node[shop=motorcycle_repair],
+node[shop=tattoo],
+node[shop=trade],
+node[shop=tyres],
+node[shop=wholesale],
+{text-color: @industry_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[amenity=bar],
+node[amenity=biergarten],
+node[amenity=cafe],
+node[amenity=fast_food],
+node[amenity=food_court],
+node[amenity=ice_cream],
+node[amenity=nightclub],
+node[amenity=pub],
+node[amenity=restaurant],
+{text-color: @food_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[amenity=casino],
+node[amenity=cinema],
+node[amenity=gambling],
+node[amenity=stripclub],
+node[amenity=theatre],
+node[leisure=adult_gaming_centre],
+node[leisure=amusement_arcade],
+node[leisure=bowling_alley],
+node[leisure=escape_game],
+node[shop=bookmaker],
+node[shop=lottery],
+node[tourism=aquarium],
+node[tourism=theme_park],
+node[tourism=zoo],
+{text-color: @entertainment_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[amenity=arts_centre],
+node[amenity=community_centre],
+node[amenity=conference_centre],
+node[amenity=events_venue],
+node[amenity=exhibition_centre],
+node[amenity=place_of_worship]
+node[amenity=social_facility],
+node[historic=ship],
+node[leisure=hackerspace],
+node[tourism=attraction],
+node[tourism=gallery],
+node[tourism=museum],
+{text-color: @culture_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[leisure=resort],
+node[tourism=alpine_hut],
+node[tourism=apartment],
+node[tourism=chalet],
+node[tourism=guest_house],
+node[tourism=hostel],
+node[tourism=hotel],
+node[tourism=motel],
+node[tourism=wilderness_hut],
+{text-color: @hotel_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[leisure=dance],
+node[leisure=fitness_centre],
+node[leisure=sports_centre],
+node[leisure=stadium],
+{text-color: @sport_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[landuse=plant_nursery],
+{text-color: @nature_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[amenity=public_bath],
+node[leisure=beach_resort],
+node[leisure=marina],
+node[leisure=sports_centre][sport=scuba_diving],
+node[leisure=sports_centre][sport=swimming],
+node[shop=massage],
+{text-color: @water_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+area[amenity=university][name]
+node[aeroway=gate],
+node[amenity=atm],
+node[amenity=bank],
+node[amenity=bureau_de_change],
+node[amenity=charging_station][motorcar?],
+node[amenity=childcare],
+node[amenity=college],
+node[amenity=courthouse],
+node[amenity=driving_school],
+node[amenity=fire_station],
+node[amenity=kindergarten],
+node[amenity=language_school],
+node[amenity=library],
+node[amenity=money_transfer],
+node[amenity=music_school],
+node[amenity=nursing_home],
+node[amenity=police],
+node[amenity=post_office],
+node[amenity=prison],
+node[amenity=school],
+node[amenity=townhall],
+node[amenity=university][name],
+node[barrier=border_control],
+node[building=guardhouse],
+node[office=security],
+node[shop=money_lender],
+node[tourism=information][information=office],
+{text-color: @neutral_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+area[amenity=hospital],
+node[amenity=clinic],
+node[amenity=dentist],
+node[amenity=doctors],
+node[amenity=hospital],
+node[amenity=pharmacy],
+node[amenity=veterinary],
+node[healthcare],
+{text-color: @healthcare_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}
+
+node[amenity=bicycle_rental],
+node[amenity=taxi],
+{text-color: @public_transport_label;text-halo-radius: 0.1;text-halo-opacity: 0.7;text-halo-color: @label_halo_light;}

data/styles/default/light/colors.mapcss

@@ -229,7 +229,7 @@
 @subway_label: #333333;
 @indigenous_label: #6A512F;
 @shop_label: #8C5678;
-@industry_label: #907E9E;
+@industry_label: #9482A3;
 @food_label: #BA7141;
 @entertainment_label: #D16B53;
 @culture_label: #82512D;

data/styles/default/light/symbols/escape_game.svg

@@ -1,9 +1 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="18px" height="18px" viewBox="0 0 18 18" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <title>escape_game</title>
-    <g id="escape_game" stroke="none" fill="none">
-        <circle id="Oval" fill="#FFFFFF" fill-rule="nonzero" opacity="0.6" cx="9" cy="9" r="9"></circle>
-        <circle id="Oval" fill="#EB785D" fill-rule="nonzero" cx="9" cy="9" r="8.25"></circle>
-        <path d="M6.544,4.525 L6.543,5.199 L7.894,5.2 L7.894,6.55 L6.543,6.55 L6.54384181,9.2696041 C7.56205461,9.56229277 8.307,10.5006507 8.307,11.613 C8.307,12.9594702 7.21547022,14.051 5.869,14.051 C4.52252978,14.051 3.431,12.9594702 3.431,11.613 C3.431,10.5010171 4.17545473,9.56291094 5.19315214,9.26989353 L5.194,4.525 L6.544,4.525 Z M5.869,10.675 C5.3509569,10.675 4.931,11.0949569 4.931,11.613 C4.931,12.1310431 5.3509569,12.551 5.869,12.551 C6.3870431,12.551 6.807,12.1310431 6.807,11.613 C6.807,11.0949569 6.3870431,10.675 5.869,10.675 Z M12.506,11.275 L10.931,11.275 L10.925,10.868 C10.925,10.3653333 11.0083333,9.952 11.175,9.628 C11.341,9.304 11.633,8.893 12.132,8.487 L12.5637778,8.12833333 C12.807037,7.92033333 12.9611111,7.77455556 13.026,7.691 C13.176,7.493 13.251,7.274 13.251,7.034 C13.251,6.70266667 13.117,6.41933333 12.849,6.184 C12.585,5.94466667 12.2283333,5.825 11.779,5.825 C11.3456667,5.825 10.9833333,5.94866667 10.692,6.196 C10.4,6.443 10.246,6.82 10.137,7.326 L8.569,7.132 C8.61366667,6.40666667 8.92133333,5.791 9.492,5.285 C10.068,4.778 10.775,4.525 11.706,4.525 C12.687,4.525 13.421,4.782 13.999,5.297 C14.579,5.807 14.869,6.40266667 14.869,7.084 C14.869,7.46 14.7616667,7.81633333 14.547,8.153 C14.2678649,8.60277864 13.927713,9.01103812 13.5366594,9.36629677 L13.296,9.573 C12.936,9.873 12.711,10.1143333 12.621,10.297 C12.537,10.4783333 12.4986667,10.8043333 12.506,11.275 Z M10.819,13.525 L10.819,11.725 L12.619,11.725 L12.619,13.525 L10.819,13.525 Z" id="Shape" fill="#FFFFFF" fill-rule="evenodd"></path>
-    </g>
-</svg>
+<svg height="18" width="18" xmlns="http://www.w3.org/2000/svg"><circle cx="9" cy="9" fill="#fff" opacity=".6" r="9"/><circle cx="9" cy="9" fill="#EB785D" r="8.25"/><path d="m5.194 4.525h1.35v5.4h-1.35z" fill="#fff" fill-rule="evenodd" stroke-width=".9"/><path d="m5.194 5.2h2.7v1.35h-2.7z" fill="#fff" fill-rule="evenodd" stroke-width=".9"/><circle cx="5.869" cy="11.613" fill="none" r="1.688" stroke="#fff" stroke-width="1.5"/><path d="m12.506 11.275h-1.575l-.006-.407q0-.754.25-1.24c.166-.324.458-.735.957-1.141q.748-.608.894-.796.225-.297.225-.657 0-.497-.402-.85-.396-.359-1.07-.359-.65 0-1.087.371c-.292.247-.446.624-.555 1.13l-1.568-.194q.067-1.088.923-1.847c.576-.507 1.283-.76 2.214-.76.981 0 1.715.257 2.293.772q.87.765.87 1.787 0 .564-.322 1.069a5.7 5.7 0 0 1 -1.251 1.42q-.54.45-.675.724-.126.272-.115.978m-1.687 2.25v-1.8h1.8v1.8z" fill="#fff" fill-rule="evenodd" stroke-width=".9"/></svg>

iphone/Maps/LocalizedStrings/de.lproj/LocalizableTypes.strings

@@ -722,7 +722,7 @@
 "type.leisure.sports_centre.sport.multi" = "Sportzentrum";
 "type.leisure.sports_centre.sport.american_football" = "Sportzentrum";
 "type.leisure.sports_centre.sport.archery" = "Sportzentrum";
-"type.leisure.sports_centre.sport.athletics" = "Sportzentrum";
+"type.leisure.sports_centre.sport.athletics" = "Sportzentrume";
 "type.leisure.sports_centre.sport.australian_football" = "Sportzentrum";
 "type.leisure.sports_centre.sport.badminton" = "Sportzentrum";
 "type.leisure.sports_centre.sport.baseball" = "Sportzentrum";
@@ -1503,7 +1503,7 @@
 "type.highway.ladder" = "Leiter";
 "type.landuse.plant_nursery" = "Gärtnerei";
 "type.leisure.firepit" = "Feuerstelle";
-"type.amenity.studio" = "Medienstudio";
+"type.amenity.studio" = "Studio";
 "type.man_made.crane" = "Kran";
 "type.railway.station.subway.qingdao" = "U-Bahn-Station";
 "type.natural.wetland.saltmarsh" = "Salzmarsch";

tools/python/maps_generator/generator/exceptions.py

@@ -1,5 +1,5 @@
 import os
-
+import subprocess
 
 class MapsGeneratorError(Exception):
     pass
@@ -35,9 +35,24 @@ class FailedTest(MapsGeneratorError):
 
 def wait_and_raise_if_fail(p):
     if p.wait() != os.EX_OK:
-        args = p.args
-        logs = p.output.name
-        if p.error.name != logs:
-            logs += " and " + p.error.name
-        msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}"
-        raise BadExitStatusError(msg)
+        if type(p) is subprocess.Popen:
+            args = p.args
+            stdout = p.stdout
+            stderr = p.stderr
+            logs = None
+            errors = None
+            if type(stdout) is not type(None):
+              logs = stdout.read(256).decode()
+            if type(stderr) is not type(None):
+              errors = stderr.read(256).decode()
+            if errors != logs:
+                logs += " and " + errors
+            msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}"
+            raise BadExitStatusError(msg)
+        else:
+            args = p.args
+            logs = p.output.name
+            if p.error.name != logs:
+                logs += " and " + p.error.name
+            msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}"
+            raise BadExitStatusError(msg)

tools/python/maps_generator/generator/stages_declaration.py

@@ -11,6 +11,7 @@ import multiprocessing
 import os
 import shutil
 import tarfile
+import errno
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import AnyStr
@@ -168,8 +169,18 @@ class StageDownloadDescriptions(Stage):
         )
         """
 
-        src = "/home/planet/descriptions"
+        # The src folder is hardcoded here and must be implemented on the map building machine
+        src = "/home/planet/wikipedia/descriptions"
+        # The dest folder will generally become build/*/intermediate_data/descriptions
         dest = env.paths.descriptions_path
+        # An empty source folder is a big problem
+        try:
+            if os.path.isdir(src):
+                print("Found %s" % (src))
+            else:
+                raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), src)
+        except OSError as e:
+            print("rmtree error: %s - %s" % (e.filename, e.strerror))
         # Empty folder "descriptions" can be already created.
         try:
             if os.path.isdir(dest):

tools/python/maps_generator/generator/stages_tests.py

@@ -9,6 +9,8 @@ from maps_generator.utils.file import download_file
 
 def make_test_booking_data(max_days):
     def test_booking_data(env: Env, logger, *args, **kwargs):
+        if not settings.HOTELS_URL:
+            return None
         base_url, _ = settings.HOTELS_URL.rsplit("/", maxsplit=1)
         url = f"{base_url}/meta.json"
         meta_path = os.path.join(env.paths.tmp_dir(), "hotels-meta.json")

tools/python/maps_generator/generator/steps.py

@@ -37,7 +37,7 @@ def multithread_run_if_one_country(func):
         if len(env.countries) == 1:
             kwargs.update({"threads_count": settings.THREADS_COUNT})
         elif country == 'Taiwan_North':
-            kwargs.update({"threads_count": 4})
+            kwargs.update({"threads_count": 16})
         func(env, country, **kwargs)
 
     return wrap

tools/python/maps_generator/var/etc/map_generator.ini.prod

@@ -0,0 +1,109 @@
+[Developer]
+# Path to the `comaps` source code repository:
+OMIM_PATH: /root/OM/comaps-init
+# A path with the generator_tool binary:
+BUILD_PATH: /root/OM/omim-build-relwithdebinfo
+#${Developer:OMIM_PATH}/../omim-build-release
+
+
+[Main]
+# A special small planet file will be downloaded if DEBUG is set to 1.
+DEBUG: 0
+# A main working directory. There is a subdirectory created for each generator run
+# which contains the planet and other downloads, temporary build files, logs and completed MWMs.
+MAIN_OUT_PATH: /root/OM/osm-maps
+# Path for storing caches for nodes, ways, relations.
+# If it's not set then caches are stored inside the directory of the current build.
+# CACHE_PATH: ${Main:MAIN_OUT_PATH}/cache
+
+
+[Generator tool]
+# Path to the data/ folder in the repository:
+USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data
+# Features stage only parallelism level. Set to 0 for auto detection.
+THREADS_COUNT_FEATURES_STAGE: 48
+# Do not change it. This is determined automatically.
+NODE_STORAGE: mem
+
+
+[Osm tools]
+# Path to osmctools binaries:
+OSM_TOOLS_PATH: /usr/bin/
+#${Developer:OMIM_PATH}/../osmctools
+# If the binaries are not found neither in the configured path nor system-wide,
+# then the tools are built from the sources:
+OSM_TOOLS_SRC_PATH: ${Developer:OMIM_PATH}/tools/osmctools
+
+[Logging]
+# maps_generator's general (python output only) log file path and name.
+# More detailed logs that include output of the `generator_tool` binary
+# are located in the `logs/` subdir of a particular build directory,
+# e.g. `maps_build/2023_06_04__20_05_07/logs/`.
+LOG_FILE_PATH: ${Main:MAIN_OUT_PATH}/generation.log
+
+
+[External]
+# Planet file location. It should be a dump of OSM data in osm.pbf format.
+# By default its an entire planet from "planet.openstreetmap.org".
+# Or set it to a particular country/region extract from e.g. [Geofabrik](http://download.geofabrik.de/index.html).
+# Note that an entire planet generation takes 40+ hours on a 256GB RAM server (and 1TB+ disk space).
+# Stick to smaller extracts unless you have a machine this large.
+# Here and further, its possible to specify either an URL (to be downloaded automatically)
+# or a local file path like file:///path/to/file.
+# A sample URL to download a latest OSM dump for North Macedonia:
+PLANET_URL: file:///home/planet/planet/planet.o5m
+# Location of the md5 checksum of the planet file:
+PLANET_MD5_URL: ${External:PLANET_URL}.md5
+# A base url to the latest_coasts.geom and latest_coasts.rawgeom files.
+# For example, if PLANET_COASTS_URL = https://somesite.com/download/
+# then the https://somesite.com/download/latest_coasts.geom url will be used to download latest_coasts.geom and
+# the https://somesite.com/download/latest_coasts.rawgeom url will be used to download latest_coasts.rawgeom.
+# Comment to skip getting the coastlines files.
+PLANET_COASTS_URL: file:///home/planet/
+# Should be 'true' for an entire planet build to make a special routing section in World.mwm
+# for alerting about absent regions without which the route can't be built.
+# NEED_BUILD_WORLD_ROADS: true
+# Subway file location, see docs/SUBWAY_GENERATION.md if you want to generate your own file.
+# Comment to disable subway layer generation.
+
+SUBWAY_URL: file:///home/planet/subway/subways.transit.json
+
+# Location of the EXPERIMENTAL GTFS-extracted public transport transit files:
+# TRANSIT_URL:
+
+# Urls for production maps generation.
+# UGC_URL:
+# HOTELS_URL:
+# PROMO_CATALOG_CITIES:
+# POPULARITY_URL:
+# FOOD_URL:
+# FOOD_TRANSLATIONS_URL:
+
+SRTM_PATH: /home/planet/SRTM-patched-europe/
+ISOLINES_PATH: /home/planet/isolines/
+ADDRESSES_PATH: /home/planet/tiger/
+
+# Local path (not url!) to .csv files.
+UK_POSTCODES_URL: /home/planet/postcodes/gb-postcode-data/gb_postcodes.csv
+US_POSTCODES_URL: /home/planet/postcodes/us-postcodes/uszips.csv
+
+
+[Stages]
+# Set to 1 to update the entire OSM planet file (as taken from "planet.openstreetmap.org")
+# via an osmupdate tool before the generation. Not for use with partial planet extracts.
+NEED_PLANET_UPDATE: 0
+# If you want to calculate diffs you need to specify where the old maps are,
+# e.g. ${Main:MAIN_OUT_PATH}/2021_03_16__09_00_00/
+DATA_ARCHIVE_DIR: ${Generator tool:USER_RESOURCE_PATH}
+# How many versions in the archive to use for diff calculation:
+DIFF_VERSION_DEPTH: 2
+
+
+[Common]
+# Default parallelism level for the most of jobs. Set to 0 for auto detection.
+THREADS_COUNT: 0
+
+
+[Stats]
+# Path to rules for calculating statistics by type:
+STATS_TYPES_CONFIG: ${Developer:OMIM_PATH}/tools/python/maps_generator/var/etc/stats_types_config.txt

tools/unix/Dockerfile

@@ -0,0 +1,73 @@
+# See run-docker.sh for cloning, building, and running the maps generator Docker routine
+FROM debian:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC 
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cargo \
+    clang \
+    cmake \
+    curl \
+    git \
+    jq \
+    libc++-dev \
+    libfreetype-dev \
+    libgeos-dev \
+    libgl1-mesa-dev \
+    libglvnd-dev \
+    libharfbuzz-dev \
+    libicu-dev \
+    libqt6positioning6 \
+    libqt6positioning6-plugins \
+    libqt6svg6-dev \
+    libsqlite3-dev \
+    libxcursor-dev \
+    libxi-dev \
+    libxinerama-dev \
+    libxrandr-dev \
+    ninja-build \
+    openssh-client \
+    osmctools \
+    osmium-tool \
+    pyosmium \
+    python3 \
+    python3-pip \
+    python3-venv \
+    python-is-python3 \
+    python3-dev \
+    qt6-base-dev \
+    qt6-positioning-dev \
+    rclone \
+    rustc \
+    sshpass \
+    vim \
+    wget \
+    wget2 \
+    zlib1g-dev \
+    && rm -rf /var/cache/apt/* /var/lib/apt/lists/*;
+
+# Allow pip to install system-wide in the container
+RUN rm -f /usr/lib/python*/EXTERNALLY-MANAGED
+RUN pip3 install "protobuf<4"
+
+RUN mkdir /root/OM
+# When running, bind mount the repo here
+RUN mkdir /root/OM/comaps-init
+# And a volume to store the large output/temp files here
+RUN mkdir /root/OM/osm-maps
+# And the output of the app build
+RUN mkdir /root/OM/omim-build-release
+RUN mkdir /root/OM/omim-build-relwithdebinfo
+# And a volume to store >10gb files for the planet here
+RUN mkdir /home/planet
+
+WORKDIR /root/OM/comaps-init
+
+# For debugging
+#CMD /bin/bash 
+
+CMD /root/OM/comaps-init/tools/unix/docker_maps_generator.sh

tools/unix/docker_maps_generator.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+set -e
+
+#Volumes/paths for downloads:
+#home/planet/planet/planet.o5m
+#home/planet/planet/planet.o5m.md5
+#PLANET_COASTS_URL:file:///home/planet/planet/
+  #home/planet/planet/latest_coasts.geom and latest_coasts.rawgeom
+#SUBWAY_URL: file:///home/planet/subway/beta.json
+  #home/planet/subway/beta.json
+#HOTELS_URL:/home/planet/planet/kayak/
+  #home/planet/planet/kayak/
+#SRTM_PATH:/home/planet/SRTM-patched-europe/
+#ISOLINES_PATH:/home/planet/planet/isolines/
+#ADDRESSES_PATH:/home/planet/planet/tiger/
+#UK_POSTCODES_URL:/home/planet/postcodes/gb-postcode-data/gb_postcodes.csv
+#US_POSTCODES_URL:/home/planet/postcodes/us-postcodes/uszips.csv
+
+echo "<$(date +%T)> Setting git as safe dir..."
+
+git config --global --add safe.directory /root/OM/comaps-init
+
+echo "<$(date +%T)> Starting..."
+
+# Prepare paths
+#
+# Already created by Dockerfile or CI/CD:
+#   /root/OM
+#   /root/OM/comaps-init
+#   /root/OM/omim-build-release
+#   /root/OM/omim-build-relwithdebinfo
+#   /root/OM/osm-maps
+#   /home/planet
+#
+mkdir -p /root/.config/CoMaps # Odd mkdir permission errors in generator_tool in Docker without these
+chmod -R 777 /root/.config
+mkdir -p ~/.config/rclone
+mkdir -p /home/planet/planet/isolines/
+mkdir -p /home/planet/planet/kayak/
+mkdir -p /home/planet/planet/tiger/
+mkdir -p /home/planet/postcodes/gb-postcode-data/
+mkdir -p /home/planet/postcodes/us-postcodes/
+mkdir -p /home/planet/SRTM-patched-europe/
+mkdir -p /home/planet/subway
+
+# echo "Writing rclone config..."
+# echo "[r2]" > ~/.config/rclone/rclone.conf
+# echo "type = s3" >> ~/.config/rclone/rclone.conf
+# echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf
+# echo "access_key_id = $S3_KEY_ID" >> ~/.config/rclone/rclone.conf
+# echo "secret_access_key = $S3_SECRET_KEY" >> ~/.config/rclone/rclone.conf
+# echo "region = auto" >> ~/.config/rclone/rclone.conf
+# echo "endpoint = $S3_ENDPOINT" >> ~/.config/rclone/rclone.conf
+# # S3_BUCKET is used below during uploading
+
+echo "<$(date +%T)> Running ./configure.sh ..."
+cd /root/OM/comaps-init
+./configure.sh --skip-map-download --skip-generate-symbols
+
+echo "<$(date +%T)> Compiling tools..."
+cd /root/OM/comaps-init
+./tools/unix/build_omim.sh -R generator_tool
+./tools/unix/build_omim.sh -R world_roads_builder_tool
+./tools/unix/build_omim.sh -R mwm_diff_tool
+cd tools/python/maps_generator
+python3 -m venv /tmp/venv
+/tmp/venv/bin/pip3 install -r requirements_dev.txt
+
+echo "<$(date +%T)> Copying map generator INI..."
+cp var/etc/map_generator.ini.prod var/etc/map_generator.ini
+
+# May be unnecessary when running world
+# /tmp/venv/bin/python -m maps_generator --coasts
+# save to /path/to/coasts WorldCoasts.geom as latest_coasts.geom and WorldCoasts.rawgeom latest_coasts.rawgeom
+# (from https://github.com/mapsme/omim/issues/11994)
+
+cd /home/planet/planet
+if [ ! -f planet-latest.osm.pbf ]; then
+  echo "<$(date +%T)> Downloading planet-latest.osm.pbf..."
+  curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf
+  echo "<$(date +%T)> Downloading planet-latest.osm.pbf.md5..."
+  curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf.md5
+else
+  echo "<$(date +%T)> planet-latest.osm.pbf exists, not downloading..."
+fi
+
+#curl -OL https://download.geofabrik.de/north-america/us-west-latest.osm.pbf
+#curl -OL https://download.geofabrik.de/north-america/us-west-latest.osm.pbf.md5
+# (rename us-west-latest to planet-latest and edit the md5 file accordingly)
+if [ ! -f planet.o5m ]; then
+  echo "<$(date +%T)> Converting planet-latest.osm.pbf to planet.o5m..."
+  osmconvert planet-latest.osm.pbf -o=planet.o5m
+else
+  echo "<$(date +%T)> planet.o5m exists, not converting..."
+fi
+# (currently unused:) /root/OM/comaps-init/tools/unix/update_planet.sh planet.o5m
+
+
+echo "<$(date +%T)> Generating maps..."
+cd /root/OM/comaps-init/tools/python
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs"
+#/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" --continue
+
+# do not use --production except for Kayak/recommendation/popularity/food data
+#/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production
+#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs"
+#/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs"
+
+echo "<$(date +%T)> DONE"
+

tools/unix/generate_drules.sh

@@ -22,14 +22,18 @@ function BuildDrawingRules() {
     -o "$DATA_PATH/drules_proto$suffix" \
     -p "$DATA_PATH/styles/$styleType/include/"
   # Output diff and store to a file
-  diff -u "$DATA_PATH"/drules_proto$suffix.txt{.prev,} | tee "$DATA_PATH"/drules_proto$suffix.txt.diff
+  if [ -f "$DATA_PATH/drules_proto$suffix.txt.prev" ]; then
+    diff -u "$DATA_PATH/drules_proto$suffix.txt.prev" "$DATA_PATH/drules_proto$suffix.txt" | tee "$DATA_PATH/drules_proto$suffix.txt.diff" || true
+  fi
 }
 
 outputs=(classificator.txt types.txt visibility.txt colors.txt patterns.txt drules_proto.txt)
 # Store old versions for diffs
 for item in ${outputs[*]}
 do
-  mv $DATA_PATH/$item{,.prev} || true
+  if [ -f "$DATA_PATH/$item" ]; then
+    mv -f "$DATA_PATH/$item" "$DATA_PATH/$item.prev"
+  fi
 done
 
 # Building drawing rules
@@ -63,7 +67,11 @@ python3 "$OMIM_PATH/tools/python/stylesheet/drules_merge.py" \
 # Output diffs and store to files
 for item in ${outputs[*]}
 do
-  diff -u "$DATA_PATH"/$item{.prev,} | tee "$DATA_PATH"/$item.diff
+  if [ -f "$DATA_PATH/$item.prev" ] && [ -f "$DATA_PATH/$item" ]; then
+    diff -u "$DATA_PATH/$item.prev" "$DATA_PATH/$item" | tee "$DATA_PATH/$item.diff" || true
+  else
+    echo "Skipping diff for $item (first run or file missing)"
+  fi
 done
 
 echo "Diffs for all changes are stored in $DATA_PATH/*.txt.diff"

tools/unix/maps/generate_subways.sh

@@ -20,9 +20,9 @@ export SKIP_PLANET_UPDATE="1"
 # If unavailable then replace with a local file.
 # TODO: keep the downloaded csv file from the latest run.
 #export CITIES_INFO_URL=""
-export TMPDIR="$BUILD_PATH/subways"
+export TMPDIR="$BUILD_PATH/subway"
 # The output file, which needs post-processing by transit_graph_generator.py
-export MAPSME="$SUBWAYS_PATH/subways.json"
+export MAPSME="$SUBWAYS_PATH/subway.json"
 
 # Produce additional files needed for https://cdn.organicmaps.app/subway/
 export HTML_DIR="$SUBWAYS_VALIDATOR_PATH"
@@ -30,7 +30,12 @@ export DUMP="$SUBWAYS_VALIDATOR_PATH"
 export GEOJSON="$SUBWAYS_VALIDATOR_PATH"
 export DUMP_CITY_LIST="$SUBWAYS_VALIDATOR_PATH/cities.txt"
 
-"$SUBWAYS_REPO_PATH/scripts/process_subways.sh" 2>&1 | tee "$SUBWAYS_LOG"
+# cd to subways repo so relative paths work in the script
+pushd "$SUBWAYS_REPO_PATH"
+echo "Running process_subways.sh:"
+ls -al
+./scripts/process_subways.sh # 2>&1 | tee "$SUBWAYS_LOG"
+popd
 
 # Make render.html available for map visualization on the web
 cp -r "$SUBWAYS_REPO_PATH"/render/* "$SUBWAYS_VALIDATOR_PATH/"
@@ -39,7 +44,7 @@ TRANSIT_TOOL_PATH="$REPO_PATH/tools/python/transit"
 SUBWAYS_GRAPH_FILE="$SUBWAYS_PATH/subways.transit.json"
 
 activate_venv_at_path "$TRANSIT_TOOL_PATH"
-"$PYTHON" "$TRANSIT_TOOL_PATH/transit_graph_generator.py" "$MAPSME" "$SUBWAYS_GRAPH_FILE" 2>&1 | tee -a "$SUBWAYS_LOG"
+"$PYTHON" "$TRANSIT_TOOL_PATH/transit_graph_generator.py" "$MAPSME" "$SUBWAYS_GRAPH_FILE" # 2>&1 | tee -a "$SUBWAYS_LOG"
 deactivate
 
 echo "Generated subways transit graph file:"

tools/unix/maps/settings.sh.prod

@@ -0,0 +1,25 @@
+## NOTE: edit the settings.sh file to customize/override the defaults.
+
+# Absolutize & normalize paths.
+REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}"
+
+BASE_PATH="${BASE_PATH:-$REPO_PATH/..}"
+# Source map data and processed outputs e.g. wiki articles
+DATA_PATH="${DATA_PATH:-/home/planet}"
+# Temporary files
+BUILD_PATH="${BUILD_PATH:-$DATA_PATH/build}"
+# Other code repositories, e.g. subways, wikiparser..
+CODE_PATH="${CODE_PATH:-$REPO_PATH/..}"
+
+# OSM planet source files
+
+PLANET_PATH="${PLANET_PATH:-$DATA_PATH/planet}"
+PLANET_PBF="${PLANET_PBF:-$PLANET_PATH/planet-latest.osm.pbf}"
+PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
+
+# Subways
+
+SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-$CODE_PATH/subways}"
+SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}"
+SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}"
+SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"

tools/unix/maps/settings_default.sh

@@ -10,16 +10,3 @@ BUILD_PATH="${BUILD_PATH:-$BASE_PATH/build}"
 CODE_PATH="${CODE_PATH:-$REPO_PATH/..}"
 # Source map data and processed outputs e.g. wiki articles
 DATA_PATH="${DATA_PATH:-$BASE_PATH/data}"
-
-# OSM planet source files
-
-PLANET_PATH="${PLANET_PATH:-$DATA_PATH/planet}"
-PLANET_PBF="${PLANET_PBF:-$PLANET_PATH/planet-latest.osm.pbf}"
-PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
-
-# Subways
-
-SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-$CODE_PATH/subways}"
-SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subways}"
-SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subways.log}"
-SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"

tools/unix/maps/upload_to_cdn.sh

@@ -2,6 +2,8 @@
 
 # Upload new maps version to all CDN nodes (in parallel).
 
+# TODO: implement removing old versions
+
 # Use following commands for deleting older maps:
 #
 # ru1 - keep max 3 maps versions
@@ -39,6 +41,44 @@ DIR=$(dirname $1)/$MAPS
 
 echo "Uploading maps folder $DIR to $MAPS"
 
+# Remove old versions before uploading new ones
+echo "Checking for old versions to remove..."
+
+# ru1 - keep max 3 versions
+echo "Cleaning ru1 (keeping 3 newest versions)..."
+OLD_VERSIONS_RU1=$(rclone lsd ru1:comaps-maps/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_RU1; do
+  if [ -n "$version" ]; then
+    echo "  Deleting ru1:comaps-maps/maps/$version"
+    rclone purge -v ru1:comaps-maps/maps/$version
+  fi
+done
+
+# fi1 - keep max 3 versions
+echo "Cleaning fi1 (keeping 3 newest versions)..."
+OLD_VERSIONS_FI1=$(rclone lsd fi1:/var/www/html/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_FI1; do
+  if [ -n "$version" ]; then
+    echo "  Deleting fi1:/var/www/html/maps/$version"
+    rclone purge -v fi1:/var/www/html/maps/$version
+  fi
+done
+
+# de1 - keep max 6 versions
+echo "Cleaning de1 (keeping 6 newest versions)..."
+OLD_VERSIONS_DE1=$(rclone lsd de1:/var/www/html/comaps-cdn/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_DE1; do
+  if [ -n "$version" ]; then
+    echo "  Deleting de1:/var/www/html/comaps-cdn/maps/$version"
+    rclone purge -v de1:/var/www/html/comaps-cdn/maps/$version
+  fi
+done
+
+# us2 - keep all versions (no cleanup)
+echo "Skipping us2 cleanup (keeping all versions)"
+
+echo "Old version cleanup complete"
+
 echo "Uploading to us2"
 # An explicit mwm/txt filter is used to skip temp files when run for an unfinished generation
 rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS &
@@ -57,4 +97,18 @@ rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$M
 
 wait
 
+echo "Running once more without parallelization to output status:"
+
+echo "us2 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS
+
+echo "ru1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS
+
+echo "fi1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS
+
+echo "de1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS
+
 echo "Upload complete"

tools/unix/run-docker.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Master file documenting how to clone, build, and run the maps generator via Docker
+
+# Prerequisutes:
+# sudo apt install docker git
+
+# To bootstrap the repo:
+#TODO: rename comaps-init to comaps here and throughout
+#cd /mnt/4tbexternal
+#if [ ! -f /mnt/4tbexternal/comaps-init ]; then
+#  git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps-init.git
+#  cd /mnt/4tbexternal/comaps-init
+#else
+#  cd /mnt/4tbexternal/comaps-init
+#  git pull origin
+#fi
+
+# And data:
+# cd /mnt/4tbexternal/comaps-init/data/
+# wget World.mwm //pastk - not needed
+# wget WorldCoasts.mwm
+
+#TODO: isolines, postcodes, subways, wiki
+
+# In tools/unix/maps, copy settings.sh.dist to settings.sh and modify if needed
+# In tools/python/maps_generator/var/etc, copy map_generator.ini.prod to map_generator.ini and modify if needed
+
+#cd /mnt/4tbexternal/comaps-init/tools/unix
+# Build with: docker build . -t maps_generator
+# (Good to rebuild each time just in case)
+# To push for ci/cd, tag for codeberg:
+#   docker login codeberg.org
+#   docker tag maps_generator codeberg.org/comaps/maps_generator:latest
+#   docker push codeberg.org/comaps/maps_generator:latest
+# You can also tag and push the image Id for posterity: codeberg.org/comaps/maps_generator:1234abcd
+# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first.
+# Or use docker via apt instead of snap.
+#
+# Edit as appropriate and run with:
+# docker run \
+#   -e S3_KEY_ID=changeme \
+#   -e S3_SECRET_KEY=changeme \
+#   -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \
+#   -e S3_BUCKET=comaps-map-files \
+#   -e SFTP_USER=changeme \
+#   -e SFTP_PASSWORD=changeme \
+#   -e SFTP_HOST=changeme.dreamhost.com \
+#   -e SFTP_PATH=cdn-us-1.comaps.app \
+#   --ulimit nofile=262144:262144 \
+#   -v /mnt/4tbexternal/comaps-init:/root/OM/comaps-init \
+#   -v /mnt/4tbexternal/osm-planet:/home/planet \
+#   -v /mnt/4tbexternal/osm-maps:/root/OM/osm-maps \
+#   -it maps_generator \
+#   /root/OM/comaps-init/tools/unix/docker_maps_generator.sh
+ 
+docker run \
+  -e S3_KEY_ID=changeme \
+  --ulimit nofile=262144:262144 \
+  -v /mnt/4tbexternal/comaps-init:/root/OM/comaps-init \
+  -v /mnt/4tbexternal/wikiparser:/root/OM/wikiparser \
+  -v /mnt/4tbexternal/osm-planet:/home/planet \
+  -v /mnt/4tbexternal/osm-maps:/root/OM/osm-maps \
+  -v /mnt/4tbexternal/subways:/root/OM/subways \
+  -v /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo \
+  -it codeberg.org/comaps/maps_generator:latest \
+  /root/OM/comaps-init/tools/unix/docker_maps_generator.sh

tools/unix/update_planet.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+OSMUPDATE=~/osmctools/osmupdate
+# osmconvert should be accessible in PATH.
+PATH="$(dirname "$OSMUPDATE"):$PATH"
+
+# Pass pbf or o5m file as a parameter
+OLD="$1"
+NEW="${1/.pbf/.new.pbf}"
+NEW="${NEW/.o5m/.new.o5m}"
+
+"$OSMUPDATE" -v --drop-authors --drop-version --hash-memory=512000 "$OLD" "$NEW"
+# Uncomment to replace old planet.
+mv "$NEW" "$OLD"
+#md5sum -b "$OLD" > "$OLD.md5"
+echo "Successfully updated $OLD"