[generator] Add chekboxes for all mapgen steps; separate pbf and o5m planet updates

Signed-off-by: Konstantin Pastbin <konstantin.pastbin@gmail.com>

.forgejo/workflows/map-generator.yml

@@ -2,26 +2,51 @@ name: map-generator
 on:
   workflow_dispatch: # Manual trigger
     inputs:
-      jobs:
-        description: 'Which job(s) to run right now?'
-        required: true
-        default: 'all-except-upload'
-        type: choice
-        options:
-          - all-except-upload
-          - copy-coasts
-          - planet
-          - wiki
-          - isolines
-          - subways
-          - tiger
-          - maps
-          - upload
       map-generator-test:
         description: 'Test (non-prod) generation?'
         required: false
         default: false
         type: boolean
+      run-copy-coasts:
+        description: 'Copy last used coastlines?'
+        required: false
+        default: true
+        type: boolean
+      run-isolines:
+        description: 'Update altitude isolines?'
+        required: false
+        default: false
+        type: boolean
+      run-tiger:
+        description: 'Update TIGER address data?'
+        required: false
+        default: true
+        type: boolean
+      run-planet-pbf:
+        description: 'Update PBF planet (for Wiki & subways)?'
+        required: false
+        default: true
+        type: boolean
+      run-subways:
+        description: 'Update subways?'
+        required: false
+        default: true
+        type: boolean
+      run-wiki:
+        description: 'Update Wikipedia descriptions?'
+        required: false
+        default: true
+        type: boolean
+      run-planet-o5m:
+        description: 'Update O5M planet (for mapgen)?'
+        required: false
+        default: true
+        type: boolean
+      run-mapgen:
+        description: 'Run maps generation?'
+        required: false
+        default: true
+        type: boolean
       map-generator-continue:
         description: 'Continue previous map generation?'
         required: false
@@ -31,6 +56,11 @@ on:
         description: 'Generate specific MWMs? (i.e. "US_New York_*, foo")'
         required: false
         type: string
+      run-upload:
+        description: 'Upload latest maps to CDN?'
+        required: false
+        default: false
+        type: boolean
       reset:
         description: 'Reset part of the system?'
         required: false
@@ -88,11 +118,9 @@ jobs:
           git clone https://codeberg.org/comaps/subways.git
 
   copy-coasts:
-    if: inputs.jobs == 'copy-coasts' || inputs.jobs == 'all-except-upload'
+    if: inputs.run-copy-coasts
     name: Copy Previously Generated Coasts
     runs-on: mapfilemaker
-    needs:
-      - clone-repos
     container:
       image: codeberg.org/comaps/maps_generator:f6d53d54f794
       volumes:
@@ -112,7 +140,7 @@ jobs:
 
           echo "Before:"
           ls -al /home/planet/latest_coasts*
-
+          # TODO: don't copy coasts from test generations
           cp -p /mnt/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.geom /home/planet/latest_coasts.geom
           cp -p /mnt/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.rawgeom /home/planet/latest_coasts.rawgeom
 
@@ -125,12 +153,103 @@ jobs:
 
           fi
 
-  update-planet:
+  update-isolines:
-    if: inputs.jobs == 'planet' || inputs.jobs == 'all-except-upload'
+    if: inputs.run-isolines
-    name: Update Planet
+    name: Update Isolines
     runs-on: mapfilemaker
     needs:
       - clone-repos
+    container:
+      image: codeberg.org/comaps/maps_generator:f6d53d54f794
+      volumes:
+        - /mnt/4tbexternal/:/mnt/4tbexternal/
+        - /mnt/4tbexternal/osm-planet:/home/planet
+    concurrency:
+      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/cache@v4
+        with:
+          path: "~"
+          key: cache-${{ github.run_id }}-${{ github.run_attempt }}
+      # TODO: we only need to update these if our SRTM or countries change
+      # TODO: after update, verify that sizable files exist: /home/planet/isolines/*.isolines
+      - name: Update Isolines
+        shell: bash
+        # TODO: preserve previous isolines version?
+        # TODO: cleanup the tmp-tiles dir after completion
+        run: |
+          cd ~/comaps/
+          ./tools/unix/build_omim.sh -p ~ -R topography_generator_tool
+          rm -rf /home/planet/isolines/
+          mkdir /home/planet/isolines/
+          ~/omim-build-relwithdebinfo/topography_generator_tool \
+          --profiles_path=./data/conf/isolines/isolines-profiles.json \
+          --countries_to_generate_path=./data/conf/isolines/countries-to-generate.json \
+          --tiles_isolines_out_dir=/home/planet/isolines/tmp-tiles/ \
+          --countries_isolines_out_dir=/home/planet/isolines/ \
+          --data_dir=./data/ \
+          --srtm_path=/home/planet/SRTM-patched-europe/ \
+          --threads=96
+      - name: Check isolines
+        shell: bash
+        run: |
+          NUMISO=$(ls -al /home/planet/isolines/*.isolines | wc -l)
+          echo "Found $NUMISO isolines"
+          if [ $NUMISO -lt 10 ]; then
+          echo "ERROR: Did generation fail?"
+          exit 1
+          fi
+      - name: Notify Zulip
+        run: |
+          curl -X POST https://comaps.zulipchat.com/api/v1/messages \
+            -u $ZULIP_BOT_EMAIL:$ZULIP_API_KEY \
+            --data-urlencode type=stream \
+            --data-urlencode 'to="DevOps"' \
+            --data-urlencode topic=codeberg-bot \
+            --data-urlencode 'content=Isolines are done!'
+
+  update-tiger:
+    if: inputs.run-tiger
+    name: Update TIGER
+    runs-on: mapfilemaker
+    needs:
+      - clone-repos
+    container:
+      image: codeberg.org/comaps/maps_generator:f6d53d54f794
+      volumes:
+        - /mnt/4tbexternal/:/mnt/4tbexternal/
+        - /mnt/4tbexternal/osm-planet:/home/planet
+    concurrency:
+      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/cache@v4
+        with:
+          path: "~"
+          key: cache-${{ github.run_id }}-${{ github.run_attempt }}
+      - name: Build address_parser
+        shell: bash
+        run: |
+          cd ~/comaps
+          #rm -rf ~/omim-build-relwithdebinfo/CMakeCache.txt
+          #rm -rf ~/omim-build-relwithdebinfo/CMakeFiles
+          ./tools/unix/build_omim.sh -p ~ -R address_parser_tool
+      - name: Update TIGER from Nominatim
+        shell: bash
+        # TODO: use curl instead of wget2
+        run: |
+          # TODO: maybe remove old osm-planet/tiger first?
+          cd /home/planet/
+          mkdir -p tiger
+          wget2 https://nominatim.org/data/tiger-nominatim-preprocessed-latest.csv.tar.gz
+          cd ~/comaps
+          tar -xOzf /home/planet/tiger-nominatim-preprocessed-latest.csv.tar.gz | ~/omim-build-relwithdebinfo/address_parser_tool --output_path=/home/planet/tiger
+
+  update-planet-pbf:
+    if: inputs.run-planet-pbf
+    name: Update PBF Planet
+    runs-on: mapfilemaker
     container:
       image: codeberg.org/comaps/maps_generator:f6d53d54f794
       volumes:
@@ -153,20 +272,13 @@ jobs:
           else
           echo "planet-latest.osm.pbf was found, raw download not required."
           fi
-      - name: Update Planet
+      - name: Update PBF Planet
         shell: bash
         run: |
           cd /home/planet/planet/
           rm -f planet-latest-new.osm.pbf
           pyosmium-up-to-date planet-latest.osm.pbf -o planet-latest-new.osm.pbf -v --size 16384
           mv planet-latest-new.osm.pbf planet-latest.osm.pbf
-      - name: Converting planet-latest.osm.pbf to planet.o5m
-        # TODO: better to run osmupdate (not convert) just before starting the maps jobs - for max fresh data.
-        run: |
-          echo "Starting..."
-          cd /home/planet/planet/
-          osmconvert -v --drop-author --drop-version --hash-memory=4000 planet-latest.osm.pbf -o=planet.o5m
-          echo "Done."
       - name: Notify Zulip
         run: |
           curl -X POST https://comaps.zulipchat.com/api/v1/messages \
@@ -174,10 +286,44 @@ jobs:
             --data-urlencode type=stream \
             --data-urlencode 'to="DevOps"' \
             --data-urlencode topic=codeberg-bot \
-            --data-urlencode 'content=Planet update is done!'
+            --data-urlencode 'content=PBF planet update is done!'
+
+  update-subways:
+    if: inputs.run-subways
+    name: Update Subways
+    runs-on: mapfilemaker
+    needs:
+      - clone-repos
+    container:
+      image: codeberg.org/comaps/maps_generator:f6d53d54f794
+      volumes:
+        - /mnt/4tbexternal/:/mnt/4tbexternal/
+        - /mnt/4tbexternal/osm-planet:/home/planet
+    concurrency:
+      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/cache@v4
+        with:
+          path: "~"
+          key: cache-${{ github.run_id }}-${{ github.run_attempt }}
+      - name: Update Subways
+        shell: bash
+        run: |
+          cd ~/comaps/
+          cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh
+          ./tools/unix/maps/generate_subways.sh
+      - name: Notify Zulip
+        run: |
+          curl -X POST https://comaps.zulipchat.com/api/v1/messages \
+            -u $ZULIP_BOT_EMAIL:$ZULIP_API_KEY \
+            --data-urlencode type=stream \
+            --data-urlencode 'to="DevOps"' \
+            --data-urlencode topic=codeberg-bot \
+            --data-urlencode 'content=Subways are done!'
 
   wiki-update:
-    if: inputs.jobs == 'wiki' || inputs.jobs == 'all-except-upload'
+    if: inputs.run-wiki
     name: Update Wikipedia
     runs-on: mapfilemaker
     needs:
@@ -284,12 +430,10 @@ jobs:
             --data-urlencode topic=codeberg-bot \
             --data-urlencode 'content=Wiki update is done!'
 
-  update-isolines:
+  update-planet-o5m:
-    if: inputs.jobs == 'isolines' || inputs.jobs == 'all-except-upload'
+    if: inputs.run-planet-o5m
-    name: Update Isolines
+    name: Update O5M Planet
     runs-on: mapfilemaker
-    needs:
-      - clone-repos
     container:
       image: codeberg.org/comaps/maps_generator:f6d53d54f794
       volumes:
@@ -299,38 +443,32 @@ jobs:
       group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
-      - uses: actions/cache@v4
+      - name: Check for O5M Planet File
-        with:
-          path: "~"
-          key: cache-${{ github.run_id }}-${{ github.run_attempt }}
-      # TODO: we only need to update these if our SRTM or countries change
-      # TODO: after update, verify that sizable files exist: /home/planet/isolines/*.isolines
-      - name: Update Isolines
-        shell: bash
-        # TODO: preserve previous isolines version?
-        # TODO: cleanup the tmp-tiles dir after completion
-        run: |
-          cd ~/comaps/
-          ./tools/unix/build_omim.sh -p ~ -R topography_generator_tool
-          rm -rf /home/planet/isolines/
-          mkdir /home/planet/isolines/
-          ~/omim-build-relwithdebinfo/topography_generator_tool \
-          --profiles_path=./data/conf/isolines/isolines-profiles.json \
-          --countries_to_generate_path=./data/conf/isolines/countries-to-generate.json \
-          --tiles_isolines_out_dir=/home/planet/isolines/tmp-tiles/ \
-          --countries_isolines_out_dir=/home/planet/isolines/ \
-          --data_dir=./data/ \
-          --srtm_path=/home/planet/SRTM-patched-europe/ \
-          --threads=96
-      - name: Check isolines
         shell: bash
         run: |
-          NUMISO=$(ls -al /home/planet/isolines/*.isolines | wc -l)
+          if [ ! -f /home/planet/planet/planet.o5m ]; then
-          echo "Found $NUMISO isolines"
+          echo "WARN: No file at /home/planet/planet/planet.o5m"
-          if [ $NUMISO -lt 10 ]; then
+
-          echo "ERROR: Did generation fail?"
+          if [ ! -f /home/planet/planet/planet-latest.osm.pbf ]; then
+          echo "ERROR: No file at /home/planet/planet/planet-latest.osm.pbf"
+          ls -al /home/planet/
+          ls -al /home/planet/planet/
           exit 1
           fi
+
+          echo "Converting planet-latest.osm.pbf to planet.o5m"
+          cd /home/planet/planet/
+          osmconvert -v --drop-author --drop-version --hash-memory=4000 planet-latest.osm.pbf -o=planet.o5m
+          echo "Conversion is done."
+          fi
+      - name: Update O5M planet
+        run: |
+          echo "Starting..."
+          cd /home/planet/planet/
+          rm -f planet-new.o5m
+          osmupdate -v --drop-author --drop-version --hash-memory=4000  --max-merge=32 --out-o5m planet.o5m planet-new.o5m
+          mv planet-new.o5m planet.o5m
+          echo "Done."
       - name: Notify Zulip
         run: |
           curl -X POST https://comaps.zulipchat.com/api/v1/messages \
@@ -338,81 +476,10 @@ jobs:
             --data-urlencode type=stream \
             --data-urlencode 'to="DevOps"' \
             --data-urlencode topic=codeberg-bot \
-            --data-urlencode 'content=Isolines are done!'
+            --data-urlencode 'content=O5M planet update is done!'
-
-  update-subways:
-    if: inputs.jobs == 'subways' || inputs.jobs == 'all-except-upload'
-    name: Update Subways
-    runs-on: mapfilemaker
-    needs:
-      - clone-repos
-    container:
-      image: codeberg.org/comaps/maps_generator:f6d53d54f794
-      volumes:
-        - /mnt/4tbexternal/:/mnt/4tbexternal/
-        - /mnt/4tbexternal/osm-planet:/home/planet
-    concurrency:
-      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
-      cancel-in-progress: true
-    steps:
-      - uses: actions/cache@v4
-        with:
-          path: "~"
-          key: cache-${{ github.run_id }}-${{ github.run_attempt }}
-      - name: Update Subways
-        shell: bash
-        run: |
-          cd ~/comaps/
-          cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh
-          ./tools/unix/maps/generate_subways.sh
-      - name: Notify Zulip
-        run: |
-          curl -X POST https://comaps.zulipchat.com/api/v1/messages \
-            -u $ZULIP_BOT_EMAIL:$ZULIP_API_KEY \
-            --data-urlencode type=stream \
-            --data-urlencode 'to="DevOps"' \
-            --data-urlencode topic=codeberg-bot \
-            --data-urlencode 'content=Subways are done!'
-
-  update-tiger:
-    if: inputs.jobs == 'tiger' || inputs.jobs == 'all-except-upload'
-    name: Update TIGER
-    runs-on: mapfilemaker
-    needs:
-      - clone-repos
-    container:
-      image: codeberg.org/comaps/maps_generator:f6d53d54f794
-      volumes:
-        - /mnt/4tbexternal/:/mnt/4tbexternal/
-        - /mnt/4tbexternal/osm-planet:/home/planet
-    concurrency:
-      group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
-      cancel-in-progress: true
-    steps:
-      - uses: actions/cache@v4
-        with:
-          path: "~"
-          key: cache-${{ github.run_id }}-${{ github.run_attempt }}
-      - name: Build address_parser
-        shell: bash
-        run: |
-          cd ~/comaps
-          #rm -rf ~/omim-build-relwithdebinfo/CMakeCache.txt
-          #rm -rf ~/omim-build-relwithdebinfo/CMakeFiles
-          ./tools/unix/build_omim.sh -p ~ -R address_parser_tool
-      - name: Update TIGER from Nominatim
-        shell: bash
-        # TODO: use curl instead of wget2
-        run: |
-          # TODO: maybe remove old osm-planet/tiger first?
-          cd /home/planet/
-          mkdir -p tiger
-          wget2 https://nominatim.org/data/tiger-nominatim-preprocessed-latest.csv.tar.gz
-          cd ~/comaps
-          tar -xOzf /home/planet/tiger-nominatim-preprocessed-latest.csv.tar.gz | ~/omim-build-relwithdebinfo/address_parser_tool --output_path=/home/planet/tiger
 
   generate-maps:
-    if: inputs.jobs == 'maps' || inputs.jobs == 'all-except-upload'
+    if: inputs.run-mapgen
     name: Generate Maps
     runs-on: mapfilemaker
     needs:
@@ -461,7 +528,7 @@ jobs:
             --data-urlencode 'content=Generator is done!'
 
   upload-maps:
-    if: inputs.jobs == 'upload'
+    if: inputs.run-upload
     name: Upload Maps
     runs-on: mapfilemaker
     container: