diff --git a/.forgejo/workflows/process_subways.yml b/.forgejo/workflows/process_subways.yml index c59de037d..027f20924 100644 --- a/.forgejo/workflows/process_subways.yml +++ b/.forgejo/workflows/process_subways.yml @@ -1,13 +1,17 @@ -name: process_subways +name: compare_subways on: workflow_dispatch: # Manual trigger + schedule: + # Run daily at 00:00 UTC + - cron: '0 0 * * *' env: #PLANET: /home/planet/planet/planet-latest.pbf #TMPDIR: /tmp - #HTML_DIR: "/mnt/4tbexternal/osm-planet/subway/validator" + HTML_DIR: "/mnt/4tbexternal/osm-planet/subway/validator" #DUMP: "$HTML_DIR" #SKIP_PLANET_UPDATE: "1" + SPREADSHEET_ID: "1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k" DEBIAN_FRONTEND: nonnteractive TZ: Etc/UTC @@ -22,7 +26,7 @@ jobs: volumes: - /mnt/4tbexternal:/mnt/4tbexternal concurrency: - group: ${{ github.workflow }}-process-subways-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true steps: - uses: actions/cache@v4 @@ -50,7 +54,7 @@ jobs: - /mnt/4tbexternal/:/mnt/4tbexternal/ - /mnt/4tbexternal/osm-planet:/home/planet concurrency: - group: ${{ github.workflow }}-process-subways-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true steps: - name: Download Planet File if Absent @@ -92,7 +96,7 @@ jobs: - /mnt/4tbexternal/:/mnt/4tbexternal/ - /mnt/4tbexternal/osm-planet:/home/planet concurrency: - group: ${{ github.workflow }}-process-subways-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true steps: - name: Check for O5M Planet File @@ -130,8 +134,8 @@ jobs: --data-urlencode topic=codeberg-bot \ --data-urlencode 'content=O5M planet update is done!' - update-subways: - name: Update Subways + compare-subways: + name: Compare Subways runs-on: mapfilemaker needs: - clone-repos @@ -141,32 +145,19 @@ jobs: - /mnt/4tbexternal/:/mnt/4tbexternal/ - /mnt/4tbexternal/osm-planet:/home/planet concurrency: - group: ${{ github.workflow }}-processs-subways-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true steps: - uses: actions/cache@v4 with: path: "~" key: cache-${{ github.run_id }}-${{ github.run_attempt }} - - name: Update Subways + - name: Compare Subways shell: bash run: | cd ~/comaps/ cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh ./tools/unix/maps/generate_subways.sh - - name: Compare with VK validation - shell: bash - run: | - cd ~/subways - export HTML_DIR=/mnt/4tbexternal/osm-planet/subway/validator - if [ -f "$HTML_DIR/cities.txt" ]; then - echo "Comparing local validation with VK's validation..." - python3 ./scripts/compare_html_validation.py "$HTML_DIR/cities.txt" \ - --remote-url "https://maps.vk.com/osm/tools/subways/latest/cities.txt" - else - echo "Local cities.txt not found at $HTML_DIR/cities.txt" - exit 1 - fi - name: Notify Zulip run: | curl -X POST https://comaps.zulipchat.com/api/v1/messages \ @@ -175,3 +166,46 @@ jobs: --data-urlencode 'to="DevOps"' \ --data-urlencode topic=codeberg-bot \ --data-urlencode 'content=Subways are done!' + - name: Compare with VK validation + shell: bash + run: | + cd ~/subways + if [ -f "$HTML_DIR/cities.txt" ]; then + echo "Comparing local validation with VK's validation..." + python3 ./scripts/compare_html_validation.py "$HTML_DIR/cities.txt" \ + --remote-url "https://maps.vk.com/osm/tools/subways/latest/cities.txt" + else + echo "Local cities.txt not found at $HTML_DIR/cities.txt" + exit 1 + fi + - name: Compare Google Sheets Data + shell: bash + run: | + set -e + + # Download the Google Sheets data + GOOGLE_SHEETS_URL="https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/export?format=csv" + echo "Downloading Google Sheets data from: $GOOGLE_SHEETS_URL" + curl -sL "$GOOGLE_SHEETS_URL" -o /tmp/google_sheets.csv + + # Normalize line endings to avoid spurious diffs + sed 's/\r$//' source_data/Rapid.csv > /tmp/local_normalized.csv + sed 's/\r$//' /tmp/google_sheets.csv > /tmp/google_normalized.csv + + # Generate unified diff + echo "" + echo "Differences between local Rapid.csv and Google Sheets:" + echo "" + + if diff -u /tmp/local_normalized.csv /tmp/google_normalized.csv > /tmp/full_diff.txt; then + echo "No differences found! Local Rapid.csv is in sync with Google Sheets." + exit 0 + else + cat /tmp/full_diff.txt + + echo "" + echo "The above diff can be saved as a patch and applied with:" + echo " patch source_data/Rapid.csv < patch-file" + echo "or by manually reviewing and applying the changes." + exit 1 + fi