add csv diff

Signed-off-by: zyphlar <zyphlar@gmail.com>

.forgejo/workflows/process_subways.yml

@@ -1,13 +1,17 @@
-name: process_subways
+name: compare_subways
 on:
   workflow_dispatch: # Manual trigger
+  schedule:
+    # Run daily at 00:00 UTC
+    - cron: '0 0 * * *'
 
 env:
   #PLANET: /home/planet/planet/planet-latest.pbf
   #TMPDIR: /tmp
-  #HTML_DIR: "/mnt/4tbexternal/osm-planet/subway/validator"
+  HTML_DIR: "/mnt/4tbexternal/osm-planet/subway/validator"
   #DUMP: "$HTML_DIR"
   #SKIP_PLANET_UPDATE: "1"
+  SPREADSHEET_ID: "1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k"
   DEBIAN_FRONTEND: nonnteractive
   TZ: Etc/UTC
 
@@ -22,7 +26,7 @@ jobs:
       volumes:
         - /mnt/4tbexternal:/mnt/4tbexternal
     concurrency:
-      group: ${{ github.workflow }}-process-subways-${{ github.event.pull_request.number || github.ref }}
+      group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
       - uses: actions/cache@v4
@@ -50,7 +54,7 @@ jobs:
         - /mnt/4tbexternal/:/mnt/4tbexternal/
         - /mnt/4tbexternal/osm-planet:/home/planet
     concurrency:
-      group: ${{ github.workflow }}-process-subways-${{ github.event.pull_request.number || github.ref }}
+      group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
       - name: Download Planet File if Absent
@@ -92,7 +96,7 @@ jobs:
         - /mnt/4tbexternal/:/mnt/4tbexternal/
         - /mnt/4tbexternal/osm-planet:/home/planet
     concurrency:
-      group: ${{ github.workflow }}-process-subways-${{ github.event.pull_request.number || github.ref }}
+      group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
       - name: Check for O5M Planet File
@@ -130,8 +134,8 @@ jobs:
             --data-urlencode topic=codeberg-bot \
             --data-urlencode 'content=O5M planet update is done!'
 
-  update-subways:
-    name: Update Subways
+  compare-subways:
+    name: Compare Subways
     runs-on: mapfilemaker
     needs:
       - clone-repos
@@ -141,32 +145,19 @@ jobs:
         - /mnt/4tbexternal/:/mnt/4tbexternal/
         - /mnt/4tbexternal/osm-planet:/home/planet
     concurrency:
-      group: ${{ github.workflow }}-processs-subways-${{ github.event.pull_request.number || github.ref }}
+      group: ${{ github.workflow }}-compare-subways-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     steps:
       - uses: actions/cache@v4
         with:
           path: "~"
           key: cache-${{ github.run_id }}-${{ github.run_attempt }}
-      - name: Update Subways
+      - name: Compare Subways
         shell: bash
         run: |
           cd ~/comaps/
           cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh
           ./tools/unix/maps/generate_subways.sh
-      - name: Compare with VK validation
-        shell: bash
-        run: |
-          cd ~/subways
-          export HTML_DIR=/mnt/4tbexternal/osm-planet/subway/validator
-          if [ -f "$HTML_DIR/cities.txt" ]; then
-            echo "Comparing local validation with VK's validation..."
-            python3 ./scripts/compare_html_validation.py "$HTML_DIR/cities.txt" \
-              --remote-url "https://maps.vk.com/osm/tools/subways/latest/cities.txt"
-          else
-            echo "Local cities.txt not found at $HTML_DIR/cities.txt"
-            exit 1
-          fi
       - name: Notify Zulip
         run: |
           curl -X POST https://comaps.zulipchat.com/api/v1/messages \
@@ -175,3 +166,46 @@ jobs:
             --data-urlencode 'to="DevOps"' \
             --data-urlencode topic=codeberg-bot \
             --data-urlencode 'content=Subways are done!'
+      - name: Compare with VK validation
+        shell: bash
+        run: |
+          cd ~/subways
+          if [ -f "$HTML_DIR/cities.txt" ]; then
+            echo "Comparing local validation with VK's validation..."
+            python3 ./scripts/compare_html_validation.py "$HTML_DIR/cities.txt" \
+              --remote-url "https://maps.vk.com/osm/tools/subways/latest/cities.txt"
+          else
+            echo "Local cities.txt not found at $HTML_DIR/cities.txt"
+            exit 1
+          fi
+      - name: Compare Google Sheets Data
+        shell: bash
+        run: |
+          set -e
+
+          # Download the Google Sheets data
+          GOOGLE_SHEETS_URL="https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/export?format=csv"
+          echo "Downloading Google Sheets data from: $GOOGLE_SHEETS_URL"
+          curl -sL "$GOOGLE_SHEETS_URL" -o /tmp/google_sheets.csv
+
+          # Normalize line endings to avoid spurious diffs
+          sed 's/\r$//' source_data/Rapid.csv > /tmp/local_normalized.csv
+          sed 's/\r$//' /tmp/google_sheets.csv > /tmp/google_normalized.csv
+
+          # Generate unified diff
+          echo ""
+          echo "Differences between local Rapid.csv and Google Sheets:"
+          echo ""
+
+          if diff -u /tmp/local_normalized.csv /tmp/google_normalized.csv > /tmp/full_diff.txt; then
+            echo "No differences found! Local Rapid.csv is in sync with Google Sheets."
+            exit 0
+          else
+            cat /tmp/full_diff.txt
+
+            echo ""
+            echo "The above diff can be saved as a patch and applied with:"
+            echo "  patch source_data/Rapid.csv < patch-file"
+            echo "or by manually reviewing and applying the changes."
+            exit 1
+          fi