drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2026-01-08 13:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e07413eda5c3fca6c2924f9c6de8b0ad41c795bc
comaps/.forgejo/workflows/map-generator.yml
zyphlar e07413eda5 use new rclone script 
Signed-off-by: zyphlar <zyphlar@gmail.com>
2025-11-02 11:45:16 -08:00

471 lines
17 KiB
YAML
Raw Blame History

name: map-generator
on:
workflow_dispatch: # Manual trigger
inputs:
jobs:
description: 'Which job(s) to run right now?'
required: true
default: 'all'
type: choice
options:
- all
- clone-repos
- copy-coasts
- planet
- wiki
- isolines
- subways
- tiger
- maps
- upload
## RCLONE_CONF is multi-line text containing keys and credentials for us2,ru1,fi1,de1 servers
env:
WIKIMEDIA_USERNAME: ${{ secrets.WIKIMEDIA_USERNAME }}
WIKIMEDIA_PASSWORD: ${{ secrets.WIKIMEDIA_PASSWORD }}
RCLONE_CONF: ${{ secrets.RCLONE_CONF }}
DEBIAN_FRONTEND: noninteractive
TZ: Etc/UTC
jobs:
clone-repos:
if: inputs.jobs == 'clone-repos' || inputs.jobs == 'all'
name: Clone Git Repos
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/mnt/4tbexternal
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Clone or update main repo if necessary
shell: bash
run: |
ls -al /mnt
ls -al /mnt/4tbexternal
if [ ! -d /mnt/4tbexternal/comaps-init ]; then
cd /mnt/4tbexternal
git clone --recurse-submodules --shallow-submodules -b rebase-generator-pastk-wb251027 --single-branch https://codeberg.org/comaps/comaps.git comaps-init
else
cd /mnt/4tbexternal/comaps-init && git pull origin rebase-generator-pastk-wb251027
fi
- name: Clone or update wikiparser repo if necessary
shell: bash
run: |
if [ ! -d /mnt/4tbexternal/wikiparser ]; then
cd /mnt/4tbexternal
git clone https://codeberg.org/comaps/wikiparser.git
else
cd /mnt/4tbexternal/wikiparser && git pull origin main
fi
- name: Clone or update subways repo if necessary
shell: bash
run: |
if [ ! -d /mnt/4tbexternal/subways ]; then
cd /mnt/4tbexternal
git clone https://codeberg.org/comaps/subways.git
else
cd /mnt/4tbexternal/subways && git pull origin master
fi
copy-coasts:
if: inputs.jobs == 'copy-coasts' || inputs.jobs == 'all'
name: Copy Previously Generated Coasts
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Copy Coasts
shell: bash
run: |
if [ -f /root/OM/osm-maps/*/intermediate_data/WorldCoasts.geom ]; then
cp /root/OM/osm-maps/*/intermediate_data/WorldCoasts.geom /home/planet/latest_coasts.geom
cp /root/OM/osm-maps/*/intermediate_data/WorldCoasts.rawgeom /home/planet/latest_coasts.rawgeom
fi
update-planet:
if: inputs.jobs == 'planet' || inputs.jobs == 'all'
name: Update Planet
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
# - name: Install dependencies
# shell: bash
# run: |
# apt-get update -y
# apt-get install -y pyosmium osmium-tool osmctools python3-venv python3-pip wget2
# rm -f /usr/lib/python*/EXTERNALLY-MANAGED
# pip3 install "protobuf<4"
- name: Download Planet File if Absent
shell: bash
run: |
if [ ! -d /home/planet/planet/ ]; then
mkdir -p /home/planet/planet/
fi
if [ ! -f /home/planet/planet/planet-latest.osm.pbf ]; then
cd /home/planet/planet/
wget2 --verbose --progress=bar --continue https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf
fi
- name: Update Planet
shell: bash
run: |
cd /home/planet/planet/
pyosmium-up-to-date planet-latest.osm.pbf -o planet-latest-new.osm.pbf -vv --size 16384
mv planet-latest-new.osm.pbf planet-latest.osm.pbf
- name: Converting planet-latest.osm.pbf to planet.o5m
run: |
echo "Starting..."
cd /home/planet/planet/
osmconvert planet-latest.osm.pbf -o=planet.o5m
echo "Done."
wiki-update:
if: inputs.jobs == 'wiki' || inputs.jobs == 'all'
name: Update Wikipedia
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
# - name: Install dependencies
# shell: bash
# run: |
# apt-get update -y
# apt-get install -y jq curl wget wget2 rustc cargo git ca-certificates
- name: Check for planet file
shell: bash
run: |
if [ ! -f /home/planet/planet/planet-latest.osm.pbf ]; then
echo "ERROR: No file at /home/planet/planet/planet-latest.osm.pbf"
ls -al /home/planet/
ls -al /home/planet/planet/
exit 1
fi
- name: Update Wikipedia from Enterprise API
shell: bash
run: |
#todo: curl in download.sh can fail when rate limited and even save error messages to the output. need to validate.
mkdir -p /home/planet/wikipedia/dumps
mkdir -p /home/planet/wikipedia/build
cd /root/OM/wikiparser
ls -al
echo "Downloading ..."
./download.sh /home/planet/wikipedia/dumps
ls -al /home/planet/wikipedia/dumps/*
echo "Running ..."
./run.sh /home/planet/wikipedia/build \
/home/planet/planet/planet-latest.osm.pbf \
/home/planet/wikipedia/dumps/latest/*.tar.gz
echo "DONE"
update-isolines:
if: inputs.jobs == 'isolines' || inputs.jobs == 'all'
name: Update Isolines
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
# - name: Install dependencies
# shell: bash
# run: |
# apt-get update -qq \
# && apt-get install -y --no-install-recommends \
# curl \
# osmctools \
# rclone \
# git \
# ca-certificates \
# openssh-client \
# sshpass \
# vim \
# wget \
# build-essential \
# clang \
# cmake \
# python3 \
# python3-pip \
# python3.12-venv \
# qt6-base-dev \
# qt6-positioning-dev \
# libc++-dev \
# libfreetype-dev \
# libglvnd-dev \
# libgl1-mesa-dev \
# libharfbuzz-dev \
# libicu-dev \
# libqt6svg6-dev \
# libqt6positioning6-plugins \
# libqt6positioning6 \
# libsqlite3-dev \
# libxrandr-dev \
# libxinerama-dev \
# libxcursor-dev \
# libxi-dev \
# zlib1g-dev
# rm -f /usr/lib/python*/EXTERNALLY-MANAGED
# pip3 install "protobuf<4"
- name: Update Isolines
shell: bash
run: |
cd /root/OM/comaps-init/
./tools/unix/build_omim.sh -R topography_generator_tool
rm -rf ../osm-planet/isolines/
mkdir ../osm-planet/isolines/
../omim-build-relwithdebinfo/topography_generator_tool \
--profiles_path=./data/conf/isolines/isolines-profiles.json \
--countries_to_generate_path=./data/conf/isolines/countries-to-generate.json \
--tiles_isolines_out_dir=../osm-planet/isolines/tmp-tiles/ \
--countries_isolines_out_dir=../osm-planet/isolines/ \
--data_dir=./data/ \
--srtm_path=../osm-planet/SRTM-patched-europe/ \
--threads=22
update-subways:
if: inputs.jobs == 'subways' || inputs.jobs == 'all'
name: Update Subways
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
# - name: Install dependencies
# shell: bash
# run: |
# apt-get update -qq && apt-get install -y --no-install-recommends curl osmctools osmium-tool python3-venv ca-certificates git python3-pip
# rm -f /usr/lib/python*/EXTERNALLY-MANAGED
# pip3 install "protobuf<4"
- name: Update Subways
shell: bash
run: |
cd /root/OM/comaps-init/
cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh
./tools/unix/maps/generate_subways.sh
update-tiger:
if: inputs.jobs == 'tiger' || inputs.jobs == 'all'
name: Update TIGER
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
- /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
# - name: Install dependencies
# shell: bash
# run: |
# apt-get update -qq && apt-get install -y --no-install-recommends \
# build-essential \
# clang \
# cmake \
# ninja-build \
# ca-certificates \
# qt6-base-dev \
# qt6-positioning-dev \
# libc++-dev \
# libfreetype-dev \
# libglvnd-dev \
# libgl1-mesa-dev \
# libharfbuzz-dev \
# libicu-dev \
# libqt6svg6-dev \
# libqt6positioning6-plugins \
# libqt6positioning6 \
# libsqlite3-dev \
# libxrandr-dev \
# libxinerama-dev \
# libxcursor-dev \
# libxi-dev \
# zlib1g-dev \
# python3 \
# python3-pip \
# git \
# wget2
# rm -f /usr/lib/python*/EXTERNALLY-MANAGED
# pip3 install "protobuf<4"
- name: Build address_parser
shell: bash
run: |
cd /root/OM/comaps-init
rm -rf ../omim-build-relwithdebinfo/CMakeCache.txt
rm -rf ../omim-build-relwithdebinfo/CMakeFiles
./tools/unix/build_omim.sh -R address_parser_tool
- name: Update TIGER from Nominatim
shell: bash
run: |
# TODO: maybe remove old osm-planet/tiger first?
cd /home/planet/
mkdir -p tiger
wget2 https://nominatim.org/data/tiger-nominatim-preprocessed-latest.csv.tar.gz
cd /root/OM/comaps-init
tar -xOzf /home/planet/tiger-nominatim-preprocessed-latest.csv.tar.gz | /root/OM/omim-build-relwithdebinfo/address_parser_tool --output_path=/home/planet/tiger
generate-maps:
if: inputs.jobs == 'maps' || inputs.jobs == 'all'
name: Generate Maps
runs-on: mapfilemaker
timeout-minutes: 40320
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
options: --ulimit nofile=262144:262144
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
# - name: Install dependencies
# shell: bash
# run: |
# apt-get update -qq \
# && apt-get install -y --no-install-recommends \
# curl \
# osmctools \
# rclone \
# git \
# ca-certificates \
# openssh-client \
# sshpass \
# vim \
# wget \
# build-essential \
# clang \
# cmake \
# ninja-build \
# python3 \
# python3-pip \
# python3.12-venv \
# qt6-base-dev \
# qt6-positioning-dev \
# libc++-dev \
# libfreetype-dev \
# libglvnd-dev \
# libgl1-mesa-dev \
# libharfbuzz-dev \
# libicu-dev \
# libqt6svg6-dev \
# libqt6positioning6-plugins \
# libqt6positioning6 \
# libsqlite3-dev \
# libxrandr-dev \
# libxinerama-dev \
# libxcursor-dev \
# libxi-dev \
# zlib1g-dev
# rm -f /usr/lib/python*/EXTERNALLY-MANAGED
# pip3 install "protobuf<4"
- name: Make output folders if necessary
shell: bash
run: |
if [ ! -d /root/OM/osm-maps ]; then
mkdir -p /root/OM/osm-maps
fi
- name: Get SRTM if necessary
shell: bash
run: |
if [ ! -d /home/planet/SRTM-patched-europe/ ]; then
echo "ERROR: NO SRTM"
exit 1
fi
# - name: Symlink paths for repo scripts
# shell: bash
# run: |
# mkdir -p /root/OM
# ln -s /media/4tbexternal/omim-build-release /root/OM/omim-build-release
# ln -s /root/OM/comaps-init /root/OM/organicmaps
# ln -s /root/OM/comaps-init /root/OM/comaps-init
# ln -s /home/planet /home/planet
# ln -s /root/OM/osm-maps /root/OM/maps_build
- name: Run docker_maps_generator.sh
shell: bash
run: |
cd /root/OM/comaps-init
./tools/unix/docker_maps_generator.sh
upload-maps:
if: inputs.jobs == 'upload' || inputs.jobs == 'all'
name: Upload Maps
runs-on: mapfilemaker
container:
image: codeberg.org/comaps/maps_generator:latest
volumes:
- /mnt/4tbexternal:/media/4tbexternal
- /mnt/4tbexternal/comaps-init:/root/OM/comaps-init
- /mnt/4tbexternal/wikiparser:/root/OM/wikiparser
- /mnt/4tbexternal/osm-planet:/home/planet
- /mnt/4tbexternal/osm-maps:/root/OM/osm-maps
options: --ulimit nofile=262144:262144
concurrency:
group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Write config file
run: |
mkdir -p ~/.config/rclone/
echo "${{ secrets.RCLONE_CONF }}" > ~/.config/rclone/rclone.conf
head -n3 ~/.config/rclone/rclone.conf
- name: Upload map files to CDNs via RClone
shell: bash
run: |
cd /root/OM/comaps-init
./tools/unix/maps/upload_to_cdn.sh
Reference in New Issue View Git Blame Copy Permalink