diff --git a/.forgejo/workflows/map-generator.yml b/.forgejo/workflows/map-generator.yml index 9b13aae82..a9be0422b 100644 --- a/.forgejo/workflows/map-generator.yml +++ b/.forgejo/workflows/map-generator.yml @@ -78,11 +78,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -102,11 +103,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -148,11 +150,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -195,11 +198,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -248,15 +252,15 @@ jobs: run: | cd /root/OM/comaps-init/ ./tools/unix/build_omim.sh -R topography_generator_tool - rm -rf ../osm-planet/isolines/ - mkdir ../osm-planet/isolines/ + rm -rf /home/planet/isolines/ + mkdir /home/planet/isolines/ ../omim-build-relwithdebinfo/topography_generator_tool \ --profiles_path=./data/conf/isolines/isolines-profiles.json \ --countries_to_generate_path=./data/conf/isolines/countries-to-generate.json \ - --tiles_isolines_out_dir=../osm-planet/isolines/tmp-tiles/ \ - --countries_isolines_out_dir=../osm-planet/isolines/ \ + --tiles_isolines_out_dir=/home/planet/isolines/tmp-tiles/ \ + --countries_isolines_out_dir=/home/planet/isolines/ \ --data_dir=./data/ \ - --srtm_path=../osm-planet/SRTM-patched-europe/ \ + --srtm_path=/home/planet/SRTM-patched-europe/ \ --threads=22 update-subways: @@ -266,11 +270,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -295,11 +300,11 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} @@ -362,11 +367,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo options: --ulimit nofile=262144:262144 concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} @@ -428,8 +434,7 @@ jobs: # - name: Symlink paths for repo scripts # shell: bash # run: | - # mkdir -p /root/OM - # ln -s /media/4tbexternal/omim-build-release /root/OM/omim-build-release + # mkdir -p /root/OM/omim-build-release /root/OM/omim-build-release # ln -s /root/OM/comaps-init /root/OM/organicmaps # ln -s /root/OM/comaps-init /root/OM/comaps-init # ln -s /home/planet /home/planet @@ -448,12 +453,12 @@ jobs: container: image: codeberg.org/comaps/maps_generator:latest volumes: - - /mnt/4tbexternal:/media/4tbexternal - /mnt/4tbexternal/comaps-init:/root/OM/comaps-init - /mnt/4tbexternal/wikiparser:/root/OM/wikiparser - /mnt/4tbexternal/osm-planet:/home/planet - /mnt/4tbexternal/osm-maps:/root/OM/osm-maps - options: --ulimit nofile=262144:262144 + - /mnt/4tbexternal/subways:/root/OM/subways + - /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo concurrency: group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh index 2a7a61da3..6caed8219 100755 --- a/tools/unix/docker_maps_generator.sh +++ b/tools/unix/docker_maps_generator.sh @@ -70,12 +70,6 @@ python3 -m venv /tmp/venv echo "<$(date +%T)> Copying map generator INI..." cp var/etc/map_generator.ini.prod var/etc/map_generator.ini -#TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources" -#echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..." -#cd /root/OM/comaps-init/tools/osmctools -#gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate -#gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert - # May be unnecessary when running world # /tmp/venv/bin/python -m maps_generator --coasts # save to /path/to/coasts WorldCoasts.geom as latest_coasts.geom and WorldCoasts.rawgeom latest_coasts.rawgeom @@ -96,7 +90,7 @@ fi # (rename us-west-latest to planet-latest and edit the md5 file accordingly) if [ ! -f planet.o5m ]; then echo "<$(date +%T)> Converting planet-latest.osm.pbf to planet.o5m..." - ~/OM/osmctools/osmconvert planet-latest.osm.pbf -o=planet.o5m + osmconvert planet-latest.osm.pbf -o=planet.o5m else echo "<$(date +%T)> planet.o5m exists, not converting..." fi @@ -113,52 +107,5 @@ cd /root/OM/comaps-init/tools/python #/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs" #/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs" -# shopt -s nullglob -# buildfolder=$(find ~/OM/maps_build/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f5) -# builddate=$(find ~/OM/maps_build/*/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f6) -# mwmfiles=( ~/OM/maps_build/$buildfolder/$builddate/*.mwm ) - -# if (( ${#mwmfiles[@]} )); then -# echo "<$(date +%T)> Uploading maps to sftp..." -# # upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app) -# # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt -# sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH < Uploading maps to cloudflare..." - # rclone --progress copy ~/OM/maps_build/$buildfolder/$builddate r2:$S3_BUCKET/maps/$builddate/ - -# else -# echo "<$(date +%T)> No MWM files in ~/OM/osm-maps/$buildfolder/$builddate/*.mwm, not uploading maps." -# echo "<$(date +%T)> Found: $(ls -alt ~/OM/osm-maps/*)" -# fi - -# echo "<$(date +%T)> Temporarily NOT Removing intermediate data..." -#rm -rf ~/OM/osm-maps/*/intermediate_data - echo "<$(date +%T)> DONE" diff --git a/tools/unix/maps/generate_subways.sh b/tools/unix/maps/generate_subways.sh index 99c555655..259ed6717 100755 --- a/tools/unix/maps/generate_subways.sh +++ b/tools/unix/maps/generate_subways.sh @@ -32,6 +32,8 @@ export DUMP_CITY_LIST="$SUBWAYS_VALIDATOR_PATH/cities.txt" # cd to subways repo so relative paths work in the script pushd "$SUBWAYS_REPO_PATH" +echo "Running process_subways.sh:" +ls -al ./scripts/process_subways.sh # 2>&1 | tee "$SUBWAYS_LOG" popd diff --git a/tools/unix/run-docker.sh b/tools/unix/run-docker.sh index c67dc57be..d0568302c 100755 --- a/tools/unix/run-docker.sh +++ b/tools/unix/run-docker.sh @@ -7,17 +7,17 @@ # To bootstrap the repo: #TODO: rename comaps-init to comaps here and throughout -#cd /media/4tbexternal -#if [ ! -f /media/4tbexternal/comaps-init ]; then +#cd /mnt/4tbexternal +#if [ ! -f /mnt/4tbexternal/comaps-init ]; then # git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps-init.git -# cd /media/4tbexternal/comaps-init +# cd /mnt/4tbexternal/comaps-init #else -# cd /media/4tbexternal/comaps-init +# cd /mnt/4tbexternal/comaps-init # git pull origin #fi # And data: -# cd /media/4tbexternal/comaps-init/data/ +# cd /mnt/4tbexternal/comaps-init/data/ # wget World.mwm //pastk - not needed # wget WorldCoasts.mwm @@ -26,7 +26,7 @@ # In tools/unix/maps, copy settings.sh.dist to settings.sh and modify if needed # In tools/python/maps_generator/var/etc, copy map_generator.ini.prod to map_generator.ini and modify if needed -#cd /media/4tbexternal/comaps-init/tools/unix +#cd /mnt/4tbexternal/comaps-init/tools/unix # Build with: docker build . -t maps_generator # (Good to rebuild each time just in case) # If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first. @@ -42,24 +42,20 @@ # -e SFTP_HOST=changeme.dreamhost.com \ # -e SFTP_PATH=cdn-us-1.comaps.app \ # --ulimit nofile=262144:262144 \ -# -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \ -# -v /media/4tbexternal/osm-planet:/home/planet \ -# -v /media/4tbexternal/osm-maps:/root/OM/maps_build \ +# -v /mnt/4tbexternal/comaps-init:/root/OM/comaps-init \ +# -v /mnt/4tbexternal/osm-planet:/home/planet \ +# -v /mnt/4tbexternal/osm-maps:/root/OM/osm-maps \ # -it maps_generator \ -# /root/OM/organicmaps/tools/unix/docker_maps_generator.sh +# /root/OM/comaps-init/tools/unix/docker_maps_generator.sh docker run \ -e S3_KEY_ID=changeme \ - -e S3_SECRET_KEY=changeme \ - -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \ - -e S3_BUCKET=comaps-map-files \ - -e SFTP_USER=changeme \ - -e SFTP_PASSWORD=changeme \ - -e SFTP_HOST=changeme.dreamhost.com \ - -e SFTP_PATH=cdn-us-1.comaps.app \ --ulimit nofile=262144:262144 \ - -v /media/4tbexternal/comaps-init:/root/OM/comaps-init \ - -v /media/4tbexternal/osm-planet:/home/planet \ - -v /media/4tbexternal/osm-maps:/root/OM/osm-maps \ - -it maps_generator \ - /root/OM/organicmaps/tools/unix/docker_maps_generator.sh + -v /mnt/4tbexternal/comaps-init:/root/OM/comaps-init \ + -v /mnt/4tbexternal/wikiparser:/root/OM/wikiparser \ + -v /mnt/4tbexternal/osm-planet:/home/planet \ + -v /mnt/4tbexternal/osm-maps:/root/OM/osm-maps \ + -v /mnt/4tbexternal/subways:/root/OM/subways \ + -v /mnt/4tbexternal/omim-build-relwithdebinfo:/root/OM/omim-build-relwithdebinfo \ + -it codeberg.org/comaps/maps_generator:latest \ + /root/OM/comaps-init/tools/unix/docker_maps_generator.sh