diff --git a/.forgejo/workflows/map-generator.yml b/.forgejo/workflows/map-generator.yml index 6baf75996..e30e23606 100644 --- a/.forgejo/workflows/map-generator.yml +++ b/.forgejo/workflows/map-generator.yml @@ -413,6 +413,7 @@ jobs: run: | mkdir -p /root/OM ln -s /media/4tbexternal/comaps-init /root/OM/organicmaps + ln -s /media/4tbexternal/comaps-init /root/OM/comaps-init ln -s /media/4tbexternal/osm-planet /home/planet ln -s /media/4tbexternal/osm-maps /root/OM/maps_build - name: Run docker_maps_generator.sh diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.prod b/tools/python/maps_generator/var/etc/map_generator.ini.prod index cbb02d1be..f5ea1eaff 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.prod +++ b/tools/python/maps_generator/var/etc/map_generator.ini.prod @@ -1,6 +1,6 @@ [Developer] -# Path to the `organicmaps` source code repository: -OMIM_PATH: ~/OM/organicmaps +# Path to the `comaps` source code repository: +OMIM_PATH: /root/OM/comaps-init # A path with the generator_tool binary: BUILD_PATH: /root/OM/omim-build-relwithdebinfo #${Developer:OMIM_PATH}/../omim-build-release diff --git a/tools/unix/Dockerfile b/tools/unix/Dockerfile index af029f52a..4cc609a90 100755 --- a/tools/unix/Dockerfile +++ b/tools/unix/Dockerfile @@ -46,17 +46,18 @@ RUN pip3 install "protobuf<4" RUN mkdir /root/OM # When running, bind mount the repo here -RUN mkdir /root/OM/organicmaps +RUN mkdir /root/OM/comaps-init # And a volume to store the large output/temp files here RUN mkdir /root/OM/maps_build # And the output of the app build RUN mkdir /root/OM/omim-build-release +RUN mkdir /root/OM/omim-build-relwithdebinfo # And a volume to store >10gb files for the planet here RUN mkdir /home/planet -WORKDIR /root/OM/organicmaps +WORKDIR /root/OM/comaps-init # For debugging #CMD /bin/bash -CMD /root/OM/organicmaps/tools/unix/docker_maps_generator.sh +CMD /root/OM/comaps-init/tools/unix/docker_maps_generator.sh diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh index b49b078dd..ee0a592c0 100755 --- a/tools/unix/docker_maps_generator.sh +++ b/tools/unix/docker_maps_generator.sh @@ -19,7 +19,7 @@ set -e echo "<$(date +%T)> Setting git as safe dir..." -git config --global --add safe.directory /root/OM/organicmaps +git config --global --add safe.directory /root/OM/comaps-init echo "<$(date +%T)> Starting..." @@ -27,8 +27,9 @@ echo "<$(date +%T)> Starting..." # # Already created by Dockerfile or CI/CD: # /root/OM -# /root/OM/organicmaps +# /root/OM/comaps-init # /root/OM/omim-build-release +# /root/OM/omim-build-relwithdebinfo # /root/OM/maps_build # /home/planet # @@ -43,22 +44,22 @@ mkdir -p /home/planet/postcodes/us-postcodes/ mkdir -p /home/planet/SRTM-patched-europe/ mkdir -p /home/planet/subway -echo "Writing rclone config..." -echo "[r2]" > ~/.config/rclone/rclone.conf -echo "type = s3" >> ~/.config/rclone/rclone.conf -echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf -echo "access_key_id = $S3_KEY_ID" >> ~/.config/rclone/rclone.conf -echo "secret_access_key = $S3_SECRET_KEY" >> ~/.config/rclone/rclone.conf -echo "region = auto" >> ~/.config/rclone/rclone.conf -echo "endpoint = $S3_ENDPOINT" >> ~/.config/rclone/rclone.conf -# S3_BUCKET is used below during uploading +# echo "Writing rclone config..." +# echo "[r2]" > ~/.config/rclone/rclone.conf +# echo "type = s3" >> ~/.config/rclone/rclone.conf +# echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf +# echo "access_key_id = $S3_KEY_ID" >> ~/.config/rclone/rclone.conf +# echo "secret_access_key = $S3_SECRET_KEY" >> ~/.config/rclone/rclone.conf +# echo "region = auto" >> ~/.config/rclone/rclone.conf +# echo "endpoint = $S3_ENDPOINT" >> ~/.config/rclone/rclone.conf +# # S3_BUCKET is used below during uploading echo "<$(date +%T)> Running ./configure.sh ..." -cd ~/OM/organicmaps +cd /root/OM/comaps-init ./configure.sh --skip-map-download --skip-generate-symbols echo "<$(date +%T)> Compiling tools..." -cd ~/OM/organicmaps +cd /root/OM/comaps-init ./tools/unix/build_omim.sh -R generator_tool ./tools/unix/build_omim.sh -R world_roads_builder_tool ./tools/unix/build_omim.sh -R mwm_diff_tool @@ -71,7 +72,7 @@ cp var/etc/map_generator.ini.prod var/etc/map_generator.ini #TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources" #echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..." -#cd ~/OM/organicmaps/tools/osmctools +#cd /root/OM/comaps-init/tools/osmctools #gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate #gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert @@ -99,11 +100,11 @@ if [ ! -f planet.o5m ]; then else echo "<$(date +%T)> planet.o5m exists, not converting..." fi -# (currently unused:) ~/OM/organicmaps/tools/unix/update_planet.sh planet.o5m +# (currently unused:) /root/OM/comaps-init/tools/unix/update_planet.sh planet.o5m echo "<$(date +%T)> Generating maps..." -cd ~/OM/organicmaps/tools/python +cd /root/OM/comaps-init/tools/python /tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" #/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" --continue @@ -112,44 +113,44 @@ cd ~/OM/organicmaps/tools/python #/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs" #/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs" -shopt -s nullglob -buildfolder=$(find ~/OM/maps_build/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f5) -builddate=$(find ~/OM/maps_build/*/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f6) -mwmfiles=( ~/OM/maps_build/$buildfolder/$builddate/*.mwm ) +# shopt -s nullglob +# buildfolder=$(find ~/OM/maps_build/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f5) +# builddate=$(find ~/OM/maps_build/*/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f6) +# mwmfiles=( ~/OM/maps_build/$buildfolder/$builddate/*.mwm ) -if (( ${#mwmfiles[@]} )); then - echo "<$(date +%T)> Uploading maps to sftp..." - # upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app) - # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt -sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH < Uploading maps to sftp..." +# # upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app) +# # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt +# sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH < Uploading maps to cloudflare..." - rclone --progress copy ~/OM/maps_build/$buildfolder/$builddate r2:$S3_BUCKET/maps/$builddate/ + # echo "<$(date +%T)> Uploading maps to cloudflare..." + # rclone --progress copy ~/OM/maps_build/$buildfolder/$builddate r2:$S3_BUCKET/maps/$builddate/ else echo "<$(date +%T)> No MWM files in ~/OM/maps_build/$buildfolder/$builddate/*.mwm, not uploading maps." diff --git a/tools/unix/run-docker.sh b/tools/unix/run-docker.sh index 7674b0ace..11c213010 100755 --- a/tools/unix/run-docker.sh +++ b/tools/unix/run-docker.sh @@ -24,7 +24,7 @@ #TODO: isolines, postcodes, subways, wiki # In tools/unix/maps, copy settings.sh.dist to settings.sh and modify if needed -# In tools/python/maps_generator, copy map_generator.ini.prod to map_generator.ini and modify if needed +# In tools/python/maps_generator/var/etc, copy map_generator.ini.prod to map_generator.ini and modify if needed #cd /media/4tbexternal/comaps-init/tools/unix # Build with: docker build . -t maps_generator @@ -58,7 +58,7 @@ docker run \ -e SFTP_HOST=changeme.dreamhost.com \ -e SFTP_PATH=cdn-us-1.comaps.app \ --ulimit nofile=262144:262144 \ - -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \ + -v /media/4tbexternal/comaps-init:/root/OM/comaps-init \ -v /media/4tbexternal/osm-planet:/home/planet \ -v /media/4tbexternal/osm-maps:/root/OM/maps_build \ -it maps_generator \