diff --git a/tools/unix/Dockerfile b/tools/unix/Dockerfile index d07d34048..027f368d2 100755 --- a/tools/unix/Dockerfile +++ b/tools/unix/Dockerfile @@ -8,7 +8,8 @@ RUN apt-get update -qq \ && apt-get install -y --no-install-recommends \ curl \ osmctools \ - s3cmd \ + rclone \ + sftp \ sshpass \ vim \ wget \ diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh index 6a6269963..ccb899860 100755 --- a/tools/unix/docker_maps_generator.sh +++ b/tools/unix/docker_maps_generator.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + #Volumes/paths for downloads: #home/planet/planet/planet.o5m #home/planet/planet/planet.o5m.md5 @@ -27,6 +29,7 @@ echo "<$(date +%T)> Starting..." # mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these chmod -R 777 /root/.config +mkdir -p ~/.config/rclone mkdir -p ~/OM/maps_build mkdir -p ~/OM/omim-build-release mkdir -p ~/OM/osmctools @@ -38,16 +41,15 @@ mkdir -p /home/planet/postcodes/us-postcodes/ mkdir -p /home/planet/SRTM-patched-europe/ mkdir -p /home/planet/subway -echo "Writing S3 config..." -echo "[default]" > ~/.s3cfg -echo "access_key = $S3_KEY_ID" >> ~/.s3cfg -echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg -echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg -echo "host_bucket = \$(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg -# S3_BUCKET is used during upload - -echo "Wrote:" -cat ~/.s3cfg +echo "Writing rclone config..." +echo "[r2]" > ~/.config/rclone/rclone.conf +echo "type = s3" >> ~/.config/rclone/rclone.conf +echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf +echo "access_key_id = $S3_KEY_ID" >> ~/.config/rclone/rclone.conf +echo "secret_access_key = $S3_SECRET_KEY" >> ~/.config/rclone/rclone.conf +echo "region = auto" >> ~/.config/rclone/rclone.conf +echo "endpoint = $S3_ENDPOINT" >> ~/.config/rclone/rclone.conf +# S3_BUCKET is used below during uploading echo "<$(date +%T)> Running ./configure.sh ..." cd ~/OM/organicmaps @@ -109,30 +111,51 @@ cd ~/OM/organicmaps/tools/python #/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs" shopt -s nullglob -mwmfiles=( ~/OM/maps_build/*/*/*.mwm ) +buildfolder=$(find ~/OM/maps_build/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f5) +builddate=$(find ~/OM/maps_build/*/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f6) +mwmfiles=( ~/OM/maps_build/$buildfolder/$builddate/*.mwm ) if (( ${#mwmfiles[@]} )); then - echo "<$(date +%T)> Uploading maps..." - # TODO: upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app) + echo "<$(date +%T)> Uploading maps to sftp..." + # upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app) # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt - #sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH < Uploading maps to cloudflare..." + rclone --progress copy ~/OM/maps_build/$buildfolder/$builddate r2:$S3_BUCKET/maps/$builddate/ + else - echo "<$(date +%T)> No MWM files, not uploading maps." + echo "<$(date +%T)> No MWM files in ~/OM/maps_build/$buildfolder/$builddate/*.mwm, not uploading maps." + echo "<$(date +%T)> Found: $(ls -alt ~/OM/maps_build/*)" fi echo "<$(date +%T)> Temporarily NOT Removing intermediate data..." #rm -rf ~/OM/maps_build/*/intermediate_data -# rm -rf ~/OM/ echo "<$(date +%T)> DONE" diff --git a/tools/unix/maps/settings_default.sh b/tools/unix/maps/settings.sh.dist similarity index 77% rename from tools/unix/maps/settings_default.sh rename to tools/unix/maps/settings.sh.dist index 74f90139f..50adbe5ed 100644 --- a/tools/unix/maps/settings_default.sh +++ b/tools/unix/maps/settings.sh.dist @@ -1,15 +1,15 @@ -# NOTE: edit the settings.sh file to customize/override the defaults. +## NOTE: edit the settings.sh file to customize/override the defaults. # Absolutize & normalize paths. REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}" -BASE_PATH="${BASE_PATH:-$REPO_PATH/../maps}" +BASE_PATH="${BASE_PATH:-$REPO_PATH/..}" # Temporary files -BUILD_PATH="${BUILD_PATH:-$BASE_PATH/build}" +BUILD_PATH="${BUILD_PATH:-$BASE_PATH/osm-planet/build}" # Other code repositories, e.g. subways, wikiparser.. CODE_PATH="${CODE_PATH:-$REPO_PATH/..}" # Source map data and processed outputs e.g. wiki articles -DATA_PATH="${DATA_PATH:-$BASE_PATH/data}" +DATA_PATH="${DATA_PATH:-$BASE_PATH/osm-planet}" # OSM planet source files diff --git a/tools/unix/run-docker.sh b/tools/unix/run-docker.sh index 3ac95bdb5..7674b0ace 100755 --- a/tools/unix/run-docker.sh +++ b/tools/unix/run-docker.sh @@ -6,17 +6,14 @@ # sudo apt install docker git # To bootstrap the repo: -#TODO: change comaps-init to comaps here and throughout -#TODO: change docker_maps_generator to most recent rebase branch +#TODO: rename comaps-init to comaps here and throughout #cd /media/4tbexternal #if [ ! -f /media/4tbexternal/comaps-init ]; then # git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps-init.git # cd /media/4tbexternal/comaps-init -# git checkout docker_maps_generator #else # cd /media/4tbexternal/comaps-init -# git checkout docker_maps_generator -# git pull origin docker_maps_generator +# git pull origin #fi # And data: @@ -24,25 +21,45 @@ # wget World.mwm //pastk - not needed # wget WorldCoasts.mwm -#TODO: shaders_complier ? //pastk - not needed - #TODO: isolines, postcodes, subways, wiki +# In tools/unix/maps, copy settings.sh.dist to settings.sh and modify if needed +# In tools/python/maps_generator, copy map_generator.ini.prod to map_generator.ini and modify if needed + #cd /media/4tbexternal/comaps-init/tools/unix # Build with: docker build . -t maps_generator # (Good to rebuild each time just in case) -# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition +# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first. # # Edit as appropriate and run with: -# docker run \ -# -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \ -# --ulimit nofile=262144:262144 \ -# -v /media/4tbexternal/comaps-init:/root/OM/organicmaps -v /media/4tbexternal/osm-planet:/home/planet \ -# -v /media/4tbexternal/osm-maps:/root/OM/maps_build -it maps_generator - - -docker run -e S3_KEY_ID=changeme -e S3_SECRET_KEY=changeme -e S3_HOST_BASE=change.example.com -e S3_BUCKET=comaps-map-files \ +# docker run \ +# -e S3_KEY_ID=changeme \ +# -e S3_SECRET_KEY=changeme \ +# -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \ +# -e S3_BUCKET=comaps-map-files \ +# -e SFTP_USER=changeme \ +# -e SFTP_PASSWORD=changeme \ +# -e SFTP_HOST=changeme.dreamhost.com \ +# -e SFTP_PATH=cdn-us-1.comaps.app \ +# --ulimit nofile=262144:262144 \ +# -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \ +# -v /media/4tbexternal/osm-planet:/home/planet \ +# -v /media/4tbexternal/osm-maps:/root/OM/maps_build \ +# -it maps_generator \ +# /root/OM/organicmaps/tools/unix/docker_maps_generator.sh + +docker run \ + -e S3_KEY_ID=changeme \ + -e S3_SECRET_KEY=changeme \ + -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \ + -e S3_BUCKET=comaps-map-files \ + -e SFTP_USER=changeme \ + -e SFTP_PASSWORD=changeme \ + -e SFTP_HOST=changeme.dreamhost.com \ + -e SFTP_PATH=cdn-us-1.comaps.app \ --ulimit nofile=262144:262144 \ - -v /media/4tbexternal/comaps-init:/root/OM/organicmaps -v /media/4tbexternal/osm-planet:/home/planet \ - -v /media/4tbexternal/osm-maps:/root/OM/maps_build -it maps_generator - + -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \ + -v /media/4tbexternal/osm-planet:/home/planet \ + -v /media/4tbexternal/osm-maps:/root/OM/maps_build \ + -it maps_generator \ + /root/OM/organicmaps/tools/unix/docker_maps_generator.sh