Update generator scripts for docker/prod

Signed-off-by: zyphlar <zyphlar@gmail.com>

tools/unix/Dockerfile

@@ -8,7 +8,8 @@ RUN apt-get update -qq \
     && apt-get install -y --no-install-recommends \
     curl \
     osmctools \
-    s3cmd \
+    rclone \
+    sftp \
     sshpass \
     vim \
     wget \

tools/unix/docker_maps_generator.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 #Volumes/paths for downloads:
 #home/planet/planet/planet.o5m
 #home/planet/planet/planet.o5m.md5
@@ -27,6 +29,7 @@ echo "<$(date +%T)> Starting..."
 #
 mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these
 chmod -R 777 /root/.config
+mkdir -p ~/.config/rclone
 mkdir -p ~/OM/maps_build
 mkdir -p ~/OM/omim-build-release
 mkdir -p ~/OM/osmctools
@@ -38,16 +41,15 @@ mkdir -p /home/planet/postcodes/us-postcodes/
 mkdir -p /home/planet/SRTM-patched-europe/
 mkdir -p /home/planet/subway
 
-echo "Writing S3 config..."
+echo "Writing rclone config..."
-echo "[default]" > ~/.s3cfg
+echo "[r2]" > ~/.config/rclone/rclone.conf
-echo "access_key = $S3_KEY_ID" >> ~/.s3cfg
+echo "type = s3" >> ~/.config/rclone/rclone.conf
-echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg
+echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf
-echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg
+echo "access_key_id = $S3_KEY_ID" >> ~/.config/rclone/rclone.conf
-echo "host_bucket = \$(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg
+echo "secret_access_key = $S3_SECRET_KEY" >> ~/.config/rclone/rclone.conf
-# S3_BUCKET is used during upload
+echo "region = auto" >> ~/.config/rclone/rclone.conf
-
+echo "endpoint = $S3_ENDPOINT" >> ~/.config/rclone/rclone.conf
-echo "Wrote:"
+# S3_BUCKET is used below during uploading
-cat ~/.s3cfg
 
 echo "<$(date +%T)> Running ./configure.sh ..."
 cd ~/OM/organicmaps
@@ -109,30 +111,51 @@ cd ~/OM/organicmaps/tools/python
 #/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs"
 
 shopt -s nullglob
-mwmfiles=( ~/OM/maps_build/*/*/*.mwm )
+buildfolder=$(find ~/OM/maps_build/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f5)
+builddate=$(find ~/OM/maps_build/*/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f6)
+mwmfiles=( ~/OM/maps_build/$buildfolder/$builddate/*.mwm )
 
 if (( ${#mwmfiles[@]} )); then
-  echo "<$(date +%T)> Uploading maps..."
+  echo "<$(date +%T)> Uploading maps to sftp..."
-  # TODO: upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app)
+  # upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app)
   # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt
-  #sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH <<EOF
+sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH <<EOF
-  #put ~/OM/maps_build/20*/2*/countries.txt
+  lcd ~/OM/maps_build/$buildfolder/$builddate
-  #put ~/OM/maps_build/20*/2*/World.mwm
+  mkdir maps/$builddate
-  #put ~/OM/maps_build/20*/2*/WorldCoasts.mwm
+  cd maps/$builddate
-  #exit
+  put countries.txt
-  #EOF
+  put World.mwm
+  put WorldCoasts.mwm
+  cd ..
+  rm latest
+  ln -s $builddate latest
+  cd ..
+  lcd /home/planet/subway/
+  put subway.json
+  put subway.log
+  put subway.transit.json
+  lcd /home/planet/subway/subway/validator
+  rm subway/js/*
+  rmdir subway/js
+  rm subway/*
+  rmdir subway
+  mkdir subway
+  cd subway
+  put *
+  exit
+EOF
+
+  # upload all files via rclone to Cloudflare (R2)
+  echo "<$(date +%T)> Uploading maps to cloudflare..."
+  rclone --progress copy ~/OM/maps_build/$buildfolder/$builddate r2:$S3_BUCKET/maps/$builddate/
 
-  # TODO: upload all files via rclone to Cloudflare (R2)
-  #s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/"
-  #s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive
-  #s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive
 else
-  echo "<$(date +%T)> No MWM files, not uploading maps."
+  echo "<$(date +%T)> No MWM files in ~/OM/maps_build/$buildfolder/$builddate/*.mwm, not uploading maps."
+  echo "<$(date +%T)> Found: $(ls -alt ~/OM/maps_build/*)"
 fi
 
 echo "<$(date +%T)> Temporarily NOT Removing intermediate data..."
 #rm -rf ~/OM/maps_build/*/intermediate_data
-# rm -rf ~/OM/
 
 echo "<$(date +%T)> DONE"
 

tools/unix/maps/settings.sh.dist

@@ -1,15 +1,15 @@
-# NOTE: edit the settings.sh file to customize/override the defaults.
+## NOTE: edit the settings.sh file to customize/override the defaults.
 
 # Absolutize & normalize paths.
 REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}"
 
-BASE_PATH="${BASE_PATH:-$REPO_PATH/../maps}"
+BASE_PATH="${BASE_PATH:-$REPO_PATH/..}"
 # Temporary files
-BUILD_PATH="${BUILD_PATH:-$BASE_PATH/build}"
+BUILD_PATH="${BUILD_PATH:-$BASE_PATH/osm-planet/build}"
 # Other code repositories, e.g. subways, wikiparser..
 CODE_PATH="${CODE_PATH:-$REPO_PATH/..}"
 # Source map data and processed outputs e.g. wiki articles
-DATA_PATH="${DATA_PATH:-$BASE_PATH/data}"
+DATA_PATH="${DATA_PATH:-$BASE_PATH/osm-planet}"
 
 # OSM planet source files
 

tools/unix/run-docker.sh

@@ -6,17 +6,14 @@
 # sudo apt install docker git
 
 # To bootstrap the repo:
-#TODO: change comaps-init to comaps here and throughout
+#TODO: rename comaps-init to comaps here and throughout
-#TODO: change docker_maps_generator to most recent rebase branch
 #cd /media/4tbexternal
 #if [ ! -f /media/4tbexternal/comaps-init ]; then
 #  git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps-init.git
 #  cd /media/4tbexternal/comaps-init
-#  git checkout docker_maps_generator
 #else
 #  cd /media/4tbexternal/comaps-init
-#  git checkout docker_maps_generator
+#  git pull origin
-#  git pull origin docker_maps_generator
 #fi
 
 # And data:
@@ -24,25 +21,45 @@
 # wget World.mwm //pastk - not needed
 # wget WorldCoasts.mwm
 
-#TODO: shaders_complier ? //pastk - not needed
-
 #TODO: isolines, postcodes, subways, wiki
 
+# In tools/unix/maps, copy settings.sh.dist to settings.sh and modify if needed
+# In tools/python/maps_generator, copy map_generator.ini.prod to map_generator.ini and modify if needed
+
 #cd /media/4tbexternal/comaps-init/tools/unix
 # Build with: docker build . -t maps_generator
 # (Good to rebuild each time just in case)
-# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition
+# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first.
 #
 # Edit as appropriate and run with:
 # docker run \
-#  -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \
+#   -e S3_KEY_ID=changeme \
+#   -e S3_SECRET_KEY=changeme \
+#   -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \
+#   -e S3_BUCKET=comaps-map-files \
+#   -e SFTP_USER=changeme \
+#   -e SFTP_PASSWORD=changeme \
+#   -e SFTP_HOST=changeme.dreamhost.com \
+#   -e SFTP_PATH=cdn-us-1.comaps.app \
 #   --ulimit nofile=262144:262144 \
-#  -v /media/4tbexternal/comaps-init:/root/OM/organicmaps -v /media/4tbexternal/osm-planet:/home/planet \
+#   -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \
-#  -v /media/4tbexternal/osm-maps:/root/OM/maps_build -it maps_generator
+#   -v /media/4tbexternal/osm-planet:/home/planet \
+#   -v /media/4tbexternal/osm-maps:/root/OM/maps_build \
+#   -it maps_generator \
+#   /root/OM/organicmaps/tools/unix/docker_maps_generator.sh
  
-
+docker run \
-docker run -e S3_KEY_ID=changeme -e S3_SECRET_KEY=changeme -e S3_HOST_BASE=change.example.com -e S3_BUCKET=comaps-map-files \
+  -e S3_KEY_ID=changeme \
+  -e S3_SECRET_KEY=changeme \
+  -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \
+  -e S3_BUCKET=comaps-map-files \
+  -e SFTP_USER=changeme \
+  -e SFTP_PASSWORD=changeme \
+  -e SFTP_HOST=changeme.dreamhost.com \
+  -e SFTP_PATH=cdn-us-1.comaps.app \
   --ulimit nofile=262144:262144 \
-  -v /media/4tbexternal/comaps-init:/root/OM/organicmaps -v /media/4tbexternal/osm-planet:/home/planet \
+  -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \
-  -v /media/4tbexternal/osm-maps:/root/OM/maps_build -it maps_generator
+  -v /media/4tbexternal/osm-planet:/home/planet \
-
+  -v /media/4tbexternal/osm-maps:/root/OM/maps_build \
+  -it maps_generator \
+  /root/OM/organicmaps/tools/unix/docker_maps_generator.sh