drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2025-12-21 13:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update generator scripts for docker/prod

Browse Source 
Signed-off-by: zyphlar <zyphlar@gmail.com>
This commit is contained in:
zyphlar
2025-06-11 00:10:54 +00:00
committed by Konstantin Pastbin
parent 306aa002f0
commit 6537ca34cd
4 changed files with 91 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tools/unix/Dockerfile
View File

@@ -8,7 +8,8 @@ RUN apt-get update -qq \
&& apt-get install -y --no-install-recommends \ && apt-get install -y --no-install-recommends \
curl \ curl \
osmctools \ osmctools \
s3cmd \ rclone \
sftp \
sshpass \ sshpass \
vim \ vim \
wget \ wget \

75
tools/unix/docker_maps_generator.sh
View File

@@ -1,5 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e
#Volumes/paths for downloads: #Volumes/paths for downloads:
#home/planet/planet/planet.o5m #home/planet/planet/planet.o5m
#home/planet/planet/planet.o5m.md5 #home/planet/planet/planet.o5m.md5
@@ -27,6 +29,7 @@ echo "<$(date +%T)> Starting..."
# #
mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these
chmod -R 777 /root/.config chmod -R 777 /root/.config
mkdir -p ~/.config/rclone
mkdir -p ~/OM/maps_build mkdir -p ~/OM/maps_build
mkdir -p ~/OM/omim-build-release mkdir -p ~/OM/omim-build-release
mkdir -p ~/OM/osmctools mkdir -p ~/OM/osmctools
@@ -38,16 +41,15 @@ mkdir -p /home/planet/postcodes/us-postcodes/
mkdir -p /home/planet/SRTM-patched-europe/ mkdir -p /home/planet/SRTM-patched-europe/
mkdir -p /home/planet/subway mkdir -p /home/planet/subway
echo "Writing S3 config..." echo "Writing rclone config..."
echo "[default]" > ~/.s3cfg echo "[r2]" > ~/.config/rclone/rclone.conf
echo "access_key = $S3_KEY_ID" >> ~/.s3cfg echo "type = s3" >> ~/.config/rclone/rclone.conf
echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf
echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg echo "access_key_id = $S3_KEY_ID" >> ~/.config/rclone/rclone.conf
echo "host_bucket = \$(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg echo "secret_access_key = $S3_SECRET_KEY" >> ~/.config/rclone/rclone.conf
# S3_BUCKET is used during upload echo "region = auto" >> ~/.config/rclone/rclone.conf
echo "endpoint = $S3_ENDPOINT" >> ~/.config/rclone/rclone.conf
echo "Wrote:" # S3_BUCKET is used below during uploading
cat ~/.s3cfg
echo "<$(date +%T)> Running ./configure.sh ..." echo "<$(date +%T)> Running ./configure.sh ..."
cd ~/OM/organicmaps cd ~/OM/organicmaps
@@ -109,30 +111,51 @@ cd ~/OM/organicmaps/tools/python
#/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs" #/tmp/venv/bin/python -m maps_generator --countries="Macedonia" --skip="MwmDiffs"
shopt -s nullglob shopt -s nullglob
mwmfiles=( ~/OM/maps_build/*/*/*.mwm ) buildfolder=$(find ~/OM/maps_build/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f5)
builddate=$(find ~/OM/maps_build/*/ -mindepth 1 -maxdepth 1 -iname 2* -type d | sort -n -r | head -1 | cut -d/ -f6)
mwmfiles=( ~/OM/maps_build/$buildfolder/$builddate/*.mwm )
if (( ${#mwmfiles[@]} )); then if (( ${#mwmfiles[@]} )); then
echo "<$(date +%T)> Uploading maps..." echo "<$(date +%T)> Uploading maps to sftp..."
# TODO: upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app) # upload limited files via SFTP to Dreamhost (cdn-us-1.comaps.app)
# Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt
#sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH <<EOF sshpass -p $SFTP_PASSWORD sftp -o StrictHostKeyChecking=no $SFTP_USER@$SFTP_HOST:$SFTP_PATH <<EOF
#put ~/OM/maps_build/20*/2*/countries.txt lcd ~/OM/maps_build/$buildfolder/$builddate
#put ~/OM/maps_build/20*/2*/World.mwm mkdir maps/$builddate
#put ~/OM/maps_build/20*/2*/WorldCoasts.mwm cd maps/$builddate
#exit put countries.txt
#EOF put World.mwm
put WorldCoasts.mwm
# TODO: upload all files via rclone to Cloudflare (R2) cd ..
#s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/" rm latest
#s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive ln -s $builddate latest
#s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive cd ..
lcd /home/planet/subway/
put subway.json
put subway.log
put subway.transit.json
lcd /home/planet/subway/subway/validator
rm subway/js/*
rmdir subway/js
rm subway/*
rmdir subway
mkdir subway
cd subway
put *
exit
EOF
# upload all files via rclone to Cloudflare (R2)
echo "<$(date +%T)> Uploading maps to cloudflare..."
rclone --progress copy ~/OM/maps_build/$buildfolder/$builddate r2:$S3_BUCKET/maps/$builddate/
else else
echo "<$(date +%T)> No MWM files, not uploading maps." echo "<$(date +%T)> No MWM files in ~/OM/maps_build/$buildfolder/$builddate/*.mwm, not uploading maps."
echo "<$(date +%T)> Found: $(ls -alt ~/OM/maps_build/*)"
fi fi
echo "<$(date +%T)> Temporarily NOT Removing intermediate data..." echo "<$(date +%T)> Temporarily NOT Removing intermediate data..."
#rm -rf ~/OM/maps_build/*/intermediate_data #rm -rf ~/OM/maps_build/*/intermediate_data
# rm -rf ~/OM/
echo "<$(date +%T)> DONE" echo "<$(date +%T)> DONE"

8
tools/unix/maps/settings_default.sh → tools/unix/maps/settings.sh.dist
View File

@@ -1,15 +1,15 @@
# NOTE: edit the settings.sh file to customize/override the defaults. ## NOTE: edit the settings.sh file to customize/override the defaults.
# Absolutize & normalize paths. # Absolutize & normalize paths.
REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}" REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}"
BASE_PATH="${BASE_PATH:-$REPO_PATH/../maps}" BASE_PATH="${BASE_PATH:-$REPO_PATH/..}"
# Temporary files # Temporary files
BUILD_PATH="${BUILD_PATH:-$BASE_PATH/build}" BUILD_PATH="${BUILD_PATH:-$BASE_PATH/osm-planet/build}"
# Other code repositories, e.g. subways, wikiparser.. # Other code repositories, e.g. subways, wikiparser..
CODE_PATH="${CODE_PATH:-$REPO_PATH/..}" CODE_PATH="${CODE_PATH:-$REPO_PATH/..}"
# Source map data and processed outputs e.g. wiki articles # Source map data and processed outputs e.g. wiki articles
DATA_PATH="${DATA_PATH:-$BASE_PATH/data}" DATA_PATH="${DATA_PATH:-$BASE_PATH/osm-planet}"
# OSM planet source files # OSM planet source files

55
tools/unix/run-docker.sh
View File

@@ -6,17 +6,14 @@
# sudo apt install docker git # sudo apt install docker git
# To bootstrap the repo: # To bootstrap the repo:
#TODO: change comaps-init to comaps here and throughout #TODO: rename comaps-init to comaps here and throughout
#TODO: change docker_maps_generator to most recent rebase branch
#cd /media/4tbexternal #cd /media/4tbexternal
#if [ ! -f /media/4tbexternal/comaps-init ]; then #if [ ! -f /media/4tbexternal/comaps-init ]; then
# git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps-init.git # git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps-init.git
# cd /media/4tbexternal/comaps-init # cd /media/4tbexternal/comaps-init
# git checkout docker_maps_generator
#else #else
# cd /media/4tbexternal/comaps-init # cd /media/4tbexternal/comaps-init
# git checkout docker_maps_generator # git pull origin
# git pull origin docker_maps_generator
#fi #fi
# And data: # And data:
@@ -24,25 +21,45 @@
# wget World.mwm //pastk - not needed # wget World.mwm //pastk - not needed
# wget WorldCoasts.mwm # wget WorldCoasts.mwm
#TODO: shaders_complier ? //pastk - not needed
#TODO: isolines, postcodes, subways, wiki #TODO: isolines, postcodes, subways, wiki
# In tools/unix/maps, copy settings.sh.dist to settings.sh and modify if needed
# In tools/python/maps_generator, copy map_generator.ini.prod to map_generator.ini and modify if needed
#cd /media/4tbexternal/comaps-init/tools/unix #cd /media/4tbexternal/comaps-init/tools/unix
# Build with: docker build . -t maps_generator # Build with: docker build . -t maps_generator
# (Good to rebuild each time just in case) # (Good to rebuild each time just in case)
# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition # If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first.
# #
# Edit as appropriate and run with: # Edit as appropriate and run with:
# docker run \ # docker run \
# -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \ # -e S3_KEY_ID=changeme \
# --ulimit nofile=262144:262144 \ # -e S3_SECRET_KEY=changeme \
# -v /media/4tbexternal/comaps-init:/root/OM/organicmaps -v /media/4tbexternal/osm-planet:/home/planet \ # -e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \
# -v /media/4tbexternal/osm-maps:/root/OM/maps_build -it maps_generator # -e S3_BUCKET=comaps-map-files \
# -e SFTP_USER=changeme \
# -e SFTP_PASSWORD=changeme \
docker run -e S3_KEY_ID=changeme -e S3_SECRET_KEY=changeme -e S3_HOST_BASE=change.example.com -e S3_BUCKET=comaps-map-files \ # -e SFTP_HOST=changeme.dreamhost.com \
# -e SFTP_PATH=cdn-us-1.comaps.app \
# --ulimit nofile=262144:262144 \
# -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \
# -v /media/4tbexternal/osm-planet:/home/planet \
# -v /media/4tbexternal/osm-maps:/root/OM/maps_build \
# -it maps_generator \
# /root/OM/organicmaps/tools/unix/docker_maps_generator.sh
docker run \
-e S3_KEY_ID=changeme \
-e S3_SECRET_KEY=changeme \
-e S3_ENDPOINT=https://changeme.r2.cloudflarestorage.com/ \
-e S3_BUCKET=comaps-map-files \
-e SFTP_USER=changeme \
-e SFTP_PASSWORD=changeme \
-e SFTP_HOST=changeme.dreamhost.com \
-e SFTP_PATH=cdn-us-1.comaps.app \
--ulimit nofile=262144:262144 \ --ulimit nofile=262144:262144 \
-v /media/4tbexternal/comaps-init:/root/OM/organicmaps -v /media/4tbexternal/osm-planet:/home/planet \ -v /media/4tbexternal/comaps-init:/root/OM/organicmaps \
-v /media/4tbexternal/osm-maps:/root/OM/maps_build -it maps_generator -v /media/4tbexternal/osm-planet:/home/planet \
-v /media/4tbexternal/osm-maps:/root/OM/maps_build \
-it maps_generator \
/root/OM/organicmaps/tools/unix/docker_maps_generator.sh