drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2025-12-20 13:23:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

On-server tweaks

Browse Source 
Signed-off-by: zyphlar <zyphlar@gmail.com>
This commit is contained in:
zyphlar
2025-05-05 19:14:39 +00:00
committed by Konstantin Pastbin
parent df70b29623
commit 54fc3d0b64
5 changed files with 78 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
tools/python/maps_generator/generator/exceptions.py
View File

@@ -1,5 +1,5 @@
import os import os
import subprocess
class MapsGeneratorError(Exception): class MapsGeneratorError(Exception):
pass pass
@@ -35,6 +35,21 @@ class FailedTest(MapsGeneratorError):
def wait_and_raise_if_fail(p): def wait_and_raise_if_fail(p):
if p.wait() != os.EX_OK: if p.wait() != os.EX_OK:
if type(p) is subprocess.Popen:
args = p.args
stdout = p.stdout
stderr = p.stderr
logs = None
errors = None
if type(stdout) is not type(None):
logs = stdout.read(256).decode()
if type(stderr) is not type(None):
errors = stderr.read(256).decode()
if errors != logs:
logs += " and " + errors
msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}"
raise BadExitStatusError(msg)
else:
args = p.args args = p.args
logs = p.output.name logs = p.output.name
if p.error.name != logs: if p.error.name != logs:

2
tools/python/maps_generator/generator/stages_tests.py
View File

@@ -9,6 +9,8 @@ from maps_generator.utils.file import download_file
def make_test_booking_data(max_days): def make_test_booking_data(max_days):
def test_booking_data(env: Env, logger, *args, **kwargs): def test_booking_data(env: Env, logger, *args, **kwargs):
if not settings.HOTELS_URL:
return None
base_url, _ = settings.HOTELS_URL.rsplit("/", maxsplit=1) base_url, _ = settings.HOTELS_URL.rsplit("/", maxsplit=1)
url = f"{base_url}/meta.json" url = f"{base_url}/meta.json"
meta_path = os.path.join(env.paths.tmp_dir(), "hotels-meta.json") meta_path = os.path.join(env.paths.tmp_dir(), "hotels-meta.json")

2
tools/python/maps_generator/var/etc/map_generator.ini.prod
View File

@@ -56,7 +56,7 @@ PLANET_MD5_URL: ${External:PLANET_URL}.md5
# then the https://somesite.com/download/latest_coasts.geom url will be used to download latest_coasts.geom and # then the https://somesite.com/download/latest_coasts.geom url will be used to download latest_coasts.geom and
# the https://somesite.com/download/latest_coasts.rawgeom url will be used to download latest_coasts.rawgeom. # the https://somesite.com/download/latest_coasts.rawgeom url will be used to download latest_coasts.rawgeom.
# Comment to skip getting the coastlines files. # Comment to skip getting the coastlines files.
PLANET_COASTS_URL: https://maps.zyphon.com/comaps PLANET_COASTS_URL: file:///home/planet/
# Should be 'true' for an entire planet build to make a special routing section in World.mwm # Should be 'true' for an entire planet build to make a special routing section in World.mwm
# for alerting about absent regions without which the route can't be built. # for alerting about absent regions without which the route can't be built.
NEED_BUILD_WORLD_ROADS: true NEED_BUILD_WORLD_ROADS: true

1
tools/unix/Dockerfile
View File

@@ -14,6 +14,7 @@ ENV TZ=Etc/UTC
RUN apt-get update -qq \ RUN apt-get update -qq \
&& apt-get install -y --no-install-recommends \ && apt-get install -y --no-install-recommends \
curl \ curl \
osmctools \
sshpass \ sshpass \
vim \ vim \
wget \ wget \

72
tools/unix/docker_maps_generator.sh
View File

@@ -25,39 +25,39 @@ echo "<$(date +%T)> Starting..."
# /root/OM/maps_build # /root/OM/maps_build
# /home/planet # /home/planet
# #
mkdir /root/.config mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these
mkdir /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these chmod -R 777 /root/.config
chmod 777 /root/.config mkdir -p ~/OM/maps_build
chmod 777 /root/.config/OMaps mkdir -p ~/OM/omim-build-release
mkdir ~/OM/maps_build mkdir -p ~/OM/osmctools
mkdir ~/OM/omim-build-release mkdir -p /home/planet/planet/isolines/
mkdir ~/OM/osmctools mkdir -p /home/planet/planet/kayak/
mkdir /home/planet/planet mkdir -p /home/planet/planet/tiger/
mkdir /home/planet/planet/isolines/ mkdir -p /home/planet/postcodes/gb-postcode-data/
mkdir /home/planet/planet/kayak mkdir -p /home/planet/postcodes/us-postcodes/
mkdir /home/planet/planet/tiger/ mkdir -p /home/planet/SRTM-patched-europe/
mkdir /home/planet/postcodes/ mkdir -p /home/planet/subway
mkdir /home/planet/postcodes/gb-postcode-data/
mkdir /home/planet/postcodes/us-postcodes/
mkdir /home/planet/SRTM-patched-europe/
mkdir /home/planet/subway
echo "Writing S3 config..." echo "Writing S3 config..."
echo "[default]" > ~/.s3cfg echo "[default]" > ~/.s3cfg
echo "access_key = $S3_KEY_ID" >> ~/.s3cfg echo "access_key = \$S3_KEY_ID" >> ~/.s3cfg
echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg echo "secret_key = \$S3_SECRET_KEY" >> ~/.s3cfg
echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg echo "host_base = \$S3_HOST_BASE" >> ~/.s3cfg
echo "host_bucket = $(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg echo "host_bucket = \$(bucket)s.\$S3_HOST_BASE" >> ~/.s3cfg
# S3_BUCKET is used during upload # S3_BUCKET is used during upload
echo "Wrote:" echo "Wrote:"
cat ~/.s3cfg cat ~/.s3cfg
echo "<$(date +%T)> Running ./configure.sh ..."
cd ~/OM/organicmaps
./configure.sh
echo "<$(date +%T)> Compiling tools..." echo "<$(date +%T)> Compiling tools..."
cd ~/OM/organicmaps cd ~/OM/organicmaps
./tools/unix/build_omim.sh -r generator_tool ./tools/unix/build_omim.sh -R generator_tool
./tools/unix/build_omim.sh -r world_roads_builder_tool ./tools/unix/build_omim.sh -R world_roads_builder_tool
./tools/unix/build_omim.sh -r mwm_diff_tool ./tools/unix/build_omim.sh -R mwm_diff_tool
cd tools/python/maps_generator cd tools/python/maps_generator
python3 -m venv /tmp/venv python3 -m venv /tmp/venv
/tmp/venv/bin/pip3 install -r requirements_dev.txt /tmp/venv/bin/pip3 install -r requirements_dev.txt
@@ -66,10 +66,10 @@ echo "<$(date +%T)> Copying map generator INI..."
cp var/etc/map_generator.ini.prod var/etc/map_generator.ini cp var/etc/map_generator.ini.prod var/etc/map_generator.ini
#TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources" #TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources"
echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..." #echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..."
cd ~/OM/organicmaps/tools/osmctools #cd ~/OM/organicmaps/tools/osmctools
gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate #gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate
gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert #gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert
# May be unnecessary when running world # May be unnecessary when running world
# /tmp/venv/bin/python -m maps_generator --coasts # /tmp/venv/bin/python -m maps_generator --coasts
@@ -79,7 +79,7 @@ gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert
cd /home/planet/planet cd /home/planet/planet
if [ ! -f planet-latest.osm.pbf ]; then if [ ! -f planet-latest.osm.pbf ]; then
echo "<$(date +%T)> Downloading planet-latest.osm.pbf..." echo "<$(date +%T)> Downloading planet-latest.osm.pbf..."
# curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf
echo "<$(date +%T)> Downloading planet-latest.osm.pbf.md5..." echo "<$(date +%T)> Downloading planet-latest.osm.pbf.md5..."
curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf.md5 curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf.md5
else else
@@ -99,10 +99,14 @@ fi
echo "<$(date +%T)> Generating maps..." echo "<$(date +%T)> Generating maps..."
cd ~/OM/organicmaps/tools/python cd ~/OM/organicmaps/tools/python
/tmp/venv/bin/python -m maps_generator --production #/tmp/venv/bin/python -m maps_generator # do not use --production except for Kayak/recommendation/popularity/food data
#/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production #/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production
#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="Coastline" /tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs"
shopt -s nullglob
mwmfiles=( ~/OM/maps_build/*/*/*.mwm )
if (( ${#mwmfiles[@]} )); then
echo "<$(date +%T)> Uploading maps..." echo "<$(date +%T)> Uploading maps..."
# maps.zyphon.com:www <<EOF # maps.zyphon.com:www <<EOF
# Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt # Needs StrictHostKeyChecking=no otherwise new containers/SFTP_HOSTs will require a manual ssh attempt
@@ -116,9 +120,13 @@ echo "<$(date +%T)> Uploading maps..."
s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/" s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/"
s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive
s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive
else
echo "<$(date +%T)> No MWM files, not uploading maps."
fi
echo "<$(date +%T)> Removing intermediate data..." echo "<$(date +%T)> NOT Removing intermediate data..."
rm -rf ~/OM/maps_build/*/intermediate_data #rm -rf ~/OM/maps_build/*/intermediate_data
# rm -rf ~/OM/ # rm -rf ~/OM/
echo "<$(date +%T)> DONE" echo "<$(date +%T)> DONE"