diff --git a/tools/python/maps_generator/generator/exceptions.py b/tools/python/maps_generator/generator/exceptions.py index 3f8c94f2e..dec292b9b 100644 --- a/tools/python/maps_generator/generator/exceptions.py +++ b/tools/python/maps_generator/generator/exceptions.py @@ -1,5 +1,5 @@ import os - +import subprocess class MapsGeneratorError(Exception): pass @@ -35,9 +35,24 @@ class FailedTest(MapsGeneratorError): def wait_and_raise_if_fail(p): if p.wait() != os.EX_OK: - args = p.args - logs = p.output.name - if p.error.name != logs: - logs += " and " + p.error.name - msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}" - raise BadExitStatusError(msg) + if type(p) is subprocess.Popen: + args = p.args + stdout = p.stdout + stderr = p.stderr + logs = None + errors = None + if type(stdout) is not type(None): + logs = stdout.read(256).decode() + if type(stderr) is not type(None): + errors = stderr.read(256).decode() + if errors != logs: + logs += " and " + errors + msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}" + raise BadExitStatusError(msg) + else: + args = p.args + logs = p.output.name + if p.error.name != logs: + logs += " and " + p.error.name + msg = f"The launch of {args.pop(0)} failed.\nArguments used: {' '.join(args)}\nSee details in {logs}" + raise BadExitStatusError(msg) diff --git a/tools/python/maps_generator/generator/stages_tests.py b/tools/python/maps_generator/generator/stages_tests.py index 821ec2add..b49f335b6 100644 --- a/tools/python/maps_generator/generator/stages_tests.py +++ b/tools/python/maps_generator/generator/stages_tests.py @@ -9,6 +9,8 @@ from maps_generator.utils.file import download_file def make_test_booking_data(max_days): def test_booking_data(env: Env, logger, *args, **kwargs): + if not settings.HOTELS_URL: + return None base_url, _ = settings.HOTELS_URL.rsplit("/", maxsplit=1) url = f"{base_url}/meta.json" meta_path = os.path.join(env.paths.tmp_dir(), "hotels-meta.json") diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.prod b/tools/python/maps_generator/var/etc/map_generator.ini.prod index e6e22bde1..c4e7cb44d 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.prod +++ b/tools/python/maps_generator/var/etc/map_generator.ini.prod @@ -56,7 +56,7 @@ PLANET_MD5_URL: ${External:PLANET_URL}.md5 # then the https://somesite.com/download/latest_coasts.geom url will be used to download latest_coasts.geom and # the https://somesite.com/download/latest_coasts.rawgeom url will be used to download latest_coasts.rawgeom. # Comment to skip getting the coastlines files. -PLANET_COASTS_URL: https://maps.zyphon.com/comaps +PLANET_COASTS_URL: file:///home/planet/ # Should be 'true' for an entire planet build to make a special routing section in World.mwm # for alerting about absent regions without which the route can't be built. NEED_BUILD_WORLD_ROADS: true diff --git a/tools/unix/Dockerfile b/tools/unix/Dockerfile index a2298f05a..83f1c35a8 100644 --- a/tools/unix/Dockerfile +++ b/tools/unix/Dockerfile @@ -14,6 +14,7 @@ ENV TZ=Etc/UTC RUN apt-get update -qq \ && apt-get install -y --no-install-recommends \ curl \ + osmctools \ sshpass \ vim \ wget \ diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh index 984e4483a..7bfe26500 100755 --- a/tools/unix/docker_maps_generator.sh +++ b/tools/unix/docker_maps_generator.sh @@ -25,39 +25,39 @@ echo "<$(date +%T)> Starting..." # /root/OM/maps_build # /home/planet # -mkdir /root/.config -mkdir /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these -chmod 777 /root/.config -chmod 777 /root/.config/OMaps -mkdir ~/OM/maps_build -mkdir ~/OM/omim-build-release -mkdir ~/OM/osmctools -mkdir /home/planet/planet -mkdir /home/planet/planet/isolines/ -mkdir /home/planet/planet/kayak -mkdir /home/planet/planet/tiger/ -mkdir /home/planet/postcodes/ -mkdir /home/planet/postcodes/gb-postcode-data/ -mkdir /home/planet/postcodes/us-postcodes/ -mkdir /home/planet/SRTM-patched-europe/ -mkdir /home/planet/subway +mkdir -p /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these +chmod -R 777 /root/.config +mkdir -p ~/OM/maps_build +mkdir -p ~/OM/omim-build-release +mkdir -p ~/OM/osmctools +mkdir -p /home/planet/planet/isolines/ +mkdir -p /home/planet/planet/kayak/ +mkdir -p /home/planet/planet/tiger/ +mkdir -p /home/planet/postcodes/gb-postcode-data/ +mkdir -p /home/planet/postcodes/us-postcodes/ +mkdir -p /home/planet/SRTM-patched-europe/ +mkdir -p /home/planet/subway echo "Writing S3 config..." echo "[default]" > ~/.s3cfg -echo "access_key = $S3_KEY_ID" >> ~/.s3cfg -echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg -echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg -echo "host_bucket = $(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg +echo "access_key = \$S3_KEY_ID" >> ~/.s3cfg +echo "secret_key = \$S3_SECRET_KEY" >> ~/.s3cfg +echo "host_base = \$S3_HOST_BASE" >> ~/.s3cfg +echo "host_bucket = \$(bucket)s.\$S3_HOST_BASE" >> ~/.s3cfg # S3_BUCKET is used during upload echo "Wrote:" cat ~/.s3cfg +echo "<$(date +%T)> Running ./configure.sh ..." +cd ~/OM/organicmaps +./configure.sh + echo "<$(date +%T)> Compiling tools..." cd ~/OM/organicmaps -./tools/unix/build_omim.sh -r generator_tool -./tools/unix/build_omim.sh -r world_roads_builder_tool -./tools/unix/build_omim.sh -r mwm_diff_tool +./tools/unix/build_omim.sh -R generator_tool +./tools/unix/build_omim.sh -R world_roads_builder_tool +./tools/unix/build_omim.sh -R mwm_diff_tool cd tools/python/maps_generator python3 -m venv /tmp/venv /tmp/venv/bin/pip3 install -r requirements_dev.txt @@ -66,10 +66,10 @@ echo "<$(date +%T)> Copying map generator INI..." cp var/etc/map_generator.ini.prod var/etc/map_generator.ini #TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources" -echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..." -cd ~/OM/organicmaps/tools/osmctools -gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate -gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert +#echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..." +#cd ~/OM/organicmaps/tools/osmctools +#gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate +#gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert # May be unnecessary when running world # /tmp/venv/bin/python -m maps_generator --coasts @@ -79,7 +79,7 @@ gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert cd /home/planet/planet if [ ! -f planet-latest.osm.pbf ]; then echo "<$(date +%T)> Downloading planet-latest.osm.pbf..." - # curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf + curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf echo "<$(date +%T)> Downloading planet-latest.osm.pbf.md5..." curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf.md5 else @@ -99,26 +99,34 @@ fi echo "<$(date +%T)> Generating maps..." cd ~/OM/organicmaps/tools/python -/tmp/venv/bin/python -m maps_generator --production +#/tmp/venv/bin/python -m maps_generator # do not use --production except for Kayak/recommendation/popularity/food data #/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production -#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="Coastline" +/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs" -echo "<$(date +%T)> Uploading maps..." -# maps.zyphon.com:www < Uploading maps..." + # maps.zyphon.com:www < No MWM files, not uploading maps." +fi -echo "<$(date +%T)> Removing intermediate data..." -rm -rf ~/OM/maps_build/*/intermediate_data +echo "<$(date +%T)> NOT Removing intermediate data..." +#rm -rf ~/OM/maps_build/*/intermediate_data # rm -rf ~/OM/ echo "<$(date +%T)> DONE" +