diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.prod b/tools/python/maps_generator/var/etc/map_generator.ini.prod new file mode 100644 index 000000000..e6e22bde1 --- /dev/null +++ b/tools/python/maps_generator/var/etc/map_generator.ini.prod @@ -0,0 +1,104 @@ +[Developer] +# Path to the `organicmaps` source code repository: +OMIM_PATH: ~/OM/organicmaps +# A path with the generator_tool binary: +BUILD_PATH: ${Developer:OMIM_PATH}/../omim-build-release + + +[Main] +# A special small planet file will be downloaded if DEBUG is set to 1. +DEBUG: 0 +# A main working directory. There is a subdirectory created for each generator run +# which contains the planet and other downloads, temporary build files, logs and completed MWMs. +MAIN_OUT_PATH: ${Developer:OMIM_PATH}/../maps_build +# Path for storing caches for nodes, ways, relations. +# If it's not set then caches are stored inside the directory of the current build. +# CACHE_PATH: ${Main:MAIN_OUT_PATH}/cache + + +[Generator tool] +# Path to the data/ folder in the repository: +USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data +# Features stage only parallelism level. Set to 0 for auto detection. +THREADS_COUNT_FEATURES_STAGE: 0 + + +[Osm tools] +# Path to osmctools binaries: +OSM_TOOLS_PATH: ${Developer:OMIM_PATH}/../osmctools +# If the binaries are not found neither in the configured path nor system-wide, +# then the tools are built from the sources: +OSM_TOOLS_SRC_PATH: ${Developer:OMIM_PATH}/tools/osmctools + + +[Logging] +# maps_generator's general (python output only) log file path and name. +# More detailed logs that include output of the `generator_tool` binary +# are located in the `logs/` subdir of a particular build directory, +# e.g. `maps_build/2023_06_04__20_05_07/logs/`. +LOG_FILE_PATH: ${Main:MAIN_OUT_PATH}/generation.log + + +[External] +# Planet file location. It should be a dump of OSM data in osm.pbf format. +# By default its an entire planet from "planet.openstreetmap.org". +# Or set it to a particular country/region extract from e.g. [Geofabrik](http://download.geofabrik.de/index.html). +# Note that an entire planet generation takes 40+ hours on a 256GB RAM server (and 1TB+ disk space). +# Stick to smaller extracts unless you have a machine this large. +# Here and further, its possible to specify either an URL (to be downloaded automatically) +# or a local file path like file:///path/to/file. +# A sample URL to download a latest OSM dump for North Macedonia: +PLANET_URL: file:///home/planet/planet/planet.o5m +# Location of the md5 checksum of the planet file: +PLANET_MD5_URL: ${External:PLANET_URL}.md5 +# A base url to the latest_coasts.geom and latest_coasts.rawgeom files. +# For example, if PLANET_COASTS_URL = https://somesite.com/download/ +# then the https://somesite.com/download/latest_coasts.geom url will be used to download latest_coasts.geom and +# the https://somesite.com/download/latest_coasts.rawgeom url will be used to download latest_coasts.rawgeom. +# Comment to skip getting the coastlines files. +PLANET_COASTS_URL: https://maps.zyphon.com/comaps +# Should be 'true' for an entire planet build to make a special routing section in World.mwm +# for alerting about absent regions without which the route can't be built. +NEED_BUILD_WORLD_ROADS: true +# Subway file location, see docs/SUBWAY_GENERATION.md if you want to generate your own file. +# Comment to disable subway layer generation. +SUBWAY_URL: file:///home/planet/subway/beta.json + +# Location of the EXPERIMENTAL GTFS-extracted public transport transit files: +# TRANSIT_URL: + +# Urls for production maps generation. +# UGC_URL: +# HOTELS_URL: +# PROMO_CATALOG_CITIES: +# POPULARITY_URL: +# FOOD_URL: +# FOOD_TRANSLATIONS_URL: +SRTM_PATH: /home/planet/SRTM-patched-europe/ +ISOLINES_PATH: /home/planet/planet/isolines/ +ADDRESSES_PATH: /home/planet/planet/tiger/ + +# Local path (not url!) to .csv files. +UK_POSTCODES_URL: /home/planet/postcodes/gb-postcode-data/gb_postcodes.csv +US_POSTCODES_URL: /home/planet/postcodes/us-postcodes/uszips.csv + + +[Stages] +# Set to 1 to update the entire OSM planet file (as taken from "planet.openstreetmap.org") +# via an osmupdate tool before the generation. Not for use with partial planet extracts. +NEED_PLANET_UPDATE: 0 +# If you want to calculate diffs you need to specify where the old maps are, +# e.g. ${Main:MAIN_OUT_PATH}/2021_03_16__09_00_00/ +DATA_ARCHIVE_DIR: ${Generator tool:USER_RESOURCE_PATH} +# How many versions in the archive to use for diff calculation: +DIFF_VERSION_DEPTH: 2 + + +[Common] +# Default parallelism level for the most of jobs. Set to 0 for auto detection. +THREADS_COUNT: 0 + + +[Stats] +# Path to rules for calculating statistics by type: +STATS_TYPES_CONFIG: ${Developer:OMIM_PATH}/tools/python/maps_generator/var/etc/stats_types_config.txt diff --git a/tools/unix/Dockerfile b/tools/unix/Dockerfile new file mode 100644 index 000000000..a2298f05a --- /dev/null +++ b/tools/unix/Dockerfile @@ -0,0 +1,59 @@ +# Check out this whole repo with submodules and shallow +# Build with: docker build . -t maps_generator +# Edit as appropriate and run with: +# docker run \ +# -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \ +# -v /path/to/organicmaps:/root/OM/organicmaps -v /tmp/osm-planet:/home/planet \ +# -v /tmp/osm-maps:/root/OM/maps_build -it maps_generator + +FROM ubuntu:noble + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +RUN apt-get update -qq \ + && apt-get install -y --no-install-recommends \ + curl \ + sshpass \ + vim \ + wget \ + build-essential \ + clang \ + cmake \ + ninja-build \ + python3 \ + python3-pip \ + python3.12-venv \ + qt6-base-dev \ + qt6-positioning-dev \ + libc++-dev \ + libfreetype-dev \ + libglvnd-dev \ + libgl1-mesa-dev \ + libharfbuzz-dev \ + libicu-dev \ + libqt6svg6-dev \ + libqt6positioning6-plugins \ + libqt6positioning6 \ + libsqlite3-dev \ + libxrandr-dev \ + libxinerama-dev \ + libxcursor-dev \ + libxi-dev \ + zlib1g-dev \ + && rm -rf /var/cache/apt/* /var/lib/apt/lists/*; + +RUN mkdir /root/OM +# When running, bind mount the repo here +RUN mkdir /root/OM/organicmaps +# And a volume to store the large output/temp files here +RUN mkdir /root/OM/maps_build +# And a volume to store >10gb files for the planet here +RUN mkdir /home/planet + +WORKDIR /root/OM/organicmaps + +# For debugging +#CMD /bin/bash + +CMD /root/OM/organicmaps/tools/unix/docker_maps_generator.sh diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh new file mode 100755 index 000000000..984e4483a --- /dev/null +++ b/tools/unix/docker_maps_generator.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash + +#Volumes/paths for downloads: +#home/planet/planet/planet.o5m +#home/planet/planet/planet.o5m.md5 +#PLANET_COASTS_URL:file:///home/planet/planet/ + #home/planet/planet/latest_coasts.geom and latest_coasts.rawgeom +#SUBWAY_URL: file:///home/planet/subway/beta.json + #home/planet/subway/beta.json +#HOTELS_URL:/home/planet/planet/kayak/ + #home/planet/planet/kayak/ +#SRTM_PATH:/home/planet/SRTM-patched-europe/ +#ISOLINES_PATH:/home/planet/planet/isolines/ +#ADDRESSES_PATH:/home/planet/planet/tiger/ +#UK_POSTCODES_URL:/home/planet/postcodes/gb-postcode-data/gb_postcodes.csv +#US_POSTCODES_URL:/home/planet/postcodes/us-postcodes/uszips.csv + +echo "<$(date +%T)> Starting..." + +# Prepare paths +# +# Already created by Dockerfile: +# /root/OM +# /root/OM/organicmaps +# /root/OM/maps_build +# /home/planet +# +mkdir /root/.config +mkdir /root/.config/OMaps # Odd mkdir permission errors in generator_tool in Docker without these +chmod 777 /root/.config +chmod 777 /root/.config/OMaps +mkdir ~/OM/maps_build +mkdir ~/OM/omim-build-release +mkdir ~/OM/osmctools +mkdir /home/planet/planet +mkdir /home/planet/planet/isolines/ +mkdir /home/planet/planet/kayak +mkdir /home/planet/planet/tiger/ +mkdir /home/planet/postcodes/ +mkdir /home/planet/postcodes/gb-postcode-data/ +mkdir /home/planet/postcodes/us-postcodes/ +mkdir /home/planet/SRTM-patched-europe/ +mkdir /home/planet/subway + +echo "Writing S3 config..." +echo "[default]" > ~/.s3cfg +echo "access_key = $S3_KEY_ID" >> ~/.s3cfg +echo "secret_key = $S3_SECRET_KEY" >> ~/.s3cfg +echo "host_base = $S3_HOST_BASE" >> ~/.s3cfg +echo "host_bucket = $(bucket)s.$S3_HOST_BASE" >> ~/.s3cfg +# S3_BUCKET is used during upload + +echo "Wrote:" +cat ~/.s3cfg + +echo "<$(date +%T)> Compiling tools..." +cd ~/OM/organicmaps +./tools/unix/build_omim.sh -r generator_tool +./tools/unix/build_omim.sh -r world_roads_builder_tool +./tools/unix/build_omim.sh -r mwm_diff_tool +cd tools/python/maps_generator +python3 -m venv /tmp/venv +/tmp/venv/bin/pip3 install -r requirements_dev.txt + +echo "<$(date +%T)> Copying map generator INI..." +cp var/etc/map_generator.ini.prod var/etc/map_generator.ini + +#TODO: may be duplicated by maps_generator at "osmctools are not found, building from the sources" +echo "<$(date +%T)> Prebuild some tools so we can make an o5m file or run update_planet..." +cd ~/OM/organicmaps/tools/osmctools +gcc osmupdate.c -l z -o ~/OM/osmctools/osmupdate +gcc osmconvert.c -l z -o ~/OM/osmctools/osmconvert + +# May be unnecessary when running world +# /tmp/venv/bin/python -m maps_generator --coasts +# save to /path/to/coasts WorldCoasts.geom as latest_coasts.geom and WorldCoasts.rawgeom latest_coasts.rawgeom +# (from https://github.com/mapsme/omim/issues/11994) + +cd /home/planet/planet +if [ ! -f planet-latest.osm.pbf ]; then + echo "<$(date +%T)> Downloading planet-latest.osm.pbf..." + # curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf + echo "<$(date +%T)> Downloading planet-latest.osm.pbf.md5..." + curl -OL https://ftpmirror.your.org/pub/openstreetmap/pbf/planet-latest.osm.pbf.md5 +else + echo "<$(date +%T)> planet-latest.osm.pbf exists, not downloading..." +fi + +#curl -OL https://download.geofabrik.de/north-america/us-west-latest.osm.pbf +#curl -OL https://download.geofabrik.de/north-america/us-west-latest.osm.pbf.md5 +# (rename us-west-latest to planet-latest and edit the md5 file accordingly) +if [ ! -f planet.o5m ]; then + echo "<$(date +%T)> Converting planet-latest.osm.pbf to planet.o5m..." + ~/OM/osmctools/osmconvert planet-latest.osm.pbf -o=planet.o5m +else + echo "<$(date +%T)> planet.o5m exists, not converting..." +fi +# (currently unused:) ~/OM/organicmaps/tools/unix/update_planet.sh planet.o5m + +echo "<$(date +%T)> Generating maps..." +cd ~/OM/organicmaps/tools/python +/tmp/venv/bin/python -m maps_generator --production +#/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production +#/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="Coastline" + +echo "<$(date +%T)> Uploading maps..." +# maps.zyphon.com:www < Removing intermediate data..." +rm -rf ~/OM/maps_build/*/intermediate_data +# rm -rf ~/OM/ + +echo "<$(date +%T)> DONE" diff --git a/tools/unix/update_planet.sh b/tools/unix/update_planet.sh new file mode 100755 index 000000000..33c6a0eb4 --- /dev/null +++ b/tools/unix/update_planet.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euxo pipefail + +OSMUPDATE=~/osmctools/osmupdate +# osmconvert should be accessible in PATH. +PATH="$(dirname "$OSMUPDATE"):$PATH" + +# Pass pbf or o5m file as a parameter +OLD="$1" +NEW="${1/.pbf/.new.pbf}" +NEW="${NEW/.o5m/.new.o5m}" + +"$OSMUPDATE" -v --drop-authors --drop-version --hash-memory=512000 "$OLD" "$NEW" +# Uncomment to replace old planet. +mv "$NEW" "$OLD" +#md5sum -b "$OLD" > "$OLD.md5" +echo "Successfully updated $OLD"