diff --git a/.gitignore b/.gitignore index f91b69f1a..03f25e08e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ Makefile.Release object_script.*.Debug object_script.*.Release compile_commands.json +*.local.* stxxl.errlog stxxl.log diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.prod b/tools/python/maps_generator/var/etc/map_generator.ini.prod index 7890a54a1..8c8c7911f 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.prod +++ b/tools/python/maps_generator/var/etc/map_generator.ini.prod @@ -21,7 +21,7 @@ MAIN_OUT_PATH: ${Developer:OMIM_PATH}/../maps_build # Path to the data/ folder in the repository: USER_RESOURCE_PATH: ${Developer:OMIM_PATH}/data # Features stage only parallelism level. Set to 0 for auto detection. -THREADS_COUNT_FEATURES_STAGE: 16 +THREADS_COUNT_FEATURES_STAGE: 12 # Do not change it. This is determined automatically. NODE_STORAGE: mem @@ -66,7 +66,7 @@ PLANET_COASTS_URL: file:///home/planet/ # Subway file location, see docs/SUBWAY_GENERATION.md if you want to generate your own file. # Comment to disable subway layer generation. -SUBWAY_URL: file:///home/planet/subways/subways.transit.json +SUBWAY_URL: file:///home/planet/subway/subway.transit.json # Location of the EXPERIMENTAL GTFS-extracted public transport transit files: # TRANSIT_URL: diff --git a/tools/unix/Dockerfile b/tools/unix/Dockerfile index f6ac1438a..d07d34048 100755 --- a/tools/unix/Dockerfile +++ b/tools/unix/Dockerfile @@ -1,25 +1,4 @@ -# Check out this whole repo with submodules and shallow: - -# cd /path/to/4tb -# git clone --recurse-submodules --shallow-submodules https://codeberg.org/comaps/comaps.git -# cd /path/to/4tb/comaps -# git checkout docker_maps_generator - -#TODO: shaders_complier ? //pastk - not needed - -# Get static data: -# cd /path/to/4tb/comaps/data/ -# wget World.mwm //pastk - not needed -# wget WorldCoasts.mwm -#TODO: isolines, postcodes, subways, wiki - -# Build with: docker build . -t maps_generator -# Edit as appropriate and run with: -# docker run \ -# -e S3_KEY_ID=foo -e S3_SECRET_KEY=bar -e S3_HOST_BASE=digitaloceanspaces.com -e S3_BUCKET=comaps-map-files \ -# -v /path/to/comaps:/root/OM/organicmaps -v /path/to/4tb/osm-planet:/home/planet \ -# -v /path/to/4tb/osm-maps:/root/OM/maps_build -it maps_generator - +# See run-docker.sh for cloning, building, and running the maps generator Docker routine FROM ubuntu:noble ARG DEBIAN_FRONTEND=noninteractive diff --git a/tools/unix/docker_maps_generator.sh b/tools/unix/docker_maps_generator.sh index 509644ec9..dbf6c4f8c 100755 --- a/tools/unix/docker_maps_generator.sh +++ b/tools/unix/docker_maps_generator.sh @@ -100,7 +100,9 @@ fi echo "<$(date +%T)> Generating maps..." cd ~/OM/organicmaps/tools/python -/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" +#/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" +/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" --continue + # do not use --production except for Kayak/recommendation/popularity/food data #/tmp/venv/bin/python -m maps_generator --countries="World, WorldCoasts, US_Oregon_*, US_California_*, US_Washington_*" --production #/tmp/venv/bin/python -m maps_generator --countries="US_Oregon_Portland" --skip="MwmDiffs" @@ -119,9 +121,9 @@ if (( ${#mwmfiles[@]} )); then #exit #EOF - s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/" - s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive - s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive + #s3cmd put ~/OM/maps_build/generation.log "s3://$S3_BUCKET/$(date +%y%m%d)/" + #s3cmd put ~/OM/maps_build/*/*/*.mwm "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive + #s3cmd put ~/OM/maps_build/*/logs "s3://$S3_BUCKET/$(date +%y%m%d)/" --recursive else echo "<$(date +%T)> No MWM files, not uploading maps." fi diff --git a/tools/unix/maps/generate_subways.sh b/tools/unix/maps/generate_subways.sh index 867210368..3808870fe 100755 --- a/tools/unix/maps/generate_subways.sh +++ b/tools/unix/maps/generate_subways.sh @@ -20,9 +20,9 @@ export SKIP_PLANET_UPDATE="1" # If unavailable then replace with a local file. # TODO: keep the downloaded csv file from the latest run. #export CITIES_INFO_URL="" -export TMPDIR="$BUILD_PATH/subways" +export TMPDIR="$BUILD_PATH/subway" # The output file, which needs post-processing by transit_graph_generator.py -export MAPSME="$SUBWAYS_PATH/subways.json" +export MAPSME="$SUBWAYS_PATH/subway.json" # Produce additional files needed for https://cdn.organicmaps.app/subway/ export HTML_DIR="$SUBWAYS_VALIDATOR_PATH" @@ -36,7 +36,7 @@ export DUMP_CITY_LIST="$SUBWAYS_VALIDATOR_PATH/cities.txt" cp -r "$SUBWAYS_REPO_PATH"/render/* "$SUBWAYS_VALIDATOR_PATH/" TRANSIT_TOOL_PATH="$REPO_PATH/tools/python/transit" -SUBWAYS_GRAPH_FILE="$SUBWAYS_PATH/subways.transit.json" +SUBWAYS_GRAPH_FILE="$SUBWAYS_PATH/subway.transit.json" activate_venv_at_path "$TRANSIT_TOOL_PATH" "$PYTHON" "$TRANSIT_TOOL_PATH/transit_graph_generator.py" "$MAPSME" "$SUBWAYS_GRAPH_FILE" 2>&1 | tee -a "$SUBWAYS_LOG" diff --git a/tools/unix/maps/settings_default.sh b/tools/unix/maps/settings_default.sh index 95cf37974..74f90139f 100644 --- a/tools/unix/maps/settings_default.sh +++ b/tools/unix/maps/settings_default.sh @@ -20,6 +20,6 @@ PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}" # Subways SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-$CODE_PATH/subways}" -SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subways}" -SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subways.log}" +SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}" +SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}" SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"