diff --git a/.forgejo/workflows/linux-check.yaml b/.forgejo/workflows/linux-check.yaml index 75874fc81..352c73ac4 100644 --- a/.forgejo/workflows/linux-check.yaml +++ b/.forgejo/workflows/linux-check.yaml @@ -31,7 +31,7 @@ on: jobs: linux-no-unity: name: Linux no unity build - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest # Cancels previous jobs if the same branch or PR was updated again. concurrency: group: ${{ github.workflow }}-no-unity-${{ github.event.pull_request.number || github.ref }} @@ -41,7 +41,7 @@ jobs: - name: Free disk space by removing .NET, Android and Haskell shell: bash run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - name: Checkout sources uses: actions/checkout@v4 @@ -55,8 +55,8 @@ jobs: - name: Install build tools and dependencies shell: bash run: | - sudo apt update -y - sudo apt install -y \ + apt update -y + apt install -y \ ninja-build \ libgl1-mesa-dev \ libglvnd-dev \ @@ -76,7 +76,7 @@ jobs: run: ./configure.sh - name: Configure ccache - uses: hendrikmuhs/ccache-action@v1.2 + uses: https://github.com/hendrikmuhs/ccache-action@v1.2 with: key: ${{ github.workflow }}-no-unity @@ -99,7 +99,7 @@ jobs: linux-matrix: name: Linux builds and tests - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -114,10 +114,10 @@ jobs: - name: Free disk space by removing .NET, Android and Haskell shell: bash run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - name: Checkout sources - uses: actions/checkout@v4 + uses: https://github.com/actions/checkout@v4 - name: Parallel submodules checkout shell: bash @@ -126,8 +126,8 @@ jobs: - name: Install build tools and dependencies shell: bash run: | - sudo apt update -y - sudo apt install -y \ + apt update -y + apt install -y \ ninja-build \ libgl1-mesa-dev \ libglvnd-dev \ @@ -147,7 +147,7 @@ jobs: run: ./configure.sh - name: Configure ccache - uses: hendrikmuhs/ccache-action@v1.2 + uses: https://github.com/hendrikmuhs/ccache-action@v1.2 with: key: ${{ github.workflow }}-unity-${{ matrix.compiler.CC }}-${{ matrix.CMAKE_BUILD_TYPE }} @@ -182,13 +182,13 @@ jobs: # world_feed_integration_tests - https://github.com/organicmaps/organicmaps/issues/215 CTEST_EXCLUDE_REGEX: "generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests" run: | - sudo locale-gen en_US - sudo locale-gen en_US.UTF-8 - sudo locale-gen es_ES - sudo locale-gen es_ES.UTF-8 - sudo locale-gen fr_FR - sudo locale-gen fr_FR.UTF-8 - sudo locale-gen ru_RU - sudo locale-gen ru_RU.UTF-8 - sudo update-locale + locale-gen en_US + locale-gen en_US.UTF-8 + locale-gen es_ES + locale-gen es_ES.UTF-8 + locale-gen fr_FR + locale-gen fr_FR.UTF-8 + locale-gen ru_RU + locale-gen ru_RU.UTF-8 + update-locale ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure diff --git a/.forgejo/workflows/map-generator.yml b/.forgejo/workflows/map-generator.yml new file mode 100644 index 000000000..41613eb50 --- /dev/null +++ b/.forgejo/workflows/map-generator.yml @@ -0,0 +1,243 @@ +name: map-generator +on: + workflow_dispatch: # Manual trigger + inputs: + jobs: + description: 'Which job(s) to run right now?' + required: true + default: 'all' + type: choice + options: + - all + - copy-coasts + - planet + - wiki + - isolines + - subways + - maps + +env: + WIKIMEDIA_USERNAME: ${{ secrets.WIKIMEDIA_USERNAME }} + WIKIMEDIA_PASSWORD: ${{ secrets.WIKIMEDIA_PASSWORD }} + S3_KEY_ID: ${{ secrets.S3_KEY_ID }} + S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} + S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} + SFTP_USER: ${{ secrets.SFTP_USER }} + SFTP_PASSWORD: ${{ secrets.SFTP_PASSWORD }} + SFTP_HOST: ${{ secrets.SFTP_HOST }} + SFTP_PATH: ${{ secrets.SFTP_PATH }} + DEBIAN_FRONTEND: noninteractive + TZ: Etc/UTC + +jobs: + copy-coasts: + if: inputs.jobs == 'copy-coasts' || inputs.jobs == 'all' + name: Copy Coasts + runs-on: mapfilemaker + container: + image: ubuntu:latest + volumes: + - /media/4tbexternal:/media/4tbexternal + concurrency: + group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Copy Coasts + shell: bash + run: | + cp /media/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.geom /media/4tbexternal/osm-planet/latest_coasts.geom + cp /media/4tbexternal/osm-maps/*/intermediate_data/WorldCoasts.rawgeom /media/4tbexternal/osm-planet/latest_coasts.rawgeom + + update-planet: + if: inputs.jobs == 'planet' || inputs.jobs == 'all' + name: Update Planet + runs-on: mapfilemaker + container: + image: ubuntu:latest + volumes: + - /media/4tbexternal:/media/4tbexternal + concurrency: + group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Install dependencies + shell: bash + run: | + apt-get update -y + apt-get install -y pyosmium osmium-tool python3-venv + - name: Update Planet + shell: bash + run: | + cd /media/4tbexternal/osm-planet/planet/ + pyosmium-up-to-date planet-latest.osm.pbf -o planet-latest-new.osm.pbf -vv --size 16384 + mv planet-latest-new.osm.pbf planet-latest.osm.pbf + + wiki-update: + if: inputs.jobs == 'wiki' || inputs.jobs == 'all' + name: Update Wikipedia + runs-on: mapfilemaker + container: + image: ubuntu:latest + volumes: + - /media/4tbexternal:/media/4tbexternal + concurrency: + group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Install dependencies + shell: bash + run: | + apt-get update -y + apt-get install -y jq curl wget2 rustc cargo + - name: Update Wikipedia from Enterprise API + shell: bash + run: | + cd /media/4tbexternal/wikiparser + ./download.sh /media/4tbexternal/osm-planet/wikipedia/dumps + ./run.sh /media/4tbexternal/osm-planet/wikipedia/build \ + /media/4tbexternal/osm-planet/planet/planet-latest.osm.pbf \ + /media/4tbexternal/osm-planet/wikipedia/dumps/latest/*.tar.gz + + update-isolines: + if: inputs.jobs == 'isolines' || inputs.jobs == 'all' + name: Update Isolines + runs-on: mapfilemaker + container: + image: ubuntu:latest + volumes: + - /media/4tbexternal:/media/4tbexternal + concurrency: + group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Update Isolines + shell: bash + run: | + apt-get update -qq \ + && apt-get install -y --no-install-recommends \ + curl \ + osmctools \ + rclone \ + openssh-client \ + sshpass \ + vim \ + wget \ + build-essential \ + clang \ + cmake \ + python3 \ + python3-pip \ + python3.12-venv \ + qt6-base-dev \ + qt6-positioning-dev \ + libc++-dev \ + libfreetype-dev \ + libglvnd-dev \ + libgl1-mesa-dev \ + libharfbuzz-dev \ + libicu-dev \ + libqt6svg6-dev \ + libqt6positioning6-plugins \ + libqt6positioning6 \ + libsqlite3-dev \ + libxrandr-dev \ + libxinerama-dev \ + libxcursor-dev \ + libxi-dev \ + zlib1g-dev + cd /media/4tbexternal/comaps-init/ + ./tools/unix/build_omim.sh -R topography_generator_tool + rm -rf ../osm-planet/isolines/ + mkdir ../osm-planet/isolines/ + ../omim-build-relwithdebinfo/topography_generator_tool \ + --profiles_path=./data/conf/isolines/isolines-profiles.json \ + --countries_to_generate_path=./data/conf/isolines/countries-to-generate.json \ + --tiles_isolines_out_dir=../osm-planet/isolines/tmp-tiles/ \ + --countries_isolines_out_dir=../osm-planet/isolines/ \ + --data_dir=./data/ \ + --srtm_path=../osm-planet/SRTM-patched-europe/ \ + --threads=22 + + update-subways: + if: inputs.jobs == 'subways' || inputs.jobs == 'all' + name: Update Subways + runs-on: mapfilemaker + container: + image: ubuntu:latest + volumes: + - /media/4tbexternal:/media/4tbexternal + concurrency: + group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Update Subways + shell: bash + run: | + apt-get update -qq \ + && apt-get install -y --no-install-recommends \ + curl osmctools osmium-tool python3-venv + cd /media/4tbexternal/comaps-init/ + cp tools/unix/maps/settings.sh.prod tools/unix/maps/settings.sh + ./tools/unix/maps/generate_subways.sh + + generate-maps: + if: inputs.jobs == 'maps' || inputs.jobs == 'all' + name: Generate Maps + runs-on: mapfilemaker + container: + image: ubuntu:latest + volumes: + - /media/4tbexternal:/media/4tbexternal + options: --ulimit nofile=262144:262144 + concurrency: + group: ${{ github.workflow }}-map-generator-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Install dependencies + shell: bash + run: | + apt-get update -qq \ + && apt-get install -y --no-install-recommends \ + curl \ + osmctools \ + rclone \ + openssh-client \ + sshpass \ + vim \ + wget \ + build-essential \ + clang \ + cmake \ + ninja-build \ + python3 \ + python3-pip \ + python3.12-venv \ + qt6-base-dev \ + qt6-positioning-dev \ + libc++-dev \ + libfreetype-dev \ + libglvnd-dev \ + libgl1-mesa-dev \ + libharfbuzz-dev \ + libicu-dev \ + libqt6svg6-dev \ + libqt6positioning6-plugins \ + libqt6positioning6 \ + libsqlite3-dev \ + libxrandr-dev \ + libxinerama-dev \ + libxcursor-dev \ + libxi-dev \ + zlib1g-dev + - name: Symlink paths for repo scripts + shell: bash + run: | + ln -s /media/4tbexternal/comaps-init /root/OM/organicmaps + ln -s /media/4tbexternal/osm-planet /home/planet + ln -s /media/4tbexternal/osm-maps /root/OM/maps_build + - name: Run docker_maps_generator.sh + shell: bash + run: | + cd /root/OM/organicmaps + ./tools/unix/docker_maps_generator.sh \ No newline at end of file