[indexer] Use ICU regex to handle unicode characters in mastodon/bluesky domains

Improve regex, (ICU required for unicode character support) Also add tests for umlauts, add generated "Testing" folder to gitignore, and clean up url::UrlEncode a bit
TODO: android doesn't build
Signed-off-by: Harry Bond <me@hbond.xyz>

.gitignore

@@ -9,7 +9,6 @@ Makefile.Release
 object_script.*.Debug
 object_script.*.Release
 compile_commands.json
-*.local.*
 
 stxxl.errlog
 stxxl.log
@@ -28,6 +27,10 @@ data/edits.xml
 data/World.mwm
 data/WorldCoasts.mwm
 
+# generated when running tests. NOT the lowercase 'testing' which should be kept.
+Testing
+!testing
+
 # Compiled Python
 *.pyc
 

android/app/build.gradle

@@ -15,6 +15,12 @@ buildscript {
   }
 }
 
+repositories {
+  google()
+  mavenCentral()
+  maven { url 'https://www.jitpack.io' } // MPAndroidChart
+}
+
 apply plugin: 'com.android.application'
 apply plugin: 'com.github.triplet.play'
 apply plugin: 'ru.cian.huawei-publish-gradle-plugin'
@@ -406,8 +412,6 @@ android {
 }
 
 dependencies {
-  implementation project(':sdk')
-
   coreLibraryDesugaring libs.android.tools.desugar
 
   // Google Play Location Services

android/app/src/google/play/listings/da-DK/full-description.txt

@@ -1,36 +0,0 @@
-En fællesskabsdrevet gratis og open source-kortapp baseret på OpenStreetMap-data og forstærket med en forpligtelse til gennemsigtighed, privatliv og non-profit.
-
-Bliv en del af fællesskabet og vær med til at skabe den bedste kortapp
-• Brug appen og fortæl andre om den
-• Giv feedback og rapporter problemer
-• Opdater kortdata i appen eller på OpenStreetMap-webstedet
-
-<i>Din feedback og 5-stjernede anmeldelser er den bedste støtte for os!</i>
-
-‣ <b>Enkel og poleret</b>: væsentlige, brugervenlige funktioner, der bare virker.
-‣ <b>Offline-focused</b>: Planlæg og naviger på din rejse i udlandet uden behov for mobilfordbindelse, søg efter rutepunkter, mens du er på en lang vandretur osv. Alle app-funktioner er designet til at fungere offline..
-‣ <b>Respekt for privatlivets fred</b>: Appen er designet med fokus på privatlivets fred – den identificerer ikke personer, sporer ikke og indsamler ikke personlige oplysninger. Annoncefri.
-‣ <b>Sparer på batteriet og plads</b>: Dræner ikke dit batteri som andre navigationsapps. Kompakte kort sparer værdifuld plads på din telefon.
-‣ <b>Gratis og udviklet af fællesskabet</b>: Folk som dig har været med til at udvikle appen ved at tilføje steder til OpenStreetMap, teste og give feedback på funktioner samt bidrage med deres udviklingskompetencer og penge.
-‣ <b>Åben og gennemsigtig beslutningstagning og økonomi, non-profit og fuldstændig open source.</b>
-
-<b>Vigtigste funktioner</b>:
-• Detaljerede kort med steder, der ikke er tilgængelige på Google Maps, kan downloades.
-• Udendørstilstand med fremhævede vandreruter, campingpladser, vandkilder, bjergtoppe, højdekurver osv.
-• Gang- og cykelstier
-• Interessepunkter som restauranter, tankstationer, hoteller, butikker, seværdigheder og meget mere
-• Søg efter navn, adresse eller efter interessepunkt-kategori
-• Navigation med stemmevejledning til gående, cyklende eller kørende
-• Bogmærk dine yndlingssteder med ét enkelt tryk
-• Offline Wikipedia-artikler
-• Lag med metrolinjer og rutevejledning
-• Sporoptagelse
-• Eksportér og importér bogmærker og ruter i KML, KMZ, GPX-formater
-• En mørk tilstand til brug om natten
-• Forbedr kortdata for alle ved hjælp af en simpel, indbygget redigeringsfunktion
-• Android Auto-understøttelse
-
-Rapportér app-problemer, kom med forslag og bliv en del af vores fællesskab på <b><i>comaps.app</i></b> webstedet.
-
-<b>Nu med frihed</b>
-Udforsk din rejse, og navigér i verden med fokus på privatliv og fællesskab!

android/app/src/google/play/listings/da-DK/title.txt

@@ -1 +0,0 @@
-CoMaps - Naviger med privatliv

android/app/src/google/play/listings/fr-FR/short-description.txt

@@ -1 +1 @@
-Navigation cartographique facile - Découvrez davantage de votre voyage - Propulsé par la communauté
+Navigation cartographique facile - Propulsé par la communauté

android/app/src/main/java/app/organicmaps/MwmActivity.java

@@ -141,6 +141,7 @@ import static app.organicmaps.leftbutton.LeftButtonsHolder.BUTTON_HELP_CODE;
 import static app.organicmaps.leftbutton.LeftButtonsHolder.BUTTON_RECORD_TRACK_CODE;
 import static app.organicmaps.leftbutton.LeftButtonsHolder.BUTTON_SETTINGS_CODE;
 import static app.organicmaps.sdk.util.PowerManagment.POWER_MANAGEMENT_TAG;
+import static app.organicmaps.sdk.util.concurrency.UiThread.runLater;
 
 
 public class MwmActivity extends BaseMwmFragmentActivity
@@ -2596,8 +2597,7 @@ public class MwmActivity extends BaseMwmFragmentActivity
 
       items.add(new MenuBottomSheetItem(R.string.download_maps, R.drawable.ic_download, getDownloadMapsCounter(), this::onDownloadMapsOptionSelected));
 
-      mDonatesUrl = Config.getDonateUrl(getApplicationContext());
-      if (!mDonatesUrl.isEmpty())
+      if (!Config.getDonateUrl(getApplicationContext()).isEmpty())
         items.add(new MenuBottomSheetItem(R.string.donate, R.drawable.ic_donate, this::onDonateOptionSelected));
 
       if (!BUTTON_SETTINGS_CODE.equals(activeLeftButton))

android/app/src/main/java/app/organicmaps/backup/BackupUtils.java

@@ -4,6 +4,7 @@ import static app.organicmaps.settings.BackupSettingsFragment.MAX_BACKUPS_DEFAUL
 import static app.organicmaps.settings.BackupSettingsFragment.MAX_BACKUPS_KEY;
 import static app.organicmaps.sdk.util.StorageUtils.isFolderWritable;
 
+import android.app.Activity;
 import android.content.Context;
 import android.content.SharedPreferences;
 import android.net.Uri;
@@ -16,6 +17,11 @@ import android.text.style.AbsoluteSizeSpan;
 import androidx.annotation.NonNull;
 import androidx.documentfile.provider.DocumentFile;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;

android/app/src/main/java/app/organicmaps/backup/PeriodicBackupRunner.java

@@ -5,6 +5,7 @@ import static app.organicmaps.backup.BackupUtils.isBackupFolderAvailable;
 import static app.organicmaps.settings.BackupSettingsFragment.BACKUP_FOLDER_PATH_KEY;
 import static app.organicmaps.settings.BackupSettingsFragment.BACKUP_INTERVAL_KEY;
 import static app.organicmaps.settings.BackupSettingsFragment.LAST_BACKUP_TIME_KEY;
+import static app.organicmaps.sdk.util.StorageUtils.isFolderWritable;
 
 import android.app.Activity;
 import android.content.SharedPreferences;

android/app/src/main/java/app/organicmaps/bookmarks/BookmarksListFragment.java

@@ -150,7 +150,9 @@ public class BookmarksListFragment extends BaseMwmRecyclerFragment<ConcatAdapter
 
     BookmarkCollectionAdapter adapter = new BookmarkCollectionAdapter(getCategoryOrThrow(),
                                                                       mCategoryItems);
-    adapter.setOnClickListener((v, item) -> BookmarkListActivity.startForResult(this, startBookmarkListForResult, item));
+    adapter.setOnClickListener((v, item) -> {
+      BookmarkListActivity.startForResult(this, startBookmarkListForResult, item);
+    });
 
     return adapter;
   }
@@ -853,18 +855,15 @@ public class BookmarksListFragment extends BaseMwmRecyclerFragment<ConcatAdapter
   @Nullable
   public ArrayList<MenuBottomSheetItem> getMenuBottomSheetItems(String id)
   {
-      switch (id) {
-          case BOOKMARKS_MENU_ID -> {
-              return getBookmarkMenuItems();
-          }
-          case TRACK_MENU_ID -> {
-              final Track track = (Track) getBookmarkListAdapter().getItem(mSelectedPosition);
-              return getTrackMenuItems(track);
-          }
-          case OPTIONS_MENU_ID -> {
-              return getOptionsMenuItems();
-          }
-      }
-      return null;
+    if (id.equals(BOOKMARKS_MENU_ID))
+      return getBookmarkMenuItems();
+    if (id.equals(TRACK_MENU_ID))
+    {
+      final Track track = (Track) getBookmarkListAdapter().getItem(mSelectedPosition);
+      return getTrackMenuItems(track);
+    }
+    if (id.equals(OPTIONS_MENU_ID))
+      return getOptionsMenuItems();
+    return null;
   }
 }

android/app/src/main/java/app/organicmaps/car/CarAppService.java

@@ -3,6 +3,7 @@ package app.organicmaps.car;
 import android.content.ComponentName;
 import android.content.Intent;
 import android.net.Uri;
+import android.os.Build;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
@@ -15,6 +16,8 @@ import androidx.car.app.validation.HostValidator;
 import androidx.core.app.NotificationChannelCompat;
 import androidx.core.app.NotificationCompat;
 import androidx.core.app.NotificationManagerCompat;
+import androidx.lifecycle.DefaultLifecycleObserver;
+import androidx.lifecycle.LifecycleOwner;
 
 import app.organicmaps.BuildConfig;
 import app.organicmaps.R;

android/app/src/main/java/app/organicmaps/car/screens/search/SearchOnMapScreen.java

@@ -27,7 +27,6 @@ import app.organicmaps.sdk.search.SearchListener;
 import app.organicmaps.sdk.search.SearchEngine;
 import app.organicmaps.sdk.search.SearchRecents;
 import app.organicmaps.sdk.search.SearchResult;
-import app.organicmaps.sdk.util.Config;
 import app.organicmaps.sdk.util.Language;
 
 public class SearchOnMapScreen extends BaseMapScreen implements SearchListener
@@ -115,8 +114,7 @@ public class SearchOnMapScreen extends BaseMapScreen implements SearchListener
       }
 
       builder.setOnClickListener(() -> {
-        if (Config.isSearchHistoryEnabled())
-          SearchRecents.add(title, getCarContext());
+        SearchRecents.add(title, getCarContext());
         SearchEngine.INSTANCE.cancel();
         SearchEngine.INSTANCE.showResult(resultIndex);
       });

android/app/src/main/java/app/organicmaps/car/screens/search/SearchScreen.java

@@ -25,7 +25,6 @@ import app.organicmaps.sdk.search.SearchListener;
 import app.organicmaps.sdk.search.SearchEngine;
 import app.organicmaps.sdk.search.SearchRecents;
 import app.organicmaps.sdk.search.SearchResult;
-import app.organicmaps.sdk.util.Config;
 import app.organicmaps.sdk.util.Language;
 
 public class SearchScreen extends BaseMapScreen implements SearchTemplate.SearchCallback, SearchListener
@@ -148,8 +147,7 @@ public class SearchScreen extends BaseMapScreen implements SearchTemplate.Search
       if (!TextUtils.isEmpty(region))
         builder.addText(region);
       builder.setOnClickListener(() -> {
-        if (Config.isSearchHistoryEnabled())
-          SearchRecents.add(title, getCarContext());
+        SearchRecents.add(title, getCarContext());
         SearchEngine.INSTANCE.cancel();
         SearchEngine.INSTANCE.showResult(resultIndex);
       });

android/app/src/main/java/app/organicmaps/car/util/RoutingHelpers.java

@@ -33,17 +33,35 @@ public final class RoutingHelpers
   public static LaneDirection createLaneDirection(@NonNull LaneWay laneWay, boolean isRecommended)
   {
     int shape = LaneDirection.SHAPE_UNKNOWN;
-      shape = switch (laneWay) {
-          case REVERSE -> LaneDirection.SHAPE_U_TURN_LEFT;
-          case SHARP_LEFT -> LaneDirection.SHAPE_SHARP_LEFT;
-          case LEFT -> LaneDirection.SHAPE_NORMAL_LEFT;
-          case SLIGHT_LEFT, MERGE_TO_LEFT -> LaneDirection.SHAPE_SLIGHT_LEFT;
-          case SLIGHT_RIGHT, MERGE_TO_RIGHT -> LaneDirection.SHAPE_SLIGHT_RIGHT;
-          case THROUGH -> LaneDirection.SHAPE_STRAIGHT;
-          case RIGHT -> LaneDirection.SHAPE_NORMAL_RIGHT;
-          case SHARP_RIGHT -> LaneDirection.SHAPE_SHARP_RIGHT;
-          default -> shape;
-      };
+    switch (laneWay)
+    {
+    case REVERSE:
+      shape = LaneDirection.SHAPE_U_TURN_LEFT;
+      break;
+    case SHARP_LEFT:
+      shape = LaneDirection.SHAPE_SHARP_LEFT;
+      break;
+    case LEFT:
+      shape = LaneDirection.SHAPE_NORMAL_LEFT;
+      break;
+    case SLIGHT_LEFT:
+    case MERGE_TO_LEFT:
+      shape = LaneDirection.SHAPE_SLIGHT_LEFT;
+      break;
+    case SLIGHT_RIGHT:
+    case MERGE_TO_RIGHT:
+      shape = LaneDirection.SHAPE_SLIGHT_RIGHT;
+      break;
+    case THROUGH:
+      shape = LaneDirection.SHAPE_STRAIGHT;
+      break;
+    case RIGHT:
+      shape = LaneDirection.SHAPE_NORMAL_RIGHT;
+      break;
+    case SHARP_RIGHT:
+      shape = LaneDirection.SHAPE_SHARP_RIGHT;
+      break;
+    }
 
     return LaneDirection.create(shape, isRecommended);
   }

android/app/src/main/java/app/organicmaps/downloader/OnmapDownloader.java

@@ -142,10 +142,9 @@ public class OnmapDownloader implements MwmActivity.LeftAnimationTrackListener
 
         if (progress)
         {
-          int roundedProgress = Math.round(mCurrentCountry.progress);
           mProgress.setPending(false);
-          mProgress.setProgress(roundedProgress);
-          sizeText = mActivity.getString(R.string.downloader_downloading) + " " + StringUtils.formatPercent(roundedProgress / 100.0);
+          mProgress.setProgress(Math.round(mCurrentCountry.progress));
+          sizeText = mActivity.getString(R.string.downloader_downloading) + " " + StringUtils.formatPercent(mCurrentCountry.progress / 100);
         }
         else
         {

android/app/src/main/java/app/organicmaps/editor/OsmLoginFragment.java

@@ -12,11 +12,12 @@ import android.widget.ScrollView;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.core.view.ViewCompat;
-
+import app.organicmaps.sdk.Framework;
 import app.organicmaps.R;
 import app.organicmaps.base.BaseMwmToolbarFragment;
 import app.organicmaps.sdk.editor.OsmOAuth;
 import app.organicmaps.sdk.util.Constants;
+import app.organicmaps.sdk.util.DateUtils;
 import app.organicmaps.util.InputUtils;
 import app.organicmaps.sdk.util.UiUtils;
 import app.organicmaps.util.Utils;
@@ -55,6 +56,7 @@ public class OsmLoginFragment extends BaseMwmToolbarFragment
     MaterialButton registerButton = view.findViewById(R.id.register);
     registerButton.setOnClickListener((v) -> Utils.openUrl(requireActivity(), Constants.Url.OSM_REGISTER));
     mProgress = view.findViewById(R.id.osm_login_progress);
+    final String dataVersion = DateUtils.getShortDateFormatter().format(Framework.getDataVersion());
 
     // TODO(@pastk): remove unused flow with users entering credentials into app's form
     // Hide login and password inputs and Forgot password button
@@ -163,7 +165,10 @@ public class OsmLoginFragment extends BaseMwmToolbarFragment
         // Finish OAuth2 auth flow and get username for UI.
         final String oauthToken = OsmOAuth.nativeAuthWithOAuth2Code(oauth2code);
         final String username = (oauthToken == null) ? null : OsmOAuth.nativeGetOsmUsername(oauthToken);
-        UiThread.run(() -> processAuth(oauthToken, username));
+        UiThread.run(() ->
+        {
+          processAuth(oauthToken, username);
+        });
       });
     }
   }

android/app/src/main/java/app/organicmaps/leftbutton/LeftButtonsHolder.java

@@ -10,6 +10,7 @@ import androidx.preference.PreferenceManager;
 import app.organicmaps.R;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 

android/app/src/main/java/app/organicmaps/maplayer/MapButtonsController.java

@@ -399,12 +399,15 @@ public class MapButtonsController extends Fragment
       final View button = entry.getValue();
       if (button.getParent() == parent)
       {
-        int toleranceOffset = switch (entry.getKey()) {
-            case zoomIn, zoomOut, zoom -> -140;
-            default -> 0;
-            // Allow offset tolerance for zoom buttons
-        };
-          showButton(getViewTopOffset(translation, button) >= toleranceOffset, entry.getKey());
+        int toleranceOffset = 0;
+        // Allow offset tolerance for zoom buttons
+        switch(entry.getKey())
+        {
+        case zoomIn: case zoomOut: case zoom:
+          toleranceOffset = -140;
+          break;
+        }
+        showButton(getViewTopOffset(translation, button) >= toleranceOffset, entry.getKey());
       }
     }
   }

android/app/src/main/java/app/organicmaps/routing/RoutingPlanController.java

@@ -12,6 +12,7 @@ import androidx.activity.result.ActivityResultLauncher;
 import androidx.annotation.DrawableRes;
 import androidx.annotation.IdRes;
 import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
 import androidx.core.view.ViewCompat;
 import androidx.core.view.WindowInsetsCompat;
 import app.organicmaps.sdk.Framework;
@@ -225,34 +226,37 @@ public class RoutingPlanController extends ToolbarController
   {
     UiUtils.invisible(mProgressVehicle, mProgressPedestrian, mProgressTransit,
                       mProgressBicycle, mProgressRuler);
-    WheelProgressView progressView = switch (router) {
-        case Vehicle -> {
-            mRouterTypes.check(R.id.vehicle);
-            yield mProgressVehicle;
-        }
-        case Pedestrian -> {
-            mRouterTypes.check(R.id.pedestrian);
-            yield mProgressPedestrian;
-        }
-        //case Taxi:
-        //    {
-        //      mRouterTypes.check(R.id.taxi);
-        //      progressView = mProgressTaxi;
-        //    }
-        case Transit -> {
-            mRouterTypes.check(R.id.transit);
-            yield mProgressTransit;
-        }
-        case Bicycle -> {
-            mRouterTypes.check(R.id.bicycle);
-            yield mProgressBicycle;
-        }
-        case Ruler -> {
-            mRouterTypes.check(R.id.ruler);
-            yield mProgressRuler;
-        }
-        default -> throw new IllegalArgumentException("unknown router: " + router);
-    };
+    WheelProgressView progressView;
+    switch (router)
+    {
+    case Vehicle:
+      mRouterTypes.check(R.id.vehicle);
+      progressView = mProgressVehicle;
+      break;
+    case Pedestrian:
+      mRouterTypes.check(R.id.pedestrian);
+      progressView = mProgressPedestrian;
+      break;
+    //case Taxi:
+    //    {
+    //      mRouterTypes.check(R.id.taxi);
+    //      progressView = mProgressTaxi;
+    //    }
+    case Transit:
+      mRouterTypes.check(R.id.transit);
+      progressView = mProgressTransit;
+      break;
+    case Bicycle:
+      mRouterTypes.check(R.id.bicycle);
+      progressView = mProgressBicycle;
+      break;
+    case Ruler:
+      mRouterTypes.check(R.id.ruler);
+      progressView = mProgressRuler;
+      break;
+    default:
+        throw new IllegalArgumentException("unknown router: " + router);
+    }
 
     RoutingToolbarButton button = mRouterTypes
         .findViewById(mRouterTypes.getCheckedRadioButtonId());

android/app/src/main/java/app/organicmaps/sdk/util/HttpClient.java

@@ -31,6 +31,7 @@ import androidx.annotation.NonNull;
 
 import app.organicmaps.downloader.Android7RootCertificateWorkaround;
 import app.organicmaps.sdk.util.log.Logger;
+import app.organicmaps.sdk.util.Utils;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;

android/app/src/main/java/app/organicmaps/search/CategoriesAdapter.java

@@ -5,6 +5,7 @@ import android.app.Activity;
 import android.content.Context;
 import android.content.res.Configuration;
 import android.content.res.Resources;
+import android.os.Build;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.view.ViewGroup;

android/app/src/main/java/app/organicmaps/settings/BackupSettingsFragment.java

@@ -2,6 +2,7 @@ package app.organicmaps.settings;
 
 import static app.organicmaps.backup.BackupUtils.formatReadableFolderPath;
 import static app.organicmaps.backup.BackupUtils.getMaxBackups;
+import static app.organicmaps.backup.BackupUtils.isBackupFolderAvailable;
 import static app.organicmaps.sdk.util.StorageUtils.isFolderWritable;
 
 import android.app.Activity;
@@ -354,11 +355,13 @@ public class BackupSettingsFragment
 
   private void showBackupErrorAlertDialog(String message)
   {
-    requireActivity().runOnUiThread(() -> new MaterialAlertDialogBuilder(requireActivity())
-        .setTitle(R.string.pref_backup_now_summary_failed)
-        .setMessage(message)
-        .setPositiveButton(android.R.string.ok, (dialog, which) -> dialog.dismiss())
-        .show());
+    requireActivity().runOnUiThread(() -> {
+      new MaterialAlertDialogBuilder(requireActivity())
+          .setTitle(R.string.pref_backup_now_summary_failed)
+          .setMessage(message)
+          .setPositiveButton(android.R.string.ok, (dialog, which) -> dialog.dismiss())
+          .show();
+    });
   }
 
   private void takePersistableUriPermission(Uri uri)

android/app/src/main/java/app/organicmaps/settings/SettingsPrefsFragment.java

@@ -24,6 +24,7 @@ import app.organicmaps.sdk.editor.OsmOAuth;
 import app.organicmaps.editor.LanguagesFragment;
 import app.organicmaps.editor.ProfileActivity;
 import app.organicmaps.sdk.editor.data.Language;
+import app.organicmaps.help.HelpActivity;
 import app.organicmaps.sdk.location.LocationHelper;
 import app.organicmaps.sdk.location.LocationProviderFactory;
 import app.organicmaps.sdk.routing.RoutingOptions;
@@ -43,6 +44,8 @@ import com.google.android.material.dialog.MaterialAlertDialogBuilder;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
 

android/app/src/main/java/app/organicmaps/util/bottomsheet/MenuAdapter.java

@@ -14,6 +14,7 @@ import com.google.android.material.textview.MaterialTextView;
 
 import app.organicmaps.R;
 import app.organicmaps.sdk.location.TrackRecorder;
+import app.organicmaps.sdk.util.Config;
 
 import java.util.ArrayList;