New cpp folder structure

Signed-off-by: Alexander Borsuk <me@alex.bio>

libs/storage/storage_integration_tests/CMakeLists.txt

@@ -0,0 +1,21 @@
+project(storage_integration_tests)
+
+set(SRC
+  download_calc_size_test.cpp
+  lightweight_matching_tests.cpp
+  storage_3levels_tests.cpp
+  storage_downloading_tests.cpp
+  storage_group_download_tests.cpp
+  storage_http_tests.cpp
+  storage_update_tests.cpp
+  test_defines.cpp
+  test_defines.hpp
+)
+
+omim_add_test(${PROJECT_NAME} ${SRC} REQUIRE_QT REQUIRE_SERVER)
+
+target_link_libraries(${PROJECT_NAME}
+  platform_tests_support
+  storage
+  map
+)

libs/storage/storage_integration_tests/download_calc_size_test.cpp

@@ -0,0 +1,71 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+#include "storage/storage.hpp"
+
+#include "platform/downloader_defines.hpp"
+#include "platform/mwm_version.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+namespace download_calc_size_test
+{
+using namespace storage;
+void InitStorage(Storage & storage, Storage::UpdateCallback const & didDownload,
+                 Storage::ProgressFunction const & progress)
+{
+  auto const changeCountryFunction = [&](CountryId const & /* countryId */) {
+    if (!storage.IsDownloadInProgress())
+    {
+      // End wait for downloading complete.
+      testing::StopEventLoop();
+    }
+  };
+
+  storage.Init(didDownload, [](CountryId const &, LocalFilePtr const) { return false; });
+  storage.RegisterAllLocalMaps();
+  storage.Subscribe(changeCountryFunction, progress);
+  storage.SetDownloadingServersForTesting({kTestWebServer});
+}
+
+UNIT_TEST(DownloadingTests_CalcOverallProgress)
+{
+  WritableDirChanger writableDirChanger(storage::kMapTestDir);
+
+  // A bunch of small islands.
+  CountriesVec const kTestCountries = { "Kiribati", "Tokelau", "Niue", "Palau", "Pitcairn Islands" };
+
+  Storage s;
+
+  s.SetDownloadingServersForTesting({storage::kTestWebServer});
+  auto baseProgress = s.GetOverallProgress(kTestCountries);
+
+  TEST_EQUAL(baseProgress.m_bytesDownloaded, 0, ());
+  TEST_EQUAL(baseProgress.m_bytesTotal, 0, ());
+
+  for (auto const &country : kTestCountries)
+  {
+    baseProgress.m_bytesTotal += s.CountrySizeInBytes(country).second;
+  }
+
+  auto progressChanged = [&s, &kTestCountries, &baseProgress](CountryId const & id,
+                                                              downloader::Progress const & /* progress */) {
+    auto const currentProgress = s.GetOverallProgress(kTestCountries);
+    LOG_SHORT(LINFO, (id, "downloading progress:", currentProgress));
+
+    TEST_GREATER_OR_EQUAL(currentProgress.m_bytesDownloaded, baseProgress.m_bytesDownloaded, ());
+    baseProgress.m_bytesDownloaded = currentProgress.m_bytesDownloaded;
+
+    TEST_LESS_OR_EQUAL(currentProgress.m_bytesDownloaded, baseProgress.m_bytesTotal, ());
+    TEST_EQUAL(currentProgress.m_bytesTotal, baseProgress.m_bytesTotal, ());
+  };
+
+  InitStorage(s, [](storage::CountryId const &, LocalFilePtr const) {}, progressChanged);
+
+  for (auto const & countryId : kTestCountries)
+    s.DownloadNode(countryId);
+
+  testing::RunEventLoop();
+}
+}  // namespace download_calc_size_test

libs/storage/storage_integration_tests/lightweight_matching_tests.cpp

@@ -0,0 +1,51 @@
+#include "testing/testing.hpp"
+
+#include "storage/country_info_reader_light.hpp"
+#include "storage/storage_defines.hpp"
+
+#include "geometry/mercator.hpp"
+#include "geometry/point2d.hpp"
+
+#include <utility>
+#include <vector>
+
+namespace lightweight_matching_tests
+{
+double constexpr kStepInMercator = 1;
+
+struct PointAndCountry
+{
+  PointAndCountry(m2::PointD && pt, storage::CountryId && country)
+    : m_pt(std::move(pt)), m_country(std::move(country))
+  {
+  }
+
+  m2::PointD m_pt;
+  storage::CountryId m_country;
+};
+
+using lightweight::CountryInfoReader;
+
+UNIT_CLASS_TEST(CountryInfoReader, LightweightMatching)
+{
+  auto const reader = storage::CountryInfoReader::CreateCountryInfoReader(GetPlatform());
+
+  LOG(LINFO, ("Generating dataset..."));
+  std::vector<PointAndCountry> dataset;
+  for (auto x = mercator::Bounds::kMinX; x <= mercator::Bounds::kMaxX; x += kStepInMercator)
+  {
+    for (auto y = mercator::Bounds::kMinY; y <= mercator::Bounds::kMaxY; y += kStepInMercator)
+    {
+      m2::PointD pt(x, y);
+      dataset.emplace_back(std::move(pt), reader->GetRegionCountryId(pt));
+    }
+  }
+
+  LOG(LINFO, ("The dataset is generated. Dataset size:", dataset.size()));
+
+  for (auto const & sample : dataset)
+  {
+    TEST_EQUAL(GetRegionCountryId(sample.m_pt), sample.m_country, (sample.m_pt));
+  }
+}
+}  // namespace lightweight_matching_tests

libs/storage/storage_integration_tests/storage_3levels_tests.cpp

@@ -0,0 +1,100 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+#include "map/framework.hpp"
+
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/scope_guard.hpp"
+
+#include <algorithm>
+#include <string>
+
+#include "defines.hpp"
+
+using namespace platform;
+using namespace storage;
+
+namespace
+{
+
+int GetLevelCount(Storage & storage, CountryId const & countryId)
+{
+  CountriesVec children;
+  storage.GetChildren(countryId, children);
+  int level = 0;
+  for (auto const & child : children)
+    level = std::max(level, GetLevelCount(storage, child));
+  return 1 + level;
+}
+} // namespace
+
+UNIT_TEST(SmallMwms_3levels_Test)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  Platform & platform = GetPlatform();
+
+  /// @todo So sick, but Framework.RoutingManager has so complicated logic with a bunch of
+  /// RunOnGui callbacks, so delete Framework also in RunOnGui.
+  auto * frm = new Framework(FrameworkParams(false /* m_enableDiffs */));
+
+  SCOPE_GUARD(deleteFramework, [frm]()
+  {
+    GetPlatform().RunTask(Platform::Thread::Gui, [frm]()
+    {
+      delete frm;
+    });
+  });
+
+  auto & storage = frm->GetStorage();
+  std::string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  CountryId country = "Germany";
+  TEST_EQUAL(3, GetLevelCount(storage, country), ());
+
+  std::string const mapDir = base::JoinPath(platform.WritableDir(), version);
+
+  auto onProgressFn = [&](CountryId const & countryId, downloader::Progress const & /* progress */) {};
+
+  auto onChangeCountryFn = [&](CountryId const & countryId) {
+    if (!storage.IsDownloadInProgress())
+      testing::StopEventLoop();
+  };
+
+  storage.Subscribe(onChangeCountryFn, onProgressFn);
+  storage.SetDownloadingServersForTesting({kTestWebServer});
+
+  /// @todo Download all Germany > 2GB takes hours here ..
+  country = "Kiribati";
+
+  NodeAttrs attrs;
+  storage.GetNodeAttrs(country, attrs);
+  TEST_EQUAL(attrs.m_status, NodeStatus::NotDownloaded, ());
+
+  Platform::FilesList files;
+  platform.GetFilesByExt(mapDir, DATA_FILE_EXTENSION, files);
+  TEST_EQUAL(0, files.size(), ());
+
+  storage.DownloadNode(country);
+  testing::RunEventLoop();
+
+  storage.GetNodeAttrs(country, attrs);
+  TEST_EQUAL(attrs.m_status, NodeStatus::OnDisk, ());
+
+  files.clear();
+  platform.GetFilesByExt(mapDir, DATA_FILE_EXTENSION, files);
+  TEST_GREATER(files.size(), 0, ());
+
+  storage.DeleteNode(country);
+
+  storage.GetNodeAttrs(country, attrs);
+  TEST_EQUAL(attrs.m_status, NodeStatus::NotDownloaded, ());
+
+  files.clear();
+  platform.GetFilesByExt(mapDir, DATA_FILE_EXTENSION, files);
+  TEST_EQUAL(0, files.size(), ());
+}

libs/storage/storage_integration_tests/storage_downloading_tests.cpp

@@ -0,0 +1,232 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+#include "storage/storage.hpp"
+
+#include "platform/local_country_file_utils.hpp"
+#include "platform/mwm_version.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/scoped_dir.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+#include "coding/file_writer.hpp"
+#include "coding/sha1.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/scope_guard.hpp"
+#include "base/string_utils.hpp"
+#include "base/thread.hpp"
+
+#include <cstdlib>
+#include <functional>
+#include <exception>
+#include <string>
+
+using namespace platform;
+using namespace storage;
+using namespace std;
+using namespace std::placeholders;
+
+// Uncomment to enable the test that requires network and downloads an mwm several times.
+//#define TEST_INTEGRITY
+#ifndef TEST_INTEGRITY_ITERATIONS
+#define TEST_INTEGRITY_ITERATIONS 5
+#endif
+
+namespace
+{
+using Runner = Platform::ThreadRunner;
+
+string const kCountryId = "Trinidad and Tobago";
+
+class InterruptException : public exception {};
+
+void Update(CountryId const &, storage::LocalFilePtr const localCountryFile)
+{
+  TEST_EQUAL(localCountryFile->GetCountryName(), kCountryId, ());
+}
+
+void ChangeCountry(Storage & storage, CountryId const & countryId)
+{
+  TEST_EQUAL(countryId, kCountryId, ());
+
+  if (!storage.IsDownloadInProgress())
+    testing::StopEventLoop();
+}
+
+void InitStorage(Storage & storage, Storage::ProgressFunction const & onProgressFn)
+{
+  storage.Init(Update, [](CountryId const &, storage::LocalFilePtr const) { return false; });
+  storage.RegisterAllLocalMaps();
+  storage.Subscribe(bind(&ChangeCountry, ref(storage), _1), onProgressFn);
+  storage.SetDownloadingServersForTesting({kTestWebServer});
+}
+
+} // namespace
+
+UNIT_TEST(SmallMwms_ReDownloadExistedMWMIgnored_Test)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+  Storage storage;
+
+  InitStorage(storage, [](CountryId const &, downloader::Progress const &) {});
+  TEST(!storage.IsDownloadInProgress(), ());
+
+  storage.DownloadNode(kCountryId);
+  TEST(storage.IsDownloadInProgress(), ());
+  testing::RunEventLoop();
+
+  TEST(!storage.IsDownloadInProgress(), ());
+  storage.DownloadNode(kCountryId);
+  TEST(!storage.IsDownloadInProgress(), ());
+}
+
+UNIT_CLASS_TEST(Runner, SmallMwms_InterruptDownloadResumeDownload_Test)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  // Start download but interrupt it
+  {
+    Storage storage;
+
+    auto const onProgressFn = [](CountryId const & countryId, downloader::Progress const & /* progress */) {
+      TEST_EQUAL(countryId, kCountryId, ());
+      // Interrupt download
+      testing::StopEventLoop();
+    };
+
+    InitStorage(storage, onProgressFn);
+
+    TEST(!storage.IsDownloadInProgress(), ());
+
+    storage.DownloadNode(kCountryId);
+    testing::RunEventLoop();
+
+    TEST(storage.IsDownloadInProgress(), ());
+
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(kCountryId, attrs);
+    TEST_EQUAL(NodeStatus::Downloading, attrs.m_status, ());
+  }
+
+  // Continue download
+  {
+    Storage storage;
+
+    bool onProgressIsCalled = false;
+    NodeAttrs onProgressAttrs;
+    auto const onProgressFn =
+      [&](CountryId const & countryId, downloader::Progress const & /* progress */)
+      {
+        TEST_EQUAL(countryId, kCountryId, ());
+
+        if (onProgressIsCalled)
+          return;
+
+        onProgressIsCalled = true;
+        storage.GetNodeAttrs(kCountryId, onProgressAttrs);
+        testing::StopEventLoop();
+      };
+
+    InitStorage(storage, onProgressFn);
+    storage.Init([](CountryId const &, storage::LocalFilePtr const localCountryFile)
+                 {
+                   TEST_EQUAL(localCountryFile->GetCountryName(), kCountryId, ());
+
+                   testing::StopEventLoop();
+                 },
+                 [](CountryId const &, storage::LocalFilePtr const)
+                 {
+                   return false;
+                 });
+
+    testing::RunEventLoop();
+
+    TEST(storage.IsDownloadInProgress(), ());
+
+    testing::RunEventLoop();
+
+    TEST_EQUAL(NodeStatus::Downloading, onProgressAttrs.m_status, ());
+
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(kCountryId, attrs);
+    TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
+  }
+}
+
+#ifdef TEST_INTEGRITY
+UNIT_CLASS_TEST(Runner, DownloadIntegrity_Test)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  string mapPath;
+  coding::SHA1::Hash mapHash;
+  {
+    SCOPE_GUARD(deleteTestFileGuard, bind(&FileWriter::DeleteFileX, ref(mapPath)));
+
+    Storage storage(COUNTRIES_FILE);
+
+    InitStorage(storage, [](CountryId const & countryId, LocalAndRemoteSize const & mapSize) {});
+    TEST(!storage.IsDownloadInProgress(), ());
+
+    storage.DownloadNode(kCountryId);
+    TEST(storage.IsDownloadInProgress(), ());
+    testing::RunEventLoop();
+
+    auto localFile = storage.GetLatestLocalFile(kCountryId);
+    mapPath = localFile->GetPath(MapFileType::Map);
+    mapHash = coding::SHA1::Calculate(mapPath);
+  }
+  TEST_NOT_EQUAL(mapHash, coding::SHA1::Hash(), ());
+
+  uint32_t constexpr kIterationsCount = TEST_INTEGRITY_ITERATIONS;
+  for (uint32_t i = 0; i < kIterationsCount; ++i)
+  {
+    // Downloading with interruption.
+    uint32_t constexpr kInterruptionsCount = 10;
+    for (uint32_t j = 0; j < kInterruptionsCount; ++j)
+    {
+      SCOPE_GUARD(deleteTestFileGuard, bind(&FileWriter::DeleteFileX, ref(mapPath)));
+
+      Storage storage(COUNTRIES_FILE);
+
+      auto onProgressFn = [i, j](CountryId const & countryId, LocalAndRemoteSize const & mapSize) {
+        TEST_EQUAL(countryId, kCountryId, ());
+        auto progress = static_cast<double>(mapSize.first) / mapSize.second;
+        auto interruptionProgress =
+            0.1 + 0.75 * static_cast<double>((i + j) % kInterruptionsCount) / kInterruptionsCount;
+        if (progress > interruptionProgress)
+          testing::StopEventLoop();
+      };
+
+      InitStorage(storage, onProgressFn);
+      storage.DownloadNode(kCountryId);
+      testing::RunEventLoop();
+      TEST(storage.IsDownloadInProgress(), ());
+    }
+
+    // Continue downloading.
+    coding::SHA1::Hash newHash;
+    {
+      Storage storage(COUNTRIES_FILE);
+
+      InitStorage(storage, [](CountryId const & countryId, LocalAndRemoteSize const & mapSize) {});
+      TEST(storage.IsDownloadInProgress(), ());
+
+      NodeAttrs attrs;
+      storage.GetNodeAttrs(kCountryId, attrs);
+      TEST_EQUAL(NodeStatus::Downloading, attrs.m_status, ());
+
+      storage.DownloadNode(kCountryId);
+      TEST(storage.IsDownloadInProgress(), ());
+      testing::RunEventLoop();
+
+      newHash = coding::SHA1::Calculate(mapPath);
+    }
+
+    // Check hashes.
+    TEST_EQUAL(mapHash, newHash, ());
+  }
+}
+#endif

libs/storage/storage_integration_tests/storage_group_download_tests.cpp

@@ -0,0 +1,294 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+#include "map/framework.hpp"
+
+#include "platform/http_request.hpp"
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/scoped_dir.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+#include "storage/storage.hpp"
+
+#include "base/assert.hpp"
+#include "base/file_name_utils.hpp"
+
+#include <string>
+
+namespace storage_group_download_tests
+{
+using namespace platform;
+using namespace std;
+using namespace storage;
+
+CountryId const kGroupCountryId = "Venezuela";
+CountriesSet const kLeafCountriesIds = { "Venezuela_North", "Venezuela_South" };
+
+string GetMwmFilePath(string const & version, CountryId const & countryId)
+{
+  return base::JoinPath(GetPlatform().WritableDir(), version, countryId + DATA_FILE_EXTENSION);
+}
+
+string GetMwmDownloadingFilePath(string const & version, CountryId const & countryId)
+{
+  return base::JoinPath(
+      GetPlatform().WritableDir(), version,
+      countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION);
+}
+
+string GetMwmResumeFilePath(string const & version, CountryId const & countryId)
+{
+  return base::JoinPath(GetPlatform().WritableDir(), version,
+                        countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION);
+}
+
+void DownloadGroup(Storage & storage, bool oneByOne)
+{
+  Platform & platform = GetPlatform();
+
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  //  Get children nodes for the group node.
+  CountriesVec children;
+  //  All nodes in subtree (including the root) for the group node.
+  storage.GetChildren(kGroupCountryId, children);
+  CountriesSet subTree;
+  storage.ForEachInSubtree(kGroupCountryId,
+                           [&subTree](CountryId const & descendantId, bool /* groupNode */) {
+                             subTree.insert(descendantId);
+                           });
+
+  CountriesSet changed;
+  auto onChangeCountryFn = [&](CountryId const & countryId) {
+    TEST(subTree.find(countryId) != subTree.end(), (countryId));
+    changed.insert(countryId);
+    if (!storage.IsDownloadInProgress())
+    {
+      // Stop waiting when all chilren will be downloaded.
+      testing::StopEventLoop();
+    }
+  };
+
+  CountriesSet downloadedChecker;
+  auto onProgressFn = [&](CountryId const & countryId, downloader::Progress const & progress) {
+    TEST(subTree.find(countryId) != subTree.end(), ());
+    if (progress.m_bytesDownloaded == progress.m_bytesTotal)
+    {
+      auto const res = downloadedChecker.insert(countryId);
+      TEST_EQUAL(res.second, true, ()); // Every child is downloaded only once.
+    }
+  };
+
+  int const subsrcibtionId = storage.Subscribe(onChangeCountryFn, onProgressFn);
+
+  // Check group node is not downloaded
+  CountriesVec downloaded, available;
+  storage.GetChildrenInGroups(storage.GetRootId(), downloaded, available);
+  TEST(downloaded.empty(), ());
+
+  // Check children nodes are not downloaded
+  storage.GetChildrenInGroups(kGroupCountryId, downloaded, available);
+  TEST(downloaded.empty(), ());
+
+  // Check status for the all children nodes is set to NotDownloaded.
+  MwmSize totalGroupSize = 0;
+  for (auto const & countryId : children)
+  {
+    TEST_EQUAL(Status::NotDownloaded, storage.CountryStatusEx(countryId), ());
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(countryId, attrs);
+    TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
+    TEST_GREATER(attrs.m_mwmSize, 0, ());
+    totalGroupSize += attrs.m_mwmSize;
+  }
+
+  // Check status for the group node is set to NotDownloaded.
+  NodeAttrs attrs;
+  storage.GetNodeAttrs(kGroupCountryId, attrs);
+  TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
+  TEST_EQUAL(attrs.m_mwmSize, totalGroupSize, ());
+  attrs = NodeAttrs();
+
+  // Check there is no mwm or any other files for the children nodes.
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
+    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
+    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+  }
+
+  // Download the group.
+  if (oneByOne)
+  {
+    for (auto const & countryId : children)
+      storage.DownloadNode(countryId);
+  }
+  else
+  {
+    storage.DownloadNode(kGroupCountryId);
+  }
+  // Wait for downloading of all children.
+  testing::RunEventLoop();
+
+  // Check if all nodes in the subtree have been downloaded and changed.
+  TEST_EQUAL(changed, subTree, ());
+  TEST_EQUAL(downloadedChecker, subTree, ());
+
+  // Check status for the group node is set to OnDisk.
+  storage.GetNodeAttrs(kGroupCountryId, attrs);
+  TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
+
+  // Check status for the all children nodes is set to OnDisk.
+  for (auto const & countryId : children)
+  {
+    TEST_EQUAL(Status::OnDisk, storage.CountryStatusEx(countryId), ());
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(countryId, attrs);
+    TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
+  }
+
+  // Check there is only mwm files are present and no any other for the children nodes.
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
+    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
+    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+  }
+
+  // Check group is downloaded.
+  storage.GetChildrenInGroups(storage.GetRootId(), downloaded, available);
+  TEST_EQUAL(downloaded, CountriesVec({kGroupCountryId}), ());
+
+  // Check all group children are downloaded.
+  storage.GetChildrenInGroups(kGroupCountryId, downloaded, available);
+  TEST_EQUAL(CountriesSet(children.begin(), children.end()),
+             CountriesSet(downloaded.begin(), downloaded.end()), ());
+
+  storage.Unsubscribe(subsrcibtionId);
+}
+
+void DeleteGroup(Storage & storage, bool oneByOne)
+{
+  Platform & platform = GetPlatform();
+
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  //  Get children nodes for the group node.
+  CountriesVec v;
+  storage.GetChildren(kGroupCountryId, v);
+  CountriesSet const children(v.begin(), v.end());
+  v.clear();
+
+  // Check group node is downloaded.
+  CountriesVec downloaded, available;
+  storage.GetChildrenInGroups(storage.GetRootId(), downloaded, available);
+  TEST_EQUAL(downloaded, CountriesVec({kGroupCountryId}), ());
+
+  // Check children nodes are downloaded.
+  storage.GetChildrenInGroups(kGroupCountryId, downloaded, available);
+  TEST_EQUAL(children, CountriesSet(downloaded.begin(), downloaded.end()), ());
+
+  // Check there are mwm files for the children nodes.
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+  }
+
+  // Delete the group
+  if (oneByOne)
+  {
+    for (auto const & countryId : children)
+      storage.DeleteNode(countryId);
+  }
+  else
+  {
+    storage.DeleteNode(kGroupCountryId);
+  }
+
+  // Check state for the group node is set to NotDownloaded and NoError.
+  NodeAttrs attrs;
+  storage.GetNodeAttrs(kGroupCountryId, attrs);
+  TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
+
+  // Check state for the all children nodes is set to NotDownloaded and NoError.
+  for (auto const & countryId : children)
+  {
+    TEST_EQUAL(Status::NotDownloaded, storage.CountryStatusEx(countryId), ());
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(countryId, attrs);
+    TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
+  }
+
+  // Check there are no mwm files for the children nodes.
+  for (auto const & countryId : children)
+  {
+    string const mwmFullPath = GetMwmFilePath(version, countryId);
+    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
+  }
+
+  // Check group is not downloaded.
+  storage.GetChildrenInGroups(storage.GetRootId(), downloaded, available);
+  TEST(downloaded.empty(), ());
+
+  // Check all children nodes are not downloaded.
+  storage.GetChildrenInGroups(kGroupCountryId, downloaded, available);
+  TEST(downloaded.empty(), ());
+}
+
+void TestDownloadDelete(bool downloadOneByOne, bool deleteOneByOne)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  Storage storage;
+  string const version = strings::to_string(storage.GetCurrentDataVersion());
+
+  auto onUpdatedFn = [&](CountryId const &, storage::LocalFilePtr const localCountryFile) {
+    CountryId const countryId = localCountryFile->GetCountryName();
+    TEST(kLeafCountriesIds.find(countryId) != kLeafCountriesIds.end(), ());
+  };
+
+  storage.Init(onUpdatedFn, [](CountryId const &, storage::LocalFilePtr const) { return false; });
+  storage.RegisterAllLocalMaps();
+  storage.SetDownloadingServersForTesting({kTestWebServer});
+
+  tests_support::ScopedDir cleanupVersionDir(version);
+
+  // Check children for the kGroupCountryId
+  CountriesVec children;
+  storage.GetChildren(kGroupCountryId, children);
+  TEST_EQUAL(CountriesSet(children.begin(), children.end()), kLeafCountriesIds, ());
+
+  DownloadGroup(storage, downloadOneByOne);
+
+  DeleteGroup(storage, deleteOneByOne);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test1)
+{
+  TestDownloadDelete(false, false);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test2)
+{
+  TestDownloadDelete(false, true);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test3)
+{
+  TestDownloadDelete(true, false);
+}
+
+UNIT_TEST(SmallMwms_GroupDownloadDelete_Test4)
+{
+  TestDownloadDelete(true, true);
+}
+} // namespace storage_group_download_tests

libs/storage/storage_integration_tests/storage_http_tests.cpp

@@ -0,0 +1,185 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+#include "storage/storage.hpp"
+
+#include "platform/local_country_file_utils.hpp"
+#include "platform/mwm_version.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/scoped_dir.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/scope_guard.hpp"
+#include "base/string_utils.hpp"
+#include "base/thread.hpp"
+
+#include <string>
+
+namespace storage_http_tests
+{
+using namespace platform;
+using namespace std;
+using namespace storage;
+
+string const kCountryId = "Trinidad and Tobago";
+string const kDisputedCountryId1 = "Jerusalem";
+string const kDisputedCountryId2 = "Crimea";
+string const kDisputedCountryId3 = "Campo de Hielo Sur";
+string const kUndisputedCountryId = "Argentina_Buenos Aires_North";
+
+void Update(CountryId const &, LocalFilePtr const localCountryFile)
+{
+  TEST_EQUAL(localCountryFile->GetCountryName(), kCountryId, ());
+}
+
+void UpdateWithoutChecks(CountryId const &, LocalFilePtr const /* localCountryFile */) {}
+
+string const GetMwmFullPath(string const & countryId, string const & version)
+{
+  return base::JoinPath(GetPlatform().WritableDir(), version, countryId + DATA_FILE_EXTENSION);
+}
+
+string const GetDownloadingFullPath(string const & countryId, string const & version)
+{
+  return base::JoinPath(
+      GetPlatform().WritableDir(), version,
+      kCountryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION);
+}
+
+string const GetResumeFullPath(string const & countryId, string const & version)
+{
+  return base::JoinPath(
+      GetPlatform().WritableDir(), version,
+      kCountryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION);
+}
+
+void InitStorage(Storage & storage, Storage::UpdateCallback const & didDownload,
+                 Storage::ProgressFunction const & progress)
+{
+  auto const changeCountryFunction = [&](CountryId const & /* countryId */) {
+    if (!storage.IsDownloadInProgress())
+    {
+      // End wait for downloading complete.
+      testing::StopEventLoop();
+    }
+  };
+
+  storage.Init(didDownload, [](CountryId const &, LocalFilePtr const) { return false; });
+  storage.RegisterAllLocalMaps();
+  storage.Subscribe(changeCountryFunction, progress);
+  storage.SetDownloadingServersForTesting({kTestWebServer});
+}
+
+class StorageHttpTest
+{
+public:
+  StorageHttpTest()
+    : m_writableDirChanger(kMapTestDir),
+      m_version(strings::to_string(m_storage.GetCurrentDataVersion())),
+      m_cleanupVersionDir(m_version)
+  {
+  }
+
+protected:
+  WritableDirChanger const m_writableDirChanger;
+  Storage m_storage;
+  string const m_version;
+  tests_support::ScopedDir const m_cleanupVersionDir;
+};
+
+UNIT_CLASS_TEST(StorageHttpTest, StorageDownloadNodeAndDeleteNode)
+{
+  auto const progressFunction = [this](CountryId const & countryId,
+                                       downloader::Progress const & progress) {
+    NodeAttrs nodeAttrs;
+    m_storage.GetNodeAttrs(countryId, nodeAttrs);
+
+    TEST_EQUAL(progress.m_bytesDownloaded,
+               nodeAttrs.m_downloadingProgress.m_bytesDownloaded, (countryId));
+    TEST_EQUAL(progress.m_bytesTotal,
+               nodeAttrs.m_downloadingProgress.m_bytesTotal, (countryId));
+    TEST_EQUAL(countryId, kCountryId, (countryId));
+  };
+
+  InitStorage(m_storage, Update, progressFunction);
+
+  string const mwmFullPath = GetMwmFullPath(kCountryId, m_version);
+  string const downloadingFullPath = GetDownloadingFullPath(kCountryId, m_version);
+  string const resumeFullPath = GetResumeFullPath(kCountryId, m_version);
+
+  // Downloading to an empty directory.
+  m_storage.DownloadNode(kCountryId);
+  // Wait for downloading complete.
+  testing::RunEventLoop();
+
+  Platform & platform = GetPlatform();
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+
+  // Downloading to directory with Angola.mwm.
+  m_storage.DownloadNode(kCountryId);
+
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
+  TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
+
+  m_storage.DeleteNode(kCountryId);
+  TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
+}
+
+UNIT_CLASS_TEST(StorageHttpTest, StorageDownloadAndDeleteDisputedNode)
+{
+  auto const progressFunction = [this](CountryId const & countryId,
+                                       downloader::Progress const & progress) {
+    NodeAttrs nodeAttrs;
+    m_storage.GetNodeAttrs(countryId, nodeAttrs);
+
+    TEST_EQUAL(progress.m_bytesDownloaded,
+               nodeAttrs.m_downloadingProgress.m_bytesDownloaded, (countryId));
+    TEST_EQUAL(progress.m_bytesTotal,
+               nodeAttrs.m_downloadingProgress.m_bytesTotal, (countryId));
+  };
+
+  InitStorage(m_storage, UpdateWithoutChecks, progressFunction);
+
+  string const mwmFullPath1 = GetMwmFullPath(kDisputedCountryId1, m_version);
+  string const mwmFullPath2 = GetMwmFullPath(kDisputedCountryId2, m_version);
+  string const mwmFullPath3 = GetMwmFullPath(kDisputedCountryId3, m_version);
+  string const mwmFullPathUndisputed = GetMwmFullPath(kUndisputedCountryId, m_version);
+
+  // Downloading to an empty directory.
+  m_storage.DownloadNode(kDisputedCountryId1);
+  m_storage.DownloadNode(kDisputedCountryId2);
+  m_storage.DownloadNode(kDisputedCountryId3);
+  m_storage.DownloadNode(kUndisputedCountryId);
+  // Wait for downloading complete.
+  testing::RunEventLoop();
+
+  Platform & platform = GetPlatform();
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath1), ());
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath2), ());
+  TEST(platform.IsFileExistsByFullPath(mwmFullPath3), ());
+  TEST(platform.IsFileExistsByFullPath(mwmFullPathUndisputed), ());
+
+  CountriesVec downloadedChildren;
+  CountriesVec availChildren;
+  m_storage.GetChildrenInGroups(m_storage.GetRootId(), downloadedChildren, availChildren);
+
+  CountriesVec const expectedDownloadedChildren = {"Argentina", kDisputedCountryId2,
+                                                   kDisputedCountryId1};
+  TEST_EQUAL(downloadedChildren, expectedDownloadedChildren, ());
+  TEST_EQUAL(availChildren.size(), 223, ());
+
+  m_storage.DeleteNode(kDisputedCountryId1);
+  m_storage.DeleteNode(kDisputedCountryId2);
+  m_storage.DeleteNode(kDisputedCountryId3);
+  m_storage.DeleteNode(kUndisputedCountryId);
+  TEST(!platform.IsFileExistsByFullPath(mwmFullPath1), ());
+  TEST(!platform.IsFileExistsByFullPath(mwmFullPath2), ());
+  TEST(!platform.IsFileExistsByFullPath(mwmFullPath3), ());
+  TEST(!platform.IsFileExistsByFullPath(mwmFullPathUndisputed), ());
+}
+}  // namespace storage_http_tests

libs/storage/storage_integration_tests/storage_update_tests.cpp

@@ -0,0 +1,193 @@
+#include "testing/testing.hpp"
+
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+#include "map/framework.hpp"
+
+#include "platform/downloader_defines.hpp"
+#include "platform/http_request.hpp"
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+#include "coding/internal/file_data.hpp"
+
+#include "storage/storage.hpp"
+
+#include "base/file_name_utils.hpp"
+
+#include <memory>
+#include <string>
+
+namespace storage_update_tests
+{
+using namespace platform;
+using namespace std;
+using namespace storage;
+
+static FrameworkParams const kFrameworkParams(false /* m_enableDiffs */);
+
+string const kCountriesTxtFile = COUNTRIES_FILE;
+
+string const kMwmVersion1 = "190830";
+//size_t const kCountriesTxtFileSize1 = 420632;
+
+string const kMwmVersion2 = "190910";
+//size_t const kCountriesTxtFileSize2 = 420634;
+
+string const kGroupCountryId = "Belarus";
+
+bool DownloadFile(string const & url,
+                  string const & filePath,
+                  size_t fileSize)
+{
+  using namespace downloader;
+
+  DownloadStatus httpStatus;
+  bool finished = false;
+
+  unique_ptr<HttpRequest> request(HttpRequest::GetFile({url}, filePath, fileSize,
+                                  [&](HttpRequest & request)
+  {
+    DownloadStatus const s = request.GetStatus();
+    if (s != DownloadStatus::InProgress)
+    {
+      httpStatus = s;
+      finished = true;
+      testing::StopEventLoop();
+    }
+  }));
+
+  testing::RunEventLoop();
+
+  return httpStatus == DownloadStatus::Completed;
+}
+
+string GetCountriesTxtWebUrl(string const version)
+{
+  return kTestWebServer + "direct/" + version + "/" + kCountriesTxtFile;
+}
+
+string GetCountriesTxtFilePath()
+{
+  return base::JoinPath(GetPlatform().WritableDir(), kCountriesTxtFile);
+}
+
+string GetMwmFilePath(string const & version, CountryId const & countryId)
+{
+  return base::JoinPath(GetPlatform().WritableDir(), version, countryId + DATA_FILE_EXTENSION);
+}
+
+/// @todo We don't have direct version links for now.
+/// Also Framework f(kFrameworkParams) will fail here, @see SmallMwms_3levels_Test.
+/*
+UNIT_TEST(SmallMwms_Update_Test)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+
+  Platform & platform = GetPlatform();
+
+  auto onProgressFn = [&](CountryId const &, downloader::Progress const &) {};
+
+  // Download countries.txt for version 1
+  TEST(DownloadFile(GetCountriesTxtWebUrl(kMwmVersion1), GetCountriesTxtFilePath(), kCountriesTxtFileSize1), ());
+
+  {
+    Framework f(kFrameworkParams);
+    auto & storage = f.GetStorage();
+    string const version = strings::to_string(storage.GetCurrentDataVersion());
+    TEST_EQUAL(version, kMwmVersion1, ());
+    auto onChangeCountryFn = [&](CountryId const & countryId) {
+      if (!storage.IsDownloadInProgress())
+        testing::StopEventLoop();
+    };
+    storage.Subscribe(onChangeCountryFn, onProgressFn);
+    storage.SetDownloadingServersForTesting({kTestWebServer});
+
+    CountriesVec children;
+    storage.GetChildren(kGroupCountryId, children);
+
+    // Download group
+    storage.DownloadNode(kGroupCountryId);
+    testing::RunEventLoop();
+
+    // Check group node status is OnDisk
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(kGroupCountryId, attrs);
+    TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
+
+    // Check mwm files for version 1 are present
+    for (auto const & child : children)
+    {
+      string const mwmFullPathV1 = GetMwmFilePath(kMwmVersion1, child);
+      TEST(platform.IsFileExistsByFullPath(mwmFullPathV1), ());
+    }
+  }
+
+  // Replace countries.txt by version 2
+  TEST(base::DeleteFileX(GetCountriesTxtFilePath()), ());
+  TEST(DownloadFile(GetCountriesTxtWebUrl(kMwmVersion2), GetCountriesTxtFilePath(), kCountriesTxtFileSize2), ());
+
+  {
+    Framework f(kFrameworkParams);
+    auto & storage = f.GetStorage();
+    string const version = strings::to_string(storage.GetCurrentDataVersion());
+    TEST_EQUAL(version, kMwmVersion2, ());
+    auto onChangeCountryFn = [&](CountryId const & countryId) {
+      if (!storage.IsDownloadInProgress())
+        testing::StopEventLoop();
+    };
+    storage.Subscribe(onChangeCountryFn, onProgressFn);
+    storage.SetDownloadingServersForTesting({kTestWebServer});
+
+    CountriesVec children;
+    storage.GetChildren(kGroupCountryId, children);
+
+    // Check group node status is OnDiskOutOfDate
+    NodeAttrs attrs;
+    storage.GetNodeAttrs(kGroupCountryId, attrs);
+    TEST_EQUAL(NodeStatus::OnDiskOutOfDate, attrs.m_status, ());
+
+    // Check children node status is OnDiskOutOfDate
+    for (auto const & child : children)
+    {
+      NodeAttrs attrs;
+      storage.GetNodeAttrs(child, attrs);
+      TEST_EQUAL(NodeStatus::OnDiskOutOfDate, attrs.m_status, ());
+    }
+
+    // Check mwm files for version 1 are present
+    for (auto const & child : children)
+    {
+      string const mwmFullPathV1 = GetMwmFilePath(kMwmVersion1, child);
+      TEST(platform.IsFileExistsByFullPath(mwmFullPathV1), ());
+    }
+
+    // Download group, new version
+    storage.DownloadNode(kGroupCountryId);
+    testing::RunEventLoop();
+
+    // Check group node status is OnDisk
+    storage.GetNodeAttrs(kGroupCountryId, attrs);
+    TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
+
+    // Check children node status is OnDisk
+    for (auto const & child : children)
+    {
+      NodeAttrs attrs;
+      storage.GetNodeAttrs(child, attrs);
+      TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
+    }
+
+    // Check mwm files for version 2 are present and not present for version 1
+    for (auto const & child : children)
+    {
+      string const mwmFullPathV1 = GetMwmFilePath(kMwmVersion1, child);
+      string const mwmFullPathV2 = GetMwmFilePath(kMwmVersion2, child);
+      TEST(platform.IsFileExistsByFullPath(mwmFullPathV2), ());
+      TEST(!platform.IsFileExistsByFullPath(mwmFullPathV1), ());
+    }
+  }
+}
+*/
+}  // namespace storage_update_tests

libs/storage/storage_integration_tests/test_defines.cpp

@@ -0,0 +1,7 @@
+#include "storage/storage_integration_tests/test_defines.hpp"
+
+namespace storage
+{
+std::string const kMapTestDir = "map-tests";
+std::string const kTestWebServer = "https://cdn.comaps.app";
+}  // namespace storage

libs/storage/storage_integration_tests/test_defines.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <string>
+
+namespace storage
+{
+extern std::string const kMapTestDir;
+extern std::string const kTestWebServer;
+}  // namespace storage