diff --git a/platform/chunks_download_strategy.cpp b/platform/chunks_download_strategy.cpp index 76f053ffd..bf36d7308 100644 --- a/platform/chunks_download_strategy.cpp +++ b/platform/chunks_download_strategy.cpp @@ -147,14 +147,14 @@ string ChunksDownloadStrategy::ChunkFinished(bool success, RangeT const & range) url = m_servers[s].m_url; if (success) { + LOG(LDEBUG, ("Completed chunk", m_servers[s].m_chunkIndex, "via", m_servers[s].m_url)); // mark server as free and chunk as ready m_servers[s].m_chunkIndex = SERVER_READY; res.first->m_status = CHUNK_COMPLETE; } else { - LOG(LINFO, ("Thread for url", m_servers[s].m_url, - "failed to download chunk number", m_servers[s].m_chunkIndex)); + LOG(LWARNING, ("Failed to dl chunk", m_servers[s].m_chunkIndex, "via", m_servers[s].m_url)); // remove failed server and mark chunk as free m_servers.erase(m_servers.begin() + s); res.first->m_status = CHUNK_FREE; @@ -201,6 +201,7 @@ ChunksDownloadStrategy::NextChunk(string & outUrl, RangeT & range) range.second = m_chunks[i+1].m_pos - 1; m_chunks[i].m_status = CHUNK_DOWNLOADING; + LOG(LDEBUG, ("Download chunk", server->m_chunkIndex, "via", outUrl)); return ENextChunk; case CHUNK_DOWNLOADING: diff --git a/platform/chunks_download_strategy.hpp b/platform/chunks_download_strategy.hpp index 6b7744505..9872f7c5b 100644 --- a/platform/chunks_download_strategy.hpp +++ b/platform/chunks_download_strategy.hpp @@ -65,6 +65,7 @@ public: void AddChunk(RangeT const & range, ChunkStatusT status); void SaveChunks(int64_t fileSize, std::string const & fName); + /// Inits the chunks list or loads from the resume file if there is an unfinished download. /// @return Already downloaded size. int64_t LoadOrInitChunks(std::string const & fName, int64_t fileSize, int64_t chunkSize); @@ -81,7 +82,7 @@ public: EDownloadFailed, EDownloadSucceeded }; - /// Should be called until returns ENextChunk + /// Get next chunk url ready to download. Should be called until returns ENextChunk. ResultT NextChunk(std::string & outUrl, RangeT & range); }; } // namespace downloader diff --git a/platform/http_request.cpp b/platform/http_request.cpp index 13e7eeab8..6f103d1de 100644 --- a/platform/http_request.cpp +++ b/platform/http_request.cpp @@ -139,6 +139,7 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback size_t m_goodChunksCount; bool m_doCleanProgressFiles; + // Starts a thread per each free/available server. ChunksDownloadStrategy::ResultT StartThreads() { string url; @@ -198,6 +199,7 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback } } + // Saves current chunks' statuses into a resume file. void SaveResumeChunks() { try diff --git a/platform/http_request.hpp b/platform/http_request.hpp index 2bc7390e3..d09398a90 100644 --- a/platform/http_request.hpp +++ b/platform/http_request.hpp @@ -53,6 +53,7 @@ public: Callback && onProgress = Callback()); /// Download file to filePath. + /// Pulls chunks simultaneously from all available servers, 1 thread per server. /// @param[in] fileSize Correct file size (needed for resuming and reserving). static HttpRequest * GetFile(std::vector const & urls, std::string const & filePath, int64_t fileSize, diff --git a/storage/map_files_downloader.cpp b/storage/map_files_downloader.cpp index 9dd98843e..ce511b2f3 100644 --- a/storage/map_files_downloader.cpp +++ b/storage/map_files_downloader.cpp @@ -166,6 +166,7 @@ MetaConfig MapFilesDownloader::LoadMetaConfig() if (!metaServerUrl.empty()) { + LOG(LINFO, ("Requesting metaserver", metaServerUrl)); platform::HttpClient request(metaServerUrl); request.SetRawHeader("X-OM-DataVersion", std::to_string(m_dataVersion)); request.SetRawHeader("X-OM-AppVersion", pl.Version()); @@ -179,7 +180,11 @@ MetaConfig MapFilesDownloader::LoadMetaConfig() { metaConfig = downloader::ParseMetaConfig(pl.DefaultUrlsJSON()); CHECK(metaConfig, ()); - LOG(LWARNING, ("Can't get meta configuration from request, using default servers:", metaConfig->m_serversList)); + LOG(LWARNING, ("Can't get metaserver configuration, using default servers:", metaConfig->m_serversList)); + } + else + { + LOG(LINFO, ("Got servers list:", metaConfig->m_serversList)); } CHECK(!metaConfig->m_serversList.empty(), ()); return *metaConfig; diff --git a/storage/map_files_downloader.hpp b/storage/map_files_downloader.hpp index 9f3c56755..b7aee2916 100644 --- a/storage/map_files_downloader.hpp +++ b/storage/map_files_downloader.hpp @@ -32,8 +32,9 @@ public: virtual ~MapFilesDownloader() = default; - /// Asynchronously downloads a map file, periodically invokes - /// onProgress callback and finally invokes onDownloaded + /// Asynchronously downloads a map file (first queries the metaserver + /// for the map servers list, if it haven't been done before), + /// periodically invokes onProgress callback and finally invokes onDownloaded /// callback. Both callbacks will be invoked on the main thread. void DownloadMapFile(QueuedCountry && queuedCountry); @@ -53,16 +54,19 @@ public: * @brief Async file download as string buffer (for small files only). * Request can be skipped if current servers list is empty. * Callback will be skipped on download error. + * NOTE: not in use at the moment. * @param[in] url Final url part like "index.json" or "maps/210415/countries.txt". * @param[in] forceReset True - force reset current request, if any. */ void DownloadAsString(std::string url, std::function && callback, bool forceReset = false); + // Used in tests only. void SetServersList(ServersList const & serversList); + void SetDownloadingPolicy(DownloadingPolicy * policy); void SetDataVersion(int64_t version) { m_dataVersion = version; } - /// @name Legacy functions for Android resourses downloading routine. + /// @name Legacy functions for Android resources downloading routine (initial World download). /// @{ void EnsureMetaConfigReady(std::function && callback); std::vector MakeUrlListLegacy(std::string const & fileName) const; @@ -70,9 +74,11 @@ public: protected: bool IsDownloadingAllowed() const; + // Produces download urls for all servers. std::vector MakeUrlList(std::string const & relativeUrl) const; - // Synchronously loads list of servers by http client. + // Synchronously loads a list of map servers from the metaserver. + // On dl failure fallbacks to the hardcoded list. MetaConfig LoadMetaConfig(); private: diff --git a/storage/map_files_downloader_with_ping.cpp b/storage/map_files_downloader_with_ping.cpp index 9fdc8ec0f..0767fb9df 100644 --- a/storage/map_files_downloader_with_ping.cpp +++ b/storage/map_files_downloader_with_ping.cpp @@ -16,6 +16,8 @@ void MapFilesDownloaderWithPing::GetMetaConfig(MetaConfigCallback const & callba CHECK(!metaConfig.m_serversList.empty(), ()); // Sort the list of servers by latency. + /// @todo(pastk:: actually the sort order is used only in MapFilesDownloader::DownloadAsString() + /// to get e.g. countries.txt but this code is not enabled at the moment. auto const sorted = Pinger::ExcludeUnavailableAndSortEndpoints(metaConfig.m_serversList); // Keep the original list if all servers are unavailable. if (!sorted.empty()) diff --git a/storage/pinger.cpp b/storage/pinger.cpp index dca6b0cda..b0fa28b54 100644 --- a/storage/pinger.cpp +++ b/storage/pinger.cpp @@ -24,6 +24,7 @@ int64_t DoPing(std::string const & url) return kInvalidPing; } + LOG(LDEBUG, ("Pinging server", url)); platform::HttpClient request(url); request.SetHttpMethod("HEAD"); request.SetTimeout(kTimeoutInSeconds); @@ -34,7 +35,7 @@ int64_t DoPing(std::string const & url) } else { - LOG(LWARNING, ("Request to server", url, "failed with code =", request.ErrorCode(), "; redirection =", request.WasRedirected())); + LOG(LWARNING, ("Ping to server", url, "failed with code =", request.ErrorCode(), "; wasRedirected =", request.WasRedirected())); } return kInvalidPing; diff --git a/storage/pinger.hpp b/storage/pinger.hpp index 6bc00a221..fe8245c4e 100644 --- a/storage/pinger.hpp +++ b/storage/pinger.hpp @@ -11,7 +11,7 @@ class Pinger { public: using Endpoints = std::vector; - // Returns list of available endpoints. Works synchronously. + // Pings all endpoints and a returns latency-sorted list of available ones. Works synchronously. static Endpoints ExcludeUnavailableAndSortEndpoints(Endpoints const & urls); }; } // namespace storage