drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2025-12-19 13:03:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

[core] Add more logging to the downloader

Browse Source 
Signed-off-by: Konstantin Pastbin <konstantin.pastbin@gmail.com>
This commit is contained in:
Konstantin Pastbin
2025-07-02 21:17:26 +07:00
committed by Konstantin Pastbin
parent 6d0c52afcc
commit ec1d2e6c82
9 changed files with 29 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
platform/chunks_download_strategy.cpp
View File

@@ -147,14 +147,14 @@ string ChunksDownloadStrategy::ChunkFinished(bool success, RangeT const & range)
url = m_servers[s].m_url;
if (success)
{
LOG(LDEBUG, ("Completed chunk", m_servers[s].m_chunkIndex, "via", m_servers[s].m_url));
// mark server as free and chunk as ready
m_servers[s].m_chunkIndex = SERVER_READY;
res.first->m_status = CHUNK_COMPLETE;
}
else
{
LOG(LINFO, ("Thread for url", m_servers[s].m_url,
"failed to download chunk number", m_servers[s].m_chunkIndex));
LOG(LWARNING, ("Failed to dl chunk", m_servers[s].m_chunkIndex, "via", m_servers[s].m_url));
// remove failed server and mark chunk as free
m_servers.erase(m_servers.begin() + s);
res.first->m_status = CHUNK_FREE;
@@ -201,6 +201,7 @@ ChunksDownloadStrategy::NextChunk(string & outUrl, RangeT & range)
range.second = m_chunks[i+1].m_pos - 1;
m_chunks[i].m_status = CHUNK_DOWNLOADING;
LOG(LDEBUG, ("Download chunk", server->m_chunkIndex, "via", outUrl));
return ENextChunk;
case CHUNK_DOWNLOADING:

3
platform/chunks_download_strategy.hpp
View File

@@ -65,6 +65,7 @@ public:
void AddChunk(RangeT const & range, ChunkStatusT status);
void SaveChunks(int64_t fileSize, std::string const & fName);
/// Inits the chunks list or loads from the resume file if there is an unfinished download.
/// @return Already downloaded size.
int64_t LoadOrInitChunks(std::string const & fName, int64_t fileSize, int64_t chunkSize);
@@ -81,7 +82,7 @@ public:
EDownloadFailed,
EDownloadSucceeded
};
/// Should be called until returns ENextChunk
/// Get next chunk url ready to download. Should be called until returns ENextChunk.
ResultT NextChunk(std::string & outUrl, RangeT & range);
};
} // namespace downloader

2
platform/http_request.cpp
View File

@@ -139,6 +139,7 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
size_t m_goodChunksCount;
bool m_doCleanProgressFiles;
// Starts a thread per each free/available server.
ChunksDownloadStrategy::ResultT StartThreads()
{
string url;
@@ -198,6 +199,7 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
}
}
// Saves current chunks' statuses into a resume file.
void SaveResumeChunks()
{
try

1
platform/http_request.hpp
View File

@@ -53,6 +53,7 @@ public:
Callback && onProgress = Callback());
/// Download file to filePath.
/// Pulls chunks simultaneously from all available servers, 1 thread per server.
/// @param[in] fileSize Correct file size (needed for resuming and reserving).
static HttpRequest * GetFile(std::vector<std::string> const & urls,
std::string const & filePath, int64_t fileSize,

7
storage/map_files_downloader.cpp
View File

@@ -166,6 +166,7 @@ MetaConfig MapFilesDownloader::LoadMetaConfig()
if (!metaServerUrl.empty())
{
LOG(LINFO, ("Requesting metaserver", metaServerUrl));
platform::HttpClient request(metaServerUrl);
request.SetRawHeader("X-OM-DataVersion", std::to_string(m_dataVersion));
request.SetRawHeader("X-OM-AppVersion", pl.Version());
@@ -179,7 +180,11 @@ MetaConfig MapFilesDownloader::LoadMetaConfig()
{
metaConfig = downloader::ParseMetaConfig(pl.DefaultUrlsJSON());
CHECK(metaConfig, ());
LOG(LWARNING, ("Can't get meta configuration from request, using default servers:", metaConfig->m_serversList));
LOG(LWARNING, ("Can't get metaserver configuration, using default servers:", metaConfig->m_serversList));
}
else
{
LOG(LINFO, ("Got servers list:", metaConfig->m_serversList));
}
CHECK(!metaConfig->m_serversList.empty(), ());
return *metaConfig;

14
storage/map_files_downloader.hpp
View File

@@ -32,8 +32,9 @@ public:
virtual ~MapFilesDownloader() = default;
/// Asynchronously downloads a map file, periodically invokes
/// onProgress callback and finally invokes onDownloaded
/// Asynchronously downloads a map file (first queries the metaserver
/// for the map servers list, if it haven't been done before),
/// periodically invokes onProgress callback and finally invokes onDownloaded
/// callback. Both callbacks will be invoked on the main thread.
void DownloadMapFile(QueuedCountry && queuedCountry);
@@ -53,16 +54,19 @@ public:
* @brief Async file download as string buffer (for small files only).
* Request can be skipped if current servers list is empty.
* Callback will be skipped on download error.
* NOTE: not in use at the moment.
* @param[in] url Final url part like "index.json" or "maps/210415/countries.txt".
* @param[in] forceReset True - force reset current request, if any.
*/
void DownloadAsString(std::string url, std::function<bool (std::string const &)> && callback, bool forceReset = false);
// Used in tests only.
void SetServersList(ServersList const & serversList);
void SetDownloadingPolicy(DownloadingPolicy * policy);
void SetDataVersion(int64_t version) { m_dataVersion = version; }
/// @name Legacy functions for Android resourses downloading routine.
/// @name Legacy functions for Android resources downloading routine (initial World download).
/// @{
void EnsureMetaConfigReady(std::function<void ()> && callback);
std::vector<std::string> MakeUrlListLegacy(std::string const & fileName) const;
@@ -70,9 +74,11 @@ public:
protected:
bool IsDownloadingAllowed() const;
// Produces download urls for all servers.
std::vector<std::string> MakeUrlList(std::string const & relativeUrl) const;
// Synchronously loads list of servers by http client.
// Synchronously loads a list of map servers from the metaserver.
// On dl failure fallbacks to the hardcoded list.
MetaConfig LoadMetaConfig();
private:

2
storage/map_files_downloader_with_ping.cpp
View File

@@ -16,6 +16,8 @@ void MapFilesDownloaderWithPing::GetMetaConfig(MetaConfigCallback const & callba
CHECK(!metaConfig.m_serversList.empty(), ());
// Sort the list of servers by latency.
/// @todo(pastk:: actually the sort order is used only in MapFilesDownloader::DownloadAsString()
/// to get e.g. countries.txt but this code is not enabled at the moment.
auto const sorted = Pinger::ExcludeUnavailableAndSortEndpoints(metaConfig.m_serversList);
// Keep the original list if all servers are unavailable.
if (!sorted.empty())

3
storage/pinger.cpp
View File

@@ -24,6 +24,7 @@ int64_t DoPing(std::string const & url)
return kInvalidPing;
}
LOG(LDEBUG, ("Pinging server", url));
platform::HttpClient request(url);
request.SetHttpMethod("HEAD");
request.SetTimeout(kTimeoutInSeconds);
@@ -34,7 +35,7 @@ int64_t DoPing(std::string const & url)
}
else
{
LOG(LWARNING, ("Request to server", url, "failed with code =", request.ErrorCode(), "; redirection =", request.WasRedirected()));
LOG(LWARNING, ("Ping to server", url, "failed with code =", request.ErrorCode(), "; wasRedirected =", request.WasRedirected()));
}
return kInvalidPing;

2
storage/pinger.hpp
View File

@@ -11,7 +11,7 @@ class Pinger
{
public:
using Endpoints = std::vector<std::string>;
// Returns list of available endpoints. Works synchronously.
// Pings all endpoints and a returns latency-sorted list of available ones. Works synchronously.
static Endpoints ExcludeUnavailableAndSortEndpoints(Endpoints const & urls);
};
} // namespace storage