[core] Use dynamic chunk size for downloading maps

Signed-off-by: Konstantin Pastbin <konstantin.pastbin@gmail.com>
This commit is contained in:
Konstantin Pastbin
2025-07-02 21:18:29 +07:00
committed by Konstantin Pastbin
parent dee47e7a66
commit 1becd3e0ba
4 changed files with 31 additions and 12 deletions

View File

@@ -171,7 +171,7 @@ extern "C"
curFile.GetRemoteSize(),
std::bind(&DownloadFileFinished, ptr, _1),
std::bind(&DownloadFileProgress, ptr, _1),
512 * 1024, false));
0, false));
});
return ERR_FILE_IN_PROGRESS;

View File

@@ -40,7 +40,31 @@ ChunksDownloadStrategy::GetChunk(RangeT const & range)
void ChunksDownloadStrategy::InitChunks(int64_t fileSize, int64_t chunkSize, ChunkStatusT status)
{
if (chunkSize == 0)
{
int64_t constexpr kMb = 1024 * 1024;
size_t const sizeMb = std::max(fileSize / kMb, static_cast<int64_t>(1));
size_t constexpr kTargetCount = 40;
size_t constexpr kMinMb = 1;
size_t constexpr kMaxMb = 16;
size_t const chunkMb = std::min(std::max(sizeMb / kTargetCount, kMinMb), kMaxMb);
size_t chunksCount = sizeMb / chunkMb;
if (static_cast<int64_t>(kMb * chunkMb * chunksCount) < fileSize)
++chunksCount;
ASSERT_GREATER_OR_EQUAL(static_cast<int64_t>(kMb * chunkMb * chunksCount), fileSize, ());
LOG(LINFO, ("File size", sizeMb, "MB; chunk size", chunkMb, "MB; chunks count", chunksCount));
m_chunks.reserve(chunksCount + 1);
chunkSize = chunkMb * kMb;
}
else
{
m_chunks.reserve(static_cast<size_t>(fileSize / chunkSize + 2));
}
for (int64_t i = 0; i < fileSize; i += chunkSize)
m_chunks.push_back(ChunkT(i, status));
// The last AUX chunk is just used to hold end of the range (eof) for the previous chunk.

View File

@@ -136,7 +136,6 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
string m_filePath;
unique_ptr<FileWriter> m_writer;
size_t m_goodChunksCount;
bool m_doCleanProgressFiles;
// Starts a thread per each free/available server.
@@ -248,13 +247,9 @@ class FileHttpRequest : public HttpRequest, public IHttpThreadCallback
else if (result == ChunksDownloadStrategy::EDownloadSucceeded)
m_status = DownloadStatus::Completed;
if (isChunkOk)
{
// save information for download resume
++m_goodChunksCount;
if (m_status != DownloadStatus::Completed && m_goodChunksCount % 10 == 0)
// Save chunks statuses into the resume file.
if (isChunkOk && m_status != DownloadStatus::Completed)
SaveResumeChunks();
}
if (m_status == DownloadStatus::InProgress)
return;
@@ -300,7 +295,7 @@ public:
int64_t chunkSize, bool doCleanProgressFiles)
: HttpRequest(std::move(onFinish), std::move(onProgress)),
m_strategy(urls), m_filePath(filePath),
m_goodChunksCount(0), m_doCleanProgressFiles(doCleanProgressFiles)
m_doCleanProgressFiles(doCleanProgressFiles)
{
ASSERT ( !urls.empty(), () );

View File

@@ -59,7 +59,7 @@ public:
std::string const & filePath, int64_t fileSize,
Callback && onFinish,
Callback && onProgress = Callback(),
int64_t chunkSize = 512 * 1024,
int64_t chunkSize = 0, // 0 for auto
bool doCleanOnCancel = true);
};
} // namespace downloader