From 6fdf306bff16a67d227e55705513feadbdccb3bd Mon Sep 17 00:00:00 2001 From: chros Date: Mon, 6 Mar 2017 19:25:50 +0000 Subject: [PATCH 1/3] Fix partially done downloads and choke groups (See #8) --- src/download/download_wrapper.cc | 13 ++++-- src/manager.cc | 2 +- src/protocol/handshake_manager.cc | 2 +- src/protocol/peer_connection_base.cc | 2 +- src/protocol/peer_connection_leech.cc | 2 +- src/torrent/data/file_list.cc | 58 +++++++++++++++++++++++++++ src/torrent/data/file_list.h | 4 ++ src/torrent/download.cc | 26 ------------ src/torrent/download.h | 9 ----- src/torrent/peer/connection_list.cc | 2 +- src/torrent/utils/resume.cc | 5 +-- 11 files changed, 78 insertions(+), 47 deletions(-) diff --git a/src/download/download_wrapper.cc b/src/download/download_wrapper.cc index 59e817814..4415dbebf 100644 --- a/src/download/download_wrapper.cc +++ b/src/download/download_wrapper.cc @@ -228,6 +228,7 @@ DownloadWrapper::receive_hash_done(ChunkHandle handle, const char* hash) { priority_queue_erase(&taskScheduler, &m_main->delay_partially_done()); priority_queue_erase(&taskScheduler, &m_main->delay_partially_restarted()); priority_queue_insert(&taskScheduler, &m_main->delay_partially_done(), cachedTime); + finished_download(); } if (!m_main->have_queue()->empty() && m_main->have_queue()->front().first >= cachedTime) @@ -325,8 +326,10 @@ DownloadWrapper::receive_tick(uint32_t ticks) { void DownloadWrapper::receive_update_priorities() { - if (m_main->chunk_selector()->empty()) + if (m_main->chunk_selector()->empty()) { + file_list()->set_selected_size_bytes(); return; + } data()->mutable_high_priority()->clear(); data()->mutable_normal_priority()->clear(); @@ -373,11 +376,15 @@ DownloadWrapper::receive_update_priorities() { priority_queue_erase(&taskScheduler, &m_main->delay_partially_done()); priority_queue_erase(&taskScheduler, &m_main->delay_partially_restarted()); - if (was_partial) + if (was_partial) { priority_queue_insert(&taskScheduler, &m_main->delay_partially_done(), cachedTime); - else + finished_download(); + } else { priority_queue_insert(&taskScheduler, &m_main->delay_partially_restarted(), cachedTime); + } } + + file_list()->set_selected_size_bytes(); } void diff --git a/src/manager.cc b/src/manager.cc index c503974db..6462cae45 100644 --- a/src/manager.cc +++ b/src/manager.cc @@ -98,7 +98,7 @@ Manager::Manager() : m_connectionManager->listen()->slot_accepted() = std::bind(&HandshakeManager::add_incoming, m_handshakeManager, std::placeholders::_1, std::placeholders::_2); - m_resourceManager->push_group("default"); + m_resourceManager->push_group("default_leech"); m_resourceManager->group_back()->up_queue()->set_heuristics(choke_queue::HEURISTICS_UPLOAD_LEECH); m_resourceManager->group_back()->down_queue()->set_heuristics(choke_queue::HEURISTICS_DOWNLOAD_LEECH); } diff --git a/src/protocol/handshake_manager.cc b/src/protocol/handshake_manager.cc index b52c8d4e5..44b6eca68 100644 --- a/src/protocol/handshake_manager.cc +++ b/src/protocol/handshake_manager.cc @@ -232,7 +232,7 @@ HandshakeManager::receive_succeeded(Handshake* handshake) { if (!download->info()->is_active()) reason = e_handshake_inactive_download; - else if (download->file_list()->is_done() && handshake->bitfield()->is_all_set()) + else if (download->file_list()->data()->is_partially_done() && handshake->bitfield()->is_all_set()) reason = e_handshake_unwanted_connection; else reason = e_handshake_duplicate; diff --git a/src/protocol/peer_connection_base.cc b/src/protocol/peer_connection_base.cc index c02998fbd..bed31fc79 100644 --- a/src/protocol/peer_connection_base.cc +++ b/src/protocol/peer_connection_base.cc @@ -197,7 +197,7 @@ PeerConnectionBase::initialize(DownloadMain* download, PeerInfo* peerInfo, Socke m_peerChunks.download_cache()->clear(); - if (!m_download->file_list()->is_done()) { + if (!m_download->file_list()->data()->is_partially_done()) { m_sendInterested = true; m_downInterested = true; } diff --git a/src/protocol/peer_connection_leech.cc b/src/protocol/peer_connection_leech.cc index 91d4dad37..085ff6cb9 100644 --- a/src/protocol/peer_connection_leech.cc +++ b/src/protocol/peer_connection_leech.cc @@ -691,7 +691,7 @@ PeerConnection::read_have_chunk(uint32_t index) { m_download->choke_group()->up_queue()->set_not_queued(this, &m_upChoke); } - if (type != Download::CONNECTION_LEECH || m_download->file_list()->is_done()) + if (type != Download::CONNECTION_LEECH || m_download->file_list()->data()->is_partially_done()) return; if (is_down_interested()) { diff --git a/src/torrent/data/file_list.cc b/src/torrent/data/file_list.cc index 4721bdbd6..33335fc7a 100644 --- a/src/torrent/data/file_list.cc +++ b/src/torrent/data/file_list.cc @@ -86,6 +86,7 @@ FileList::FileList() : m_isOpen(false), m_torrentSize(0), + m_selectedSize(0), m_chunkSize(0), m_maxFileSize(~uint64_t()) { } @@ -98,6 +99,7 @@ FileList::~FileList() { base_type::clear(); m_torrentSize = 0; + m_selectedSize = 0; } bool @@ -193,6 +195,60 @@ FileList::set_max_file_size(uint64_t size) { m_maxFileSize = size; } +// This function should be called from receive_update_priorities(). +// It offloads continous calculation by updating the m_selectedSize property. +// Its purpose is to replace size_bytes() with selected_size_bytes() +// by taking into account partial downloads. +void +FileList::set_selected_size_bytes() { + if (is_done()) { + m_selectedSize = m_torrentSize; + return; + } + + uint64_t completedBytes = completed_bytes(); + + if (data()->is_partially_done()) { + m_selectedSize = completedBytes; + return; + } + + uint32_t selectedSizeChunks = 0; + uint32_t prevChunk = -1; + bool areAllFilesSelected = true; + bool isLastFileSelected = false; + + for (FileList::const_iterator itr = begin(), last = end(); itr != last; itr++) { + + if ((*itr)->priority() != PRIORITY_OFF) { + selectedSizeChunks += (*itr)->size_chunks() - ((*itr)->range_first() == prevChunk ? 1 : 0); + prevChunk = (*itr)->range_second() - 1; + + if (itr == end() - 1) + isLastFileSelected = true; + } else { + areAllFilesSelected = false; + } + + } + + if (areAllFilesSelected) { + m_selectedSize = m_torrentSize; + return; + } + + uint64_t selectedSizeBytes = (uint64_t)(selectedSizeChunks * m_chunkSize); + + // Dealing with size of last chunk as it's usually smaller than the rest. + uint64_t remainder = m_torrentSize % (uint64_t)m_chunkSize; + + if (isLastFileSelected && remainder != 0) + selectedSizeBytes = selectedSizeBytes - (uint64_t)m_chunkSize + remainder; + + // Set completed bytes if some files (e.g. all of them) were set to Off later. + m_selectedSize = (selectedSizeBytes < completedBytes ? completedBytes : selectedSizeBytes); +} + // This function should really ensure that we arn't dealing files // spread over multiple mount-points. uint64_t @@ -376,6 +432,7 @@ FileList::initialize(uint64_t torrentSize, uint32_t chunkSize) { m_chunkSize = chunkSize; m_torrentSize = torrentSize; + m_selectedSize = torrentSize; m_rootDir = "."; m_data.mutable_completed_bitfield()->set_size_bits((size_bytes() + chunk_size() - 1) / chunk_size()); @@ -725,6 +782,7 @@ FileList::reset_filesize(int64_t size) { close(); m_chunkSize = size; m_torrentSize = size; + m_selectedSize = size; (*begin())->set_size_bytes(size); (*begin())->set_range(m_chunkSize); diff --git a/src/torrent/data/file_list.h b/src/torrent/data/file_list.h index 2cb64f669..55be387f8 100644 --- a/src/torrent/data/file_list.h +++ b/src/torrent/data/file_list.h @@ -109,6 +109,9 @@ class LIBTORRENT_EXPORT FileList : private std::vector { uint64_t size_bytes() const { return m_torrentSize; } uint32_t size_chunks() const { return bitfield()->size_bits(); } + uint64_t selected_size_bytes() const { return m_selectedSize; } + void set_selected_size_bytes(); + uint32_t completed_chunks() const { return bitfield()->size_set(); } uint64_t completed_bytes() const; uint64_t left_bytes() const; @@ -187,6 +190,7 @@ class LIBTORRENT_EXPORT FileList : private std::vector { uint64_t m_torrentSize; uint32_t m_chunkSize; uint64_t m_maxFileSize; + uint64_t m_selectedSize; std::string m_rootDir; diff --git a/src/torrent/download.cc b/src/torrent/download.cc index edddedfbf..01876e921 100644 --- a/src/torrent/download.cc +++ b/src/torrent/download.cc @@ -565,32 +565,6 @@ Download::set_connection_type(ConnectionType t) { m_ptr->set_connection_type(t); } -Download::HeuristicType -Download::upload_choke_heuristic() const { - return (Download::HeuristicType)m_ptr->main()->choke_group()->up_queue()->heuristics(); -} - -void -Download::set_upload_choke_heuristic(HeuristicType t) { - if ((choke_queue::heuristics_enum)t >= choke_queue::HEURISTICS_MAX_SIZE) - throw input_error("Invalid heuristics value."); - - m_ptr->main()->choke_group()->up_queue()->set_heuristics((choke_queue::heuristics_enum)t); -} - -Download::HeuristicType -Download::download_choke_heuristic() const { - return (Download::HeuristicType)m_ptr->main()->choke_group()->down_queue()->heuristics(); -} - -void -Download::set_download_choke_heuristic(HeuristicType t) { - if ((choke_queue::heuristics_enum)t >= choke_queue::HEURISTICS_MAX_SIZE) - throw input_error("Invalid heuristics value."); - - m_ptr->main()->choke_group()->down_queue()->set_heuristics((choke_queue::heuristics_enum)t); -} - void Download::update_priorities() { m_ptr->receive_update_priorities(); diff --git a/src/torrent/download.h b/src/torrent/download.h index 4968c2de2..2ff8746ff 100644 --- a/src/torrent/download.h +++ b/src/torrent/download.h @@ -183,15 +183,6 @@ class LIBTORRENT_EXPORT Download { ConnectionType connection_type() const; void set_connection_type(ConnectionType t); - typedef enum { - } HeuristicType; - - HeuristicType upload_choke_heuristic() const; - void set_upload_choke_heuristic(HeuristicType t); - - HeuristicType download_choke_heuristic() const; - void set_download_choke_heuristic(HeuristicType t); - // Call this when you want the modifications of the download priorities // in the entries to take effect. It is slightly expensive as it rechecks // all the peer bitfields to see if we are still interested. diff --git a/src/torrent/peer/connection_list.cc b/src/torrent/peer/connection_list.cc index 9e596803a..34d2d8b15 100644 --- a/src/torrent/peer/connection_list.cc +++ b/src/torrent/peer/connection_list.cc @@ -79,7 +79,7 @@ ConnectionList::clear() { bool ConnectionList::want_connection(PeerInfo* p, Bitfield* bitfield) { - if (m_download->file_list()->is_done() || m_download->initial_seeding() != NULL) + if (m_download->file_list()->data()->is_partially_done() || m_download->initial_seeding() != NULL) return !bitfield->is_all_set(); if (!m_download->info()->is_accepting_seeders()) diff --git a/src/torrent/utils/resume.cc b/src/torrent/utils/resume.cc index 3f528c148..187321ead 100644 --- a/src/torrent/utils/resume.cc +++ b/src/torrent/utils/resume.cc @@ -254,10 +254,7 @@ resume_save_progress(Download download, Object& object) { // } else if ((*listItr)->completed_chunks() == (*listItr)->size_chunks()) { - } else if (fileList->bitfield()->is_all_set()) { - // Currently only checking if we're finished. This needs to be - // smarter when it comes to downloading partial torrents, etc. - + } else if (download.data()->is_partially_done()) { // This assumes the syncs are properly called before // resume_save_progress gets called after finishing a torrent. filesItr->insert_key("mtime", (int64_t)fs.modified_time()); From a16efc7e923cdfe0a601560b15df8f30e37ef351 Mon Sep 17 00:00:00 2001 From: chros Date: Sat, 10 Mar 2018 18:09:00 +0000 Subject: [PATCH 2/3] Fix casting error in set_selected_size_bytes function (See #15) --- src/torrent/data/file_list.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torrent/data/file_list.cc b/src/torrent/data/file_list.cc index 33335fc7a..74365c720 100644 --- a/src/torrent/data/file_list.cc +++ b/src/torrent/data/file_list.cc @@ -237,7 +237,7 @@ FileList::set_selected_size_bytes() { return; } - uint64_t selectedSizeBytes = (uint64_t)(selectedSizeChunks * m_chunkSize); + uint64_t selectedSizeBytes = (uint64_t)selectedSizeChunks * (uint64_t)m_chunkSize; // Dealing with size of last chunk as it's usually smaller than the rest. uint64_t remainder = m_torrentSize % (uint64_t)m_chunkSize; From 5af9e512a05c2aad84a770de4635773da0aa8a48 Mon Sep 17 00:00:00 2001 From: chros Date: Sat, 10 Mar 2018 21:10:38 +0000 Subject: [PATCH 3/3] Save selected_size_bytes into session and reload it during restart (See #8) --- src/torrent/data/file_list.cc | 7 ++++++- src/torrent/data/file_list.h | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/torrent/data/file_list.cc b/src/torrent/data/file_list.cc index 74365c720..6155970ff 100644 --- a/src/torrent/data/file_list.cc +++ b/src/torrent/data/file_list.cc @@ -200,7 +200,12 @@ FileList::set_max_file_size(uint64_t size) { // Its purpose is to replace size_bytes() with selected_size_bytes() // by taking into account partial downloads. void -FileList::set_selected_size_bytes() { +FileList::set_selected_size_bytes(uint64_t bytes) { + if (bytes > 0) { + m_selectedSize = bytes; + return; + } + if (is_done()) { m_selectedSize = m_torrentSize; return; diff --git a/src/torrent/data/file_list.h b/src/torrent/data/file_list.h index 55be387f8..22ecad18e 100644 --- a/src/torrent/data/file_list.h +++ b/src/torrent/data/file_list.h @@ -110,7 +110,7 @@ class LIBTORRENT_EXPORT FileList : private std::vector { uint32_t size_chunks() const { return bitfield()->size_bits(); } uint64_t selected_size_bytes() const { return m_selectedSize; } - void set_selected_size_bytes(); + void set_selected_size_bytes(uint64_t bytes = 0); uint32_t completed_chunks() const { return bitfield()->size_set(); } uint64_t completed_bytes() const;