From fb193b1cd43f0e8c3b7e5f69f183b9abe7e83761 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 15 Mar 2016 14:50:37 +0100 Subject: Get accurate progress reporting in apt update again For the non-pdiff case, we have can have accurate progress reporting because after fetching the {,In}Release files we know how many IndexFiles will be fetched and what size they have. Therefore init the filesize early (in pkgAcqIndex::Init) and ensure that in Acquire::Pulse() looks at already downloaded bits when calculating the progress in Acquire::Pulse. Also improve debug output of Debug::acquire::progress --- apt-pkg/acquire-item.cc | 4 ++++ apt-pkg/acquire.cc | 31 ++++++++++++++++----------- test/integration/test-apt-progress-fd | 8 +++---- test/integration/test-apt-update-not-modified | 6 ++++-- test/integration/test-apt-update-reporting | 31 +++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 19 deletions(-) create mode 100755 test/integration/test-apt-update-reporting diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 717751a6e..2180a5a41 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -2610,6 +2610,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, DestFile = GetPartialFileNameFromURI(URI); NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false); + // store file size of the download to ensure the fetcher gives + // accurate progress reporting + FileSize = GetExpectedHashes().FileSize(); + if (CurrentCompressionExtension == "uncompressed") { Desc.URI = URI; diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 17ee691d7..757fab057 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -1129,7 +1129,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) CurrentBytes = 0; TotalItems = 0; CurrentItems = 0; - + // Compute the total number of bytes to fetch unsigned int Unknown = 0; unsigned int Count = 0; @@ -1141,16 +1141,12 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) TotalItems++; if ((*I)->Status == pkgAcquire::Item::StatDone) ++CurrentItems; - - // Totally ignore local items - if ((*I)->Local == true) - continue; // see if the method tells us to expect more TotalItems += (*I)->ExpectedAdditionalItems; // check if there are unfetched Release files - if ((*I)->Complete == false && (*I)->ExpectedAdditionalItems > 0) + if ((*I)->Status != pkgAcquire::Item::StatDone && (*I)->ExpectedAdditionalItems > 0) UnfetchedReleaseFiles = true; TotalBytes += (*I)->FileSize; @@ -1159,7 +1155,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) if ((*I)->FileSize == 0 && (*I)->Complete == false) ++Unknown; } - + // Compute the current completion unsigned long long ResumeSize = 0; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; @@ -1187,12 +1183,6 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) if (CurrentBytes > TotalBytes) CurrentBytes = TotalBytes; - // debug - if (_config->FindB("Debug::acquire::progress", false) == true) - std::clog << " Bytes: " - << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) - << std::endl; - // Compute the CPS struct timeval NewTime; gettimeofday(&NewTime,0); @@ -1220,6 +1210,21 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) // use both files and bytes because bytes can be unreliable Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + 0.2 * (CurrentItems/float(TotalItems)*100.0)); + + // debug + if (_config->FindB("Debug::acquire::progress", false) == true) + { + std::clog + << "[" + << std::setw(5) << std::setprecision(4) << std::showpoint << Percent + << "]" + << " Bytes: " + << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) + << " # Files: " + << CurrentItems << " / " << TotalItems + << std::endl; + } + double const DiffPercent = Percent - OldPercent; if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true) return true; diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd index f0d208967..c6650bc2e 100755 --- a/test/integration/test-apt-progress-fd +++ b/test/integration/test-apt-progress-fd @@ -16,7 +16,7 @@ setupaptarchive exec 3> apt-progress.log testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:20:Retrieving file 1 of 1 +dlstatus:1:100:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) pmstatus:testing:16.6667:Preparing testing (amd64) @@ -32,7 +32,7 @@ pmstatus:dpkg-exec:83.3333:Running dpkg' exec 3> apt-progress.log testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:20:Retrieving file 1 of 1 +dlstatus:1:100:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) pmstatus:testing:16.6667:Preparing testing (amd64) @@ -48,7 +48,7 @@ pmstatus:dpkg-exec:83.3333:Running dpkg' exec 3> apt-progress.log testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:20:Retrieving file 1 of 1 +dlstatus:1:100:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) pmstatus:testing:16.6667:Preparing testing (amd64) @@ -74,7 +74,7 @@ pmstatus:dpkg-exec:75:Running dpkg' exec 3> apt-progress.log testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:20:Retrieving file 1 of 1 +dlstatus:1:100:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing2:0:Installing testing2 (i386) pmstatus:testing2:16.6667:Preparing testing2 (i386) diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified index 82e69d5d5..ce3bc0dd3 100755 --- a/test/integration/test-apt-update-not-modified +++ b/test/integration/test-apt-update-not-modified @@ -34,6 +34,7 @@ Reading package lists..." aptget update # readd arch so its downloaded again… configarchitecture 'amd64' 'i386' # … but oh noes, hashsum mismatch! + SIZE=$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') find aptarchive/dists/unstable/main/binary-amd64/ -type f -delete cat >> aptarchive/dists/unstable/main/binary-amd64/Packages <> aptarchive/dists/unstable/main/binary-amd64/Packages <progress.log >ignore.out + +prev_percent=0.0 +while read line; do + percent="$(echo "$line"|cut -b2-6)" + # need to cut the decimal point and digits because sh can not do + # float compare + if [ "${percent%%.*}" -lt "${prev_percent%%.*}" ]; then + echo "progress goes backwards" + cat progress.log + exit 1 + fi + prev_percent="$percent" +done < progress.log + -- cgit v1.2.3