From 4dbfe436c60880f2625e4d3a9d0127a83dd6276e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 7 Oct 2014 01:46:30 +0200 Subject: display errortext for all Err as well as Ign logs consistently using Item::Failed in all specializec classes helps setting up some information bits otherwise unset, so some errors had an empty reason as an error. Ign is upgraded to display the error message we ignored to further help in understanding what happens. --- apt-pkg/acquire-item.cc | 77 ++++++++-------- apt-private/acqprogress.cc | 2 + test/integration/framework | 13 ++- .../integration/test-apt-get-update-unauth-warning | 7 +- test/integration/test-apt-update-ims | 5 +- test/integration/test-apt-update-nofallback | 2 +- test/integration/test-apt-update-rollback | 101 ++++++++++----------- .../test-bug-595691-empty-and-broken-archive-files | 2 +- 8 files changed, 108 insertions(+), 101 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 6bfd14992..8fafca33b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -125,7 +125,6 @@ pkgAcquire::Item::~Item() fetch this object */ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - Status = StatIdle; if(ErrorText == "") ErrorText = LookupTag(Message,"Message"); UsedMirror = LookupTag(Message,"UsedMirror"); @@ -134,7 +133,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* This indicates that the file is not available right now but might be sometime later. If we do a retry cycle then this should be retried [CDROMs] */ - if (Cnf->LocalOnly == true && + if (Cnf != NULL && Cnf->LocalOnly == true && StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) { Status = StatIdle; @@ -143,8 +142,11 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } Status = StatError; + Complete = false; Dequeue(); - } + } + else + Status = StatIdle; // report mirror failure back to LP if we actually use a mirror string FailReason = LookupTag(Message, "FailReason"); @@ -208,11 +210,8 @@ bool pkgAcquire::Item::Rename(string From,string To) void pkgAcquire::Item::QueueURI(ItemDesc &Item) { - if (access(DestFile.c_str(), R_OK) == 0) - { + if (RealFileExists(DestFile)) changeOwnerAndPermissionOfFile("preparePartialFile", DestFile.c_str(), "_apt", "root", 0600); - std::cerr << "QUEUE ITEM: " << DestFile << std::endl; - } Owner->Enqueue(Item); } void pkgAcquire::Item::Dequeue() @@ -559,7 +558,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ return false; } /*}}}*/ -void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ +void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/ { if(Debug) std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl @@ -567,9 +566,8 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); - Complete = false; + Item::Failed(Message,Cnf); Status = StatDone; - Dequeue(); } /*}}}*/ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -596,7 +594,7 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList } if(!ParseDiffIndex(DestFile)) - return Failed("", NULL); + return Failed("Message: Couldn't parse pdiff index", Cnf); // queue for final move string FinalFile; @@ -713,7 +711,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ if(!FileExists(FinalFile)) { - Failed("No FinalFile " + FinalFile + " available", NULL); + Failed("Message: No FinalFile " + FinalFile + " available", NULL); return false; } @@ -747,7 +745,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ // error checking and falling back if no patch was found if(available_patches.empty() == true) { - Failed("No patches available", NULL); + Failed("Message: No patches available", NULL); return false; } @@ -852,13 +850,13 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, QueueURI(Desc); } /*}}}*/ -void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ +void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/ { if(Debug) std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl; - Complete = false; + + Item::Failed(Message,Cnf); Status = StatDone; - Dequeue(); // check if we are the first to fail, otherwise we are done here State = StateDoneDiff; @@ -1284,7 +1282,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FileName; QueueURI(Desc); - + SetActiveSubprocess("copy"); return; } @@ -1305,7 +1303,6 @@ void pkgAcqIndex::StageDownloadDone(string Message, DestFile += ".decomp"; Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - SetActiveSubprocess(decompProg); } /*}}}*/ @@ -1383,18 +1380,15 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) return; } + Item::Failed(Message,Cnf); + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { // Ignore this Status = StatDone; - Complete = false; - Dequeue(); - return; } - - Item::Failed(Message,Cnf); } /*}}}*/ // AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ @@ -1639,7 +1633,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } else { _error->Error("%s", downgrade_msg.c_str()); Rename(MetaIndexFile, MetaIndexFile+".FAILED"); - Status = pkgAcquire::Item::StatError; + Item::Failed("Message: " + downgrade_msg, Cnf); TransactionManager->AbortTransaction(); return; } @@ -1663,17 +1657,15 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ _error->Warning("Use --allow-insecure-repositories to force the update"); } + Item::Failed(Message,Cnf); + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor - if (Cnf->LocalOnly == true || + if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) - { + { // Ignore this Status = StatDone; - Complete = false; - Dequeue(); - return; } - Item::Failed(Message,Cnf); } /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ @@ -2003,9 +1995,12 @@ bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/ } /*}}}*/ // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ -void pkgAcqMetaIndex::Failed(string /*Message*/, - pkgAcquire::MethodConfig * /*Cnf*/) +void pkgAcqMetaIndex::Failed(string Message, + pkgAcquire::MethodConfig * Cnf) { + pkgAcquire::Item::Failed(Message, Cnf); + Status = StatDone; + string FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); _error->Warning(_("The repository '%s' does not have a Release file. " @@ -2115,6 +2110,8 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long /*Size*/, /*}}}*/ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { + Item::Failed(Message, Cnf); + // we failed, we will not get additional items from this method ExpectedAdditionalItems = 0; @@ -2126,14 +2123,12 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile.append(URItoFileName(RealURI)); TransactionManager->TransactionStageRemoval(this, FinalFile); + Status = StatDone; new pkgAcqMetaIndex(Owner, TransactionManager, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); - if (Cnf->LocalOnly == true || - StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) - Dequeue(); } else { @@ -2149,9 +2144,11 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* // only allow going further if the users explicitely wants it if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { + Status = StatDone; + /* Always move the meta index, even if gpgv failed. This ensures * that PackageFile objects are correctly filled in */ - if (FileExists(DestFile)) + if (FileExists(DestFile)) { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); @@ -2161,19 +2158,17 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* "Release"); FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, "Release"); - - + // Done, queue for rename on transaction finished TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); } QueueIndexes(false); } else { - // warn if the repository is unsinged + // warn if the repository is unsigned _error->Warning("Use --allow-insecure-repositories to force the update"); TransactionManager->AbortTransaction(); Status = StatError; - return; - } + } } } /*}}}*/ diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc index d6ce192ad..aa88d5334 100644 --- a/apt-private/acqprogress.cc +++ b/apt-private/acqprogress.cc @@ -117,6 +117,8 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm) if (Itm.Owner->Status == pkgAcquire::Item::StatDone) { cout << _("Ign ") << Itm.Description << endl; + if (Itm.Owner->ErrorText.empty() == false) + cout << " " << Itm.Owner->ErrorText << endl; } else { diff --git a/test/integration/framework b/test/integration/framework index 688a1abf2..29e5fafe6 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -715,7 +715,7 @@ buildaptarchivefromincoming() { aptftparchive -qq generate ftparchive.conf cd - > /dev/null msgdone "info" - generatereleasefiles + generatereleasefiles "$@" } buildaptarchivefromfiles() { @@ -830,14 +830,19 @@ setupflataptarchive() { } setupaptarchive() { - buildaptarchive + local NOUPDATE=0 + if [ "$1" = '--no-update' ]; then + NOUPDATE=1 + shift + fi + buildaptarchive "$@" if [ -e aptarchive/dists ]; then setupdistsaptarchive else setupflataptarchive fi - signreleasefiles - if [ "$1" != '--no-update' ]; then + signreleasefiles 'Joe Sixpack' "$@" + if [ "1" != "$NOUPDATE" ]; then testsuccess aptget update -o Debug::pkgAcquire::Worker=true -o Debug::Acquire::gpgv=true fi } diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 27160b5f9..8e212a3c4 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -20,11 +20,12 @@ rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated files leads to warning testequal "Ign file: unstable InRelease + File not found Err file: unstable Release - + File not found W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository. W: Use --allow-insecure-repositories to force the update -W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release +W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release File not found E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update @@ -34,7 +35,9 @@ testequal "partial" ls rootdir/var/lib/apt/lists # allow override testequal "Ign file: unstable InRelease + File not found Ign file: unstable Release + File not found Reading package lists... W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories # ensure we can not install the package diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 61b808b0f..8aa5a7262 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -44,8 +44,9 @@ echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex runtest msgmsg "Release/Release.gpg" -# with Release/Release.gpg +# with Release/Release.gpg EXPECT="Ign http://localhost:8080 unstable InRelease + 404 Not Found Hit http://localhost:8080 unstable Release Hit http://localhost:8080 unstable Release.gpg Hit http://localhost:8080 unstable/main Sources @@ -65,8 +66,10 @@ runtest # no Release.gpg or InRelease msgmsg "Release only" EXPECT="Ign http://localhost:8080 unstable InRelease + 404 Not Found Hit http://localhost:8080 unstable Release Ign http://localhost:8080 unstable Release.gpg + 404 Not Found Hit http://localhost:8080 unstable/main Sources Hit http://localhost:8080 unstable/main amd64 Packages Hit http://localhost:8080 unstable/main Translation-en diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback index c400dcc36..321472c2e 100755 --- a/test/integration/test-apt-update-nofallback +++ b/test/integration/test-apt-update-nofallback @@ -161,7 +161,7 @@ test_inrelease_to_invalid_inrelease() testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) -W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease +W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index ee8bc6926..5b9c200fe 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -19,44 +19,44 @@ create_fresh_archive() insertpackage 'unstable' 'old' 'all' '1.0' - setupaptarchive + setupaptarchive --no-update } add_new_package() { insertpackage "unstable" "new" "all" "1.0" insertsource "unstable" "new" "all" "1.0" - setupaptarchive --no-update - - avoid_ims_hit + setupaptarchive --no-update "$@" } break_repository_sources_index() { - printf "xxx" > $APTARCHIVE/dists/unstable/main/source/Sources - gzip -c $APTARCHIVE/dists/unstable/main/source/Sources > \ - $APTARCHIVE/dists/unstable/main/source/Sources.gz - avoid_ims_hit + printf 'xxx' > $APTARCHIVE/dists/unstable/main/source/Sources + compressfile "$APTARCHIVE/dists/unstable/main/source/Sources" "$@" } -test_inrelease_to_new_inrelease() { - msgmsg "Test InRelease to new InRelease works fine" +start_with_good_inrelease() { create_fresh_archive + testsuccess aptget update testequal "old/unstable 1.0 all" apt list -q +} - add_new_package +test_inrelease_to_new_inrelease() { + msgmsg 'Test InRelease to new InRelease works fine' + start_with_good_inrelease + add_new_package '+1hour' testsuccess aptget update -o Debug::Acquire::Transaction=1 - testequal "new/unstable 1.0 all old/unstable 1.0 all" apt list -q } test_inrelease_to_broken_hash_reverts_all() { - msgmsg "Test InRelease to broken InRelease reverts everything" - create_fresh_archive - add_new_package + msgmsg 'Test InRelease to broken InRelease reverts everything' + start_with_good_inrelease + + add_new_package '+1hour' # break the Sources file - break_repository_sources_index + break_repository_sources_index '+1hour' # test the error condition testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch @@ -66,14 +66,14 @@ E: Some index files failed to download. They have been ignored, or old ones used testequal "E: Unable to locate package new" aptget install new -s -qq } -test_inreleae_to_valid_release() { - msgmsg "Test InRelease to valid Release" - create_fresh_archive - add_new_package - # switch to a unsinged repo now +test_inrelease_to_valid_release() { + msgmsg 'Test InRelease to valid Release' + start_with_good_inrelease + + add_new_package '+1hour' + # switch to a unsigned repo now rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg - avoid_ims_hit # update fails testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq @@ -85,16 +85,17 @@ test_inreleae_to_valid_release() { testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release } -test_inreleae_to_release_reverts_all() { - msgmsg "Test InRelease to broken Release reverts everything" - create_fresh_archive +test_inrelease_to_release_reverts_all() { + msgmsg 'Test InRelease to broken Release reverts everything' + start_with_good_inrelease - # switch to a unsinged repo now - add_new_package + # switch to a unsigned repo now + add_new_package '+1hour' rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg + # break it - break_repository_sources_index + break_repository_sources_index '+1hour' # ensure error testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1 @@ -107,21 +108,19 @@ test_inreleae_to_release_reverts_all() { } test_unauthenticated_to_invalid_inrelease() { - msgmsg "Test UnAuthenticated to invalid InRelease reverts everything" + msgmsg 'Test UnAuthenticated to invalid InRelease reverts everything' create_fresh_archive - rm -rf rootdir/var/lib/apt/lists/* rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg - avoid_ims_hit - - testsuccess aptget update -qq --allow-insecure-repositories + + testsuccess aptget update --allow-insecure-repositories testequal "WARNING: The following packages cannot be authenticated! old E: There are problems and -y was used without --force-yes" aptget install -qq -y old - + # go to authenticated but not correct - add_new_package - break_repository_sources_index + add_new_package '+1hour' + break_repository_sources_index '+1hour' testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch @@ -134,14 +133,14 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y } test_inrelease_to_unauth_inrelease() { - msgmsg "Test InRelease to InRelease without sig" - create_fresh_archive - signreleasefiles 'Marvin Paranoid' - avoid_ims_hit - + msgmsg 'Test InRelease to InRelease without good sig' + start_with_good_inrelease + + signreleasefiles 'Marvin Paranoid' '+1hour' + testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2 -W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease +W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2 W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq @@ -150,13 +149,13 @@ W: Some index files failed to download. They have been ignored, or old ones used test_inrelease_to_broken_gzip() { msgmsg "Test InRelease to broken gzip" - create_fresh_archive - # append junk at the end of the gzip, this + start_with_good_inrelease + + # append junk at the end of the compressed file echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz - # remove uncompressed file, otherwise apt will just fallback fetching - # that + touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz + # remove uncompressed file to avoid fallback rm $APTARCHIVE/dists/unstable/main/source/Sources - avoid_ims_hit testfailure aptget update } @@ -174,7 +173,7 @@ ROOTDIR=${TMPWORKINGDIRECTORY}/rootdir APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" # test the following cases: -# - InRelease -> broken InRelease revert to previous state +# - InRelease -> broken InRelease revert to previous state # - empty lists dir and broken remote leaves nothing on the system # - InRelease -> hashsum mismatch for one file reverts all files to previous state # - Release/Release.gpg -> hashsum mismatch @@ -184,13 +183,13 @@ APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" # - unauthenticated -> invalid InRelease # stuff to do: -# - ims-hit +# - ims-hit # - gzip-index tests test_inrelease_to_new_inrelease test_inrelease_to_broken_hash_reverts_all -test_inreleae_to_valid_release -test_inreleae_to_release_reverts_all +test_inrelease_to_valid_release +test_inrelease_to_release_reverts_all test_unauthenticated_to_invalid_inrelease test_inrelease_to_unauth_inrelease test_inrelease_to_broken_gzip diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index 683c174bd..fedf82c92 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -13,7 +13,7 @@ setupflataptarchive testaptgetupdate() { rm -rf rootdir/var/lib/apt aptget update 2>> testaptgetupdate.diff >> testaptgetupdate.diff || true - sed -i -e '/Ign / d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff + sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff GIVEN="$1" shift msgtest "Test for correctness of" "apt-get update with $*" -- cgit v1.2.3