From 564720959e4ae47921b795fe6c5ce46e1e1bdc95 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 18 Jul 2014 23:21:46 +0200 Subject: WIP transaction based update --- apt-pkg/acquire-item.cc | 118 +++++++++++++------ apt-pkg/acquire-item.h | 302 ++++++++++++++++++++++++++---------------------- apt-pkg/acquire.cc | 50 ++++++++ apt-pkg/acquire.h | 4 + 4 files changed, 303 insertions(+), 171 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 221d2a267..bd11ba66c 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -66,7 +66,7 @@ static void printHashSumComparision(std::string const &URI, HashStringList const // Acquire::Item::Item - Constructor /*{{{*/ pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0), ExpectedAdditionalItems(0), + Local(false), QueueCounter(0), TransactionID(0), ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes) { Owner->Add(this); @@ -353,11 +353,12 @@ bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ * patches. If anything goes wrong in that process, it will fall back to * the original packages file */ -pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, +pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcqMetaIndex *MetaOwner, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser) + : pkgAcqBaseIndex(MetaOwner, Target, ExpectedHashes, + MetaIndexParser) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -455,7 +456,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ std::clog << "Package file is up-to-date" << std::endl; // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + new pkgAcqIndexDiffs(MetaOwner, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return true; } @@ -542,14 +543,14 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (pdiff_merge == false) { - new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + new pkgAcqIndexDiffs(MetaOwner, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); } else { std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) - (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target, + (*diffs)[i] = new pkgAcqIndexMergeDiffs(MetaOwner, Target, ExpectedHashes, MetaIndexParser, available_patches[i], @@ -577,7 +578,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; @@ -619,13 +620,13 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList /* The package diff is added to the queue. one object is constructed * for each diff and the index */ -pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcqMetaIndex *MetaOwner, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { @@ -657,7 +658,7 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@ -797,7 +798,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, Target, + new pkgAcqIndexDiffs(MetaOwner, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return Finish(); @@ -807,13 +808,13 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi } /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ -pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, +pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcqMetaIndex *MetaOwner, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -856,7 +857,7 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*C // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -954,6 +955,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, Init(URI, URIDesc, ShortDesc); } +#if 0 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) @@ -961,6 +963,27 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, RealURI(Target->URI) { // autoselect the compression method + AutoSelectCompression(); + Init(Target->URI, Target->Description, Target->ShortDesc); +} +#endif + /*}}}*/ +pkgAcqIndex::pkgAcqIndex(pkgAcqMetaIndex *MetaOwner, + IndexTarget const *Target, + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser) + : pkgAcqBaseIndex(MetaOwner->GetOwner(), Target, ExpectedHash, + MetaIndexParser), RealURI(Target->URI) +{ + // autoselect the compression method + AutoSelectCompression(); + Init(Target->URI, Target->Description, Target->ShortDesc); + + TransactionID = (unsigned long)MetaOwner; +} + /*}}}*/ +void pkgAcqIndex::AutoSelectCompression() +{ std::vector types = APT::Configuration::getCompressionTypes(); CompressionExtension = ""; if (ExpectedHashes.usable()) @@ -976,10 +999,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, } if (CompressionExtension.empty() == false) CompressionExtension.erase(CompressionExtension.end()-1); - - Init(Target->URI, Target->Description, Target->ShortDesc); } - /*}}}*/ // AcqIndex::Init - defered Constructor /*{{{*/ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Decompression = false; @@ -1092,6 +1112,9 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ } Item::Failed(Message,Cnf); + + /// cancel the entire transaction + Owner->AbortTransaction(TransactionID); } /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ @@ -1112,6 +1135,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con { RenameOnError(HashSumMismatch); printHashSumComparision(RealURI, ExpectedHashes, Hashes); + Failed(Message, Cfg); return; } @@ -1132,16 +1156,18 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false) { RenameOnError(InvalidFormat); + Failed(Message, Cfg); return; } } - // Done, move it into position + // Done, queue for rename on transaction finished + PartialFile = DestFile; + string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - + DestFile = FinalFile; +#if 0 /* We restore the original name to DestFile so that the clean operation will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -1150,7 +1176,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // Remove the compressed version. if (Erase == true) unlink(DestFile.c_str()); - +#endif return; } @@ -1237,9 +1263,10 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") { } -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target, + /*}}}*/ +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcqMetaIndex *MetaOwner, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser) + : pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser) { // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); @@ -1388,11 +1415,18 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) Rename(LastGoodSig, DestFile); + // queue for copy + PartialFile = DestFile; + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved - new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, - MetaIndexShortDesc, DestFile, IndexTargets, - MetaIndexParser); + pkgAcqMetaIndex *mi = new pkgAcqMetaIndex( + Owner, MetaIndexURI, MetaIndexURIDesc, + MetaIndexShortDesc, DestFile, IndexTargets, + MetaIndexParser); + TransactionID = (unsigned long)mi; } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ @@ -1446,6 +1480,8 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); + TransactionID = (unsigned long)this; + // Create the item Desc.Description = URIDesc; Desc.Owner = this; @@ -1536,8 +1572,7 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList FinalFile += URItoFileName(RealURI); if (SigFile == DestFile) SigFile = FinalFile; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); + PartialFile = DestFile; DestFile = FinalFile; } } @@ -1609,6 +1644,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ // Download further indexes with verification QueueIndexes(true); +#if 0 // is it a clearsigned MetaIndex file? if (DestFile == SigFile) return; @@ -1618,6 +1654,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ URItoFileName(RealURI) + ".gpg"; Rename(SigFile,VerifiedSigFile); chmod(VerifiedSigFile.c_str(),0644); +#endif } /*}}}*/ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ @@ -1700,9 +1737,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(this, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndexTrans(this, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@ -1713,9 +1750,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(this, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndex(this, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@ -1799,8 +1836,10 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/ // pkgAcqMetaIndex::Failed - no Release file present or no signature file present /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) +void pkgAcqMetaIndex::Failed(string /*Message*/, + pkgAcquire::MethodConfig * /*Cnf*/) { +#if 0 if (AuthPass == true) { // gpgv method failed, if we have a good signature @@ -1838,7 +1877,7 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) // gpgv method failed ReportMirrorFailure("GPGFailure"); } - +#endif /* Always move the meta index, even if gpgv failed. This ensures * that PackageFile objects are correctly filled in */ if (FileExists(DestFile)) { @@ -1864,6 +1903,15 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) QueueIndexes(false); } /*}}}*/ + +void pkgAcqMetaIndex::Finished() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "Finished: " << DestFile <CommitTransaction((unsigned long)this); +} + + pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &URI, string const &URIDesc, string const &ShortDesc, string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 0500a3627..6c9fec695 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -47,6 +47,7 @@ class indexRecords; class pkgRecords; class pkgSourceList; class IndexTarget; +class pkgAcqMetaIndex; /** \brief Represents the process by which a pkgAcquire object should {{{ * retrieve a file or a collection of files. @@ -116,7 +117,7 @@ class pkgAcquire::Item : public WeakPointable /** \brief The item was could not be downloaded because of * a transient network error (e.g. network down) */ - StatTransientNetworkError + StatTransientNetworkError, } Status; /** \brief Contains a textual description of the error encountered @@ -168,6 +169,9 @@ class pkgAcquire::Item : public WeakPointable */ unsigned int QueueCounter; + /** \brief TransactionID */ + unsigned long TransactionID; + /** \brief The number of additional fetch items that are expected * once this item is done. * @@ -183,6 +187,9 @@ class pkgAcquire::Item : public WeakPointable */ std::string DestFile; + /** \brief storge name until a transaction is finished */ + std::string PartialFile; + /** \brief Invoked by the acquire worker when the object couldn't * be fetched. * @@ -368,6 +375,137 @@ class pkgAcqSubIndex : public pkgAcquire::Item }; /*}}}*/ +/** \brief An item that is responsible for downloading the meta-index {{{ + * file (i.e., Release) itself and verifying its signature. + * + * Once the download and verification are complete, the downloads of + * the individual index files are queued up using pkgAcqDiffIndex. + * If the meta-index file had a valid signature, the expected hashsums + * of the index files will be the md5sums listed in the meta-index; + * otherwise, the expected hashsums will be "" (causing the + * authentication of the index files to be bypassed). + */ +class pkgAcqMetaIndex : public pkgAcquire::Item +{ + protected: + /** \brief The URI that is actually being downloaded; never + * modified by pkgAcqMetaIndex. + */ + std::string RealURI; + + /** \brief The file in which the signature for this index was stored. + * + * If empty, the signature and the md5sums of the individual + * indices will not be checked. + */ + std::string SigFile; + + /** \brief The index files to download. */ + const std::vector* IndexTargets; + + /** \brief The parser for the meta-index file. */ + indexRecords* MetaIndexParser; + + /** \brief If \b true, the index's signature is currently being verified. + */ + bool AuthPass; + // required to deal gracefully with problems caused by incorrect ims hits + bool IMSHit; + + /** \brief Check that the release file is a release file for the + * correct distribution. + * + * \return \b true if no fatal errors were encountered. + */ + bool VerifyVendor(std::string Message); + + /** \brief Called when a file is finished being retrieved. + * + * If the file was not downloaded to DestFile, a copy process is + * set up to copy it to DestFile; otherwise, Complete is set to \b + * true and the file is moved to its final location. + * + * \param Message The message block received from the fetch + * subprocess. + */ + void RetrievalDone(std::string Message); + + /** \brief Called when authentication succeeded. + * + * Sanity-checks the authenticated file, queues up the individual + * index files for download, and saves the signature in the lists + * directory next to the authenticated list file. + * + * \param Message The message block received from the fetch + * subprocess. + */ + void AuthDone(std::string Message); + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected hashsum + * can be determined from the meta-index will be downloaded, and + * the hashsums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no hashsum checking will be performed. + */ + void QueueIndexes(bool verify); + + public: + + // Specialized action members + virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cnf); + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return RealURI; }; + virtual void Finished(); + + /** \brief Create a new pkgAcqMetaIndex. */ + pkgAcqMetaIndex(pkgAcquire *Owner, + std::string URI,std::string URIDesc, std::string ShortDesc, + std::string SigFile, + const std::vector* IndexTargets, + indexRecords* MetaIndexParser); +}; + /*}}}*/ +/** \brief An item repsonsible for downloading clearsigned metaindexes {{{*/ +class pkgAcqMetaClearSig : public pkgAcqMetaIndex +{ + /** \brief The URI of the meta-index file for the detached signature */ + std::string MetaIndexURI; + + /** \brief A "URI-style" description of the meta-index file */ + std::string MetaIndexURIDesc; + + /** \brief A brief description of the meta-index file */ + std::string MetaIndexShortDesc; + + /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */ + std::string MetaSigURI; + + /** \brief A "URI-style" description of the meta-signature file */ + std::string MetaSigURIDesc; + + /** \brief A brief description of the meta-signature file */ + std::string MetaSigShortDesc; + +public: + void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); + virtual std::string Custom600Headers() const; + + /** \brief Create a new pkgAcqMetaClearSig. */ + pkgAcqMetaClearSig(pkgAcquire *Owner, + std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc, + std::string const &MetaIndexURI, std::string const &MetaIndexURIDesc, std::string const &MetaIndexShortDesc, + std::string const &MetaSigURI, std::string const &MetaSigURIDesc, std::string const &MetaSigShortDesc, + const std::vector* IndexTargets, + indexRecords* MetaIndexParser); + virtual ~pkgAcqMetaClearSig(); +}; + /*}}}*/ + + /** \brief Common base class for all classes that deal with fetching {{{ indexes */ @@ -378,13 +516,21 @@ class pkgAcqBaseIndex : public pkgAcquire::Item */ const struct IndexTarget * Target; indexRecords *MetaIndexParser; + pkgAcqMetaIndex *MetaOwner; + + pkgAcqBaseIndex(pkgAcqMetaIndex *MetaOwner, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : Item(MetaOwner->GetOwner(), ExpectedHashes), Target(Target), + MetaIndexParser(MetaIndexParser), MetaOwner(MetaOwner) {}; pkgAcqBaseIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) : Item(Owner, ExpectedHashes), Target(Target), - MetaIndexParser(MetaIndexParser) {}; + MetaIndexParser(MetaIndexParser), MetaOwner(0) {}; }; /*}}}*/ @@ -451,7 +597,7 @@ class pkgAcqDiffIndex : public pkgAcqBaseIndex * * \param ExpectedHashes The list file's hashsums which are expected. */ - pkgAcqDiffIndex(pkgAcquire *Owner, + pkgAcqDiffIndex(pkgAcqMetaIndex *MetaIndexOwner, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); @@ -539,7 +685,7 @@ class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex * \param allPatches contains all related items so that each item can * check if it was the last one to complete the download step */ - pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + pkgAcqIndexMergeDiffs(pkgAcqMetaIndex *MetaIndexOwner, struct IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser, @@ -667,7 +813,7 @@ class pkgAcqIndexDiffs : public pkgAcqBaseIndex * should be ordered so that each diff appears before any diff * that depends on it. */ - pkgAcqIndexDiffs(pkgAcquire *Owner, + pkgAcqIndexDiffs(pkgAcqMetaIndex *MetaIndexOwner, struct IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser, @@ -710,6 +856,9 @@ class pkgAcqIndex : public pkgAcqBaseIndex /** \brief Do the changes needed to fetch via AptByHash (if needed) */ void InitByHashIfNeeded(const std::string MetaKey); + /** \brief Auto select the right compression to use */ + void AutoSelectCompression(); + public: // Specialized action members @@ -741,10 +890,17 @@ class pkgAcqIndex : public pkgAcqBaseIndex pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, HashStringList const &ExpectedHashes, std::string compressExt=""); +#if 0 pkgAcqIndex(pkgAcquire *Owner, IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser); +#endif + pkgAcqIndex(pkgAcqMetaIndex *MetaIndexOwner, + IndexTarget const * const Target, + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser); + void Init(std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc); }; @@ -774,10 +930,12 @@ class pkgAcqIndexTrans : public pkgAcqIndex * * \param ShortDesc A brief description of this index file. */ - pkgAcqIndexTrans(pkgAcquire *Owner,std::string URI,std::string URIDesc, + pkgAcqIndexTrans(pkgAcquire *Owner, + std::string URI,std::string URIDesc, std::string ShortDesc); - pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target, - HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); + pkgAcqIndexTrans(pkgAcqMetaIndex *Owner, IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser); }; /*}}}*/ /** \brief Information about an index file. */ /*{{{*/ @@ -891,134 +1049,6 @@ class pkgAcqMetaSig : public pkgAcquire::Item virtual ~pkgAcqMetaSig(); }; /*}}}*/ -/** \brief An item that is responsible for downloading the meta-index {{{ - * file (i.e., Release) itself and verifying its signature. - * - * Once the download and verification are complete, the downloads of - * the individual index files are queued up using pkgAcqDiffIndex. - * If the meta-index file had a valid signature, the expected hashsums - * of the index files will be the md5sums listed in the meta-index; - * otherwise, the expected hashsums will be "" (causing the - * authentication of the index files to be bypassed). - */ -class pkgAcqMetaIndex : public pkgAcquire::Item -{ - protected: - /** \brief The URI that is actually being downloaded; never - * modified by pkgAcqMetaIndex. - */ - std::string RealURI; - - /** \brief The file in which the signature for this index was stored. - * - * If empty, the signature and the md5sums of the individual - * indices will not be checked. - */ - std::string SigFile; - - /** \brief The index files to download. */ - const std::vector* IndexTargets; - - /** \brief The parser for the meta-index file. */ - indexRecords* MetaIndexParser; - - /** \brief If \b true, the index's signature is currently being verified. - */ - bool AuthPass; - // required to deal gracefully with problems caused by incorrect ims hits - bool IMSHit; - - /** \brief Check that the release file is a release file for the - * correct distribution. - * - * \return \b true if no fatal errors were encountered. - */ - bool VerifyVendor(std::string Message); - - /** \brief Called when a file is finished being retrieved. - * - * If the file was not downloaded to DestFile, a copy process is - * set up to copy it to DestFile; otherwise, Complete is set to \b - * true and the file is moved to its final location. - * - * \param Message The message block received from the fetch - * subprocess. - */ - void RetrievalDone(std::string Message); - - /** \brief Called when authentication succeeded. - * - * Sanity-checks the authenticated file, queues up the individual - * index files for download, and saves the signature in the lists - * directory next to the authenticated list file. - * - * \param Message The message block received from the fetch - * subprocess. - */ - void AuthDone(std::string Message); - - /** \brief Starts downloading the individual index files. - * - * \param verify If \b true, only indices whose expected hashsum - * can be determined from the meta-index will be downloaded, and - * the hashsums of indices will be checked (reporting - * #StatAuthError if there is a mismatch). If verify is \b false, - * no hashsum checking will be performed. - */ - void QueueIndexes(bool verify); - - public: - - // Specialized action members - virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, - pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers() const; - virtual std::string DescURI() const {return RealURI; }; - - /** \brief Create a new pkgAcqMetaIndex. */ - pkgAcqMetaIndex(pkgAcquire *Owner, - std::string URI,std::string URIDesc, std::string ShortDesc, - std::string SigFile, - const std::vector* IndexTargets, - indexRecords* MetaIndexParser); -}; - /*}}}*/ -/** \brief An item repsonsible for downloading clearsigned metaindexes {{{*/ -class pkgAcqMetaClearSig : public pkgAcqMetaIndex -{ - /** \brief The URI of the meta-index file for the detached signature */ - std::string MetaIndexURI; - - /** \brief A "URI-style" description of the meta-index file */ - std::string MetaIndexURIDesc; - - /** \brief A brief description of the meta-index file */ - std::string MetaIndexShortDesc; - - /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */ - std::string MetaSigURI; - - /** \brief A "URI-style" description of the meta-signature file */ - std::string MetaSigURIDesc; - - /** \brief A brief description of the meta-signature file */ - std::string MetaSigShortDesc; - -public: - void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers() const; - - /** \brief Create a new pkgAcqMetaClearSig. */ - pkgAcqMetaClearSig(pkgAcquire *Owner, - std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc, - std::string const &MetaIndexURI, std::string const &MetaIndexURIDesc, std::string const &MetaIndexShortDesc, - std::string const &MetaSigURI, std::string const &MetaSigURIDesc, std::string const &MetaSigShortDesc, - const std::vector* IndexTargets, - indexRecords* MetaIndexParser); - virtual ~pkgAcqMetaClearSig(); -}; - /*}}}*/ /** \brief An item that is responsible for fetching a package file. {{{ * * If the package file already exists in the cache, nothing will be diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 8467dab5b..2e2e39d51 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -37,6 +37,7 @@ #include #include #include +#include #include /*}}}*/ @@ -168,6 +169,55 @@ void pkgAcquire::Remove(Item *Itm) } } /*}}}*/ +// Acquire::AbortTransaction - Remove a transaction /*{{{*/ +void pkgAcquire::AbortTransaction(unsigned long TransactionID) +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "AbortTransaction: " << TransactionID << std::endl; + + std::vector Transaction; + for (ItemIterator I = Items.begin(); I != Items.end(); ++I) + if((*I)->TransactionID == TransactionID) + Transaction.push_back(*I); + + for (std::vector::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << " Cancel: " << (*I)->DestFile << std::endl; + Dequeue(*I); + (*I)->Status = pkgAcquire::Item::StatError; + } +} + /*}}}*/ +// Acquire::CommitTransaction - Commit a transaction /*{{{*/ +void pkgAcquire::CommitTransaction(unsigned long TransactionID) +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "CommitTransaction: " << TransactionID << std::endl; + + std::vector Transaction; + for (ItemIterator I = Items.begin(); I != Items.end(); ++I) + if((*I)->TransactionID == TransactionID) + Transaction.push_back(*I); + + for (std::vector::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + if((*I)->PartialFile != "" && + (*I)->Status == pkgAcquire::Item::StatDone) + { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "mv " + << (*I)->PartialFile << " -> " + << (*I)->DestFile << std::endl; + Rename((*I)->PartialFile, (*I)->DestFile); + chmod((*I)->DestFile.c_str(),0644); + } + } +} + /*}}}*/ + // Acquire::Add - Add a worker /*{{{*/ // --------------------------------------------------------------------- /* A list of workers is kept so that the select loop can direct their FD diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 0113021b2..0b955cc76 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -248,6 +248,10 @@ class pkgAcquire public: + /** \brief Abort a given transaction */ + void AbortTransaction(unsigned long TransactionID); + void CommitTransaction(unsigned long TransactionID); + /** \brief Retrieve information about a fetch method by name. * * \param Access The name of the method to look up. -- cgit v1.2.3 From 47aca3cfc17ee23c37693b4e53c675a74b38decd Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 18 Jul 2014 23:41:29 +0200 Subject: add pkgAcquire::TransactionHasError() --- apt-pkg/acquire-item.cc | 7 ++++--- apt-pkg/acquire.cc | 23 ++++++++++++++++++++--- apt-pkg/acquire.h | 1 + test/integration/test-apt-update-transactions | 23 +++++++++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) create mode 100755 test/integration/test-apt-update-transactions diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index bd11ba66c..81afdf4b1 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1421,12 +1421,12 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList DestFile += URItoFileName(RealURI); // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved - pkgAcqMetaIndex *mi = new pkgAcqMetaIndex( + pkgAcqMetaIndex *metaindex = new pkgAcqMetaIndex( Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, DestFile, IndexTargets, MetaIndexParser); - TransactionID = (unsigned long)mi; + TransactionID = (unsigned long)metaindex; } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ @@ -1908,7 +1908,8 @@ void pkgAcqMetaIndex::Finished() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "Finished: " << DestFile <CommitTransaction((unsigned long)this); + if(Owner->TransactionHasError((unsigned long)this) == false) + Owner->CommitTransaction((unsigned long)this); } diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 2e2e39d51..4b82fa46d 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -185,11 +185,21 @@ void pkgAcquire::AbortTransaction(unsigned long TransactionID) { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << " Cancel: " << (*I)->DestFile << std::endl; - Dequeue(*I); + //Dequeue(*I); (*I)->Status = pkgAcquire::Item::StatError; } } /*}}}*/ +bool pkgAcquire::TransactionHasError(unsigned long TransactionID) +{ + std::vector Transaction; + for (ItemIterator I = Items.begin(); I != Items.end(); ++I) + if((*I)->TransactionID == TransactionID) + if((*I)->Status == pkgAcquire::Item::StatError || + (*I)->Status == pkgAcquire::Item::StatAuthError) + return true; + return false; +} // Acquire::CommitTransaction - Commit a transaction /*{{{*/ void pkgAcquire::CommitTransaction(unsigned long TransactionID) { @@ -201,18 +211,25 @@ void pkgAcquire::CommitTransaction(unsigned long TransactionID) if((*I)->TransactionID == TransactionID) Transaction.push_back(*I); + // move new files into place *and* remove files that are not + // part of the transaction but are still on disk for (std::vector::iterator I = Transaction.begin(); I != Transaction.end(); ++I) { - if((*I)->PartialFile != "" && - (*I)->Status == pkgAcquire::Item::StatDone) + if((*I)->PartialFile != "") { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "mv " << (*I)->PartialFile << " -> " << (*I)->DestFile << std::endl; + Rename((*I)->PartialFile, (*I)->DestFile); chmod((*I)->DestFile.c_str(),0644); + } else { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "rm " + << (*I)->DestFile << std::endl; + unlink((*I)->DestFile.c_str()); } } } diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 0b955cc76..7f51dd8f5 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -251,6 +251,7 @@ class pkgAcquire /** \brief Abort a given transaction */ void AbortTransaction(unsigned long TransactionID); void CommitTransaction(unsigned long TransactionID); + bool TransactionHasError(unsigned long TransactionID); /** \brief Retrieve information about a fetch method by name. * diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions new file mode 100755 index 000000000..ee8d20dbf --- /dev/null +++ b/test/integration/test-apt-update-transactions @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +insertpackage 'unstable' 'foo' 'all' '1.0' + +setupaptarchive --no-update +changetowebserver + +# break package file +cat > aptarchive/dists/unstable/main/binary-i386/Packages < Date: Mon, 21 Jul 2014 11:19:37 +0200 Subject: Download Release first, then Release.gpg The old way of handling this was that pkgAcqMetaIndex was responsible to check/move both Release and Release.gpg in place. This breaks the assumption of the transaction that each pkgAcquire::Item has a single File that its responsible for. --- apt-pkg/acquire-item.cc | 93 ++++++++++++++------------- apt-pkg/acquire-item.h | 35 +++++----- apt-pkg/deb/debmetaindex.cc | 16 ++--- methods/gpgv.cc | 2 +- test/integration/test-apt-update-transactions | 1 + 5 files changed, 79 insertions(+), 68 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 81afdf4b1..fa41bca7a 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -88,7 +88,8 @@ pkgAcquire::Item::~Item() void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { Status = StatIdle; - ErrorText = LookupTag(Message,"Message"); + if(ErrorText == "") + ErrorText = LookupTag(Message,"Message"); UsedMirror = LookupTag(Message,"UsedMirror"); if (QueueCounter <= 1) { @@ -1314,19 +1315,20 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, /*{{{*/ string URI,string URIDesc,string ShortDesc, - string MetaIndexURI, string MetaIndexURIDesc, - string MetaIndexShortDesc, + string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI), - MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), - MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) + Item(MetaOwner->GetOwner(), HashStringList()), RealURI(URI), + MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), + IndexTargets(IndexTargets), AuthPass(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); + TransactionID = (unsigned long)MetaOwner; + // remove any partial downloaded sig-file in partial/. // it may confuse proxies and is too small to warrant a // partial download anyway @@ -1337,7 +1339,8 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ Desc.Owner = this; Desc.ShortDesc = ShortDesc; Desc.URI = URI; - + +#if 0 string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); if (RealFileExists(Final) == true) @@ -1350,9 +1353,9 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ LastGoodSig = DestFile+".reverify"; Rename(Final,LastGoodSig); } - +#endif // we expect the indextargets + one additional Release file - ExpectedAdditionalItems = IndexTargets->size() + 1; + //ExpectedAdditionalItems = IndexTargets->size() + 1; QueueURI(Desc); } @@ -1404,10 +1407,17 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList return; } - Complete = true; + // queue for verify + if(AuthPass == false) + { + AuthPass = true; + Desc.URI = "gpgv:" + DestFile; + DestFile = MetaIndexFile; + QueueURI(Desc); + return; + } - // at this point pkgAcqMetaIndex takes over - ExpectedAdditionalItems = 0; + Complete = true; // put the last known good file back on i-m-s hit (it will // be re-verified again) @@ -1416,10 +1426,13 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList Rename(LastGoodSig, DestFile); // queue for copy - PartialFile = DestFile; + PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; + PartialFile += URItoFileName(RealURI); + DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); +#if 0 // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved pkgAcqMetaIndex *metaindex = new pkgAcqMetaIndex( Owner, MetaIndexURI, MetaIndexURIDesc, @@ -1427,15 +1440,12 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList MetaIndexParser); TransactionID = (unsigned long)metaindex; +#endif } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - - // at this point pkgAcqMetaIndex takes over - ExpectedAdditionalItems = 0; - // if we get a network error we fail gracefully if(Status == StatTransientNetworkError) { @@ -1451,11 +1461,11 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // Delete any existing sigfile when the acquire failed unlink(Final.c_str()); - +#if 0 // queue a pkgAcqMetaIndex with no sigfile new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, - "", IndexTargets, MetaIndexParser); - + "", IndexTargets, MetaIndexParser); +#endif if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -1471,11 +1481,13 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string URI,string URIDesc,string ShortDesc, - string SigFile, + string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), - MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false) + Item(Owner, HashStringList()), RealURI(URI), IndexTargets(IndexTargets), + MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), + MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), + MetaIndexSigShortDesc(MetaIndexSigShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -1490,7 +1502,6 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ // we expect more item ExpectedAdditionalItems = IndexTargets->size(); - QueueURI(Desc); } /*}}}*/ @@ -1534,10 +1545,9 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList if (SigFile == "") { - // There was no signature file, so we are finished. Download - // the indexes and do only hashsum verification if possible + // load indexes, the signature will downloaded afterwards MetaIndexParser->Load(DestFile); - QueueIndexes(false); + QueueIndexes(true); } else { @@ -1615,6 +1625,13 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ } DestFile = FinalFile; } + + // queue a signature + if(SigFile != DestFile) + new pkgAcqMetaSig(this, MetaIndexSigURI, MetaIndexSigURIDesc, + MetaIndexSigShortDesc, DestFile, IndexTargets, + MetaIndexParser); + Complete = true; } /*}}}*/ @@ -1659,16 +1676,6 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ /*}}}*/ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { -#if 0 - /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642) - * FIXME: Disabled; it breaks unsigned repositories without hashes */ - if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile)) - { - Status = StatError; - ErrorText = MetaIndexParser->ErrorText; - return; - } -#endif bool transInRelease = false; { std::vector const keys = MetaIndexParser->MetaKeys(); @@ -1919,9 +1926,9 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), - MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), - MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) + pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), + MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { SigFile = DestFile; @@ -1983,9 +1990,9 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* if (FileExists(FinalFile)) unlink(FinalFile.c_str()); - new pkgAcqMetaSig(Owner, - MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, + new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, + MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 6c9fec695..6235c353b 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -450,6 +450,15 @@ class pkgAcqMetaIndex : public pkgAcquire::Item * no hashsum checking will be performed. */ void QueueIndexes(bool verify); + + /** \brief The URI of the meta-index file for the detached signature */ + std::string MetaIndexSigURI; + + /** \brief A "URI-style" description of the meta-index file */ + std::string MetaIndexSigURIDesc; + + /** \brief A brief description of the meta-index file */ + std::string MetaIndexSigShortDesc; public: @@ -464,7 +473,7 @@ class pkgAcqMetaIndex : public pkgAcquire::Item /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, std::string URI,std::string URIDesc, std::string ShortDesc, - std::string SigFile, + std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc, const std::vector* IndexTargets, indexRecords* MetaIndexParser); }; @@ -1009,22 +1018,12 @@ class pkgAcqMetaSig : public pkgAcquire::Item */ std::string RealURI; - /** \brief The URI of the meta-index file to be fetched after the signature. */ - std::string MetaIndexURI; - - /** \brief A "URI-style" description of the meta-index file to be - * fetched after the signature. - */ - std::string MetaIndexURIDesc; - - /** \brief A brief description of the meta-index file to be fetched - * after the signature. - */ - std::string MetaIndexShortDesc; - /** \brief A package-system-specific parser for the meta-index file. */ indexRecords* MetaIndexParser; + /** \brief The file we need to verify */ + std::string MetaIndexFile; + /** \brief The index files which should be looked up in the meta-index * and then downloaded. * @@ -1032,6 +1031,9 @@ class pkgAcqMetaSig : public pkgAcquire::Item */ const std::vector* IndexTargets; + /** \brief if we are in fetching or download state */ + bool AuthPass; + public: // Specialized action members @@ -1042,8 +1044,9 @@ class pkgAcqMetaSig : public pkgAcquire::Item virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaSig. */ - pkgAcqMetaSig(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, - std::string MetaIndexURI, std::string MetaIndexURIDesc, std::string MetaIndexShortDesc, + pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, + std::string URI,std::string URIDesc, std::string ShortDesc, + std::string MetaIndexFile, const std::vector* IndexTargets, indexRecords* MetaIndexParser); virtual ~pkgAcqMetaSig(); diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 73010e867..98f99e888 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -267,24 +267,24 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const if (tryInRelease == false) new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), new indexRecords (Dist)); } if (tryInRelease == true) - new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"), - MetaIndexInfo("InRelease"), "InRelease", + new pkgAcqMetaClearSig(Owner, + MetaIndexURI("InRelease"), MetaIndexInfo("InRelease"), "InRelease", MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), new indexRecords (Dist)); else - new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"), - MetaIndexInfo("Release.gpg"), "Release.gpg", - MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - ComputeIndexTargets(), - new indexRecords (Dist)); + new pkgAcqMetaIndex(Owner, + MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", + ComputeIndexTargets(), + new indexRecords (Dist)); return true; } diff --git a/methods/gpgv.cc b/methods/gpgv.cc index ae521a2ed..30fd217bd 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -159,7 +159,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, waitpid(pid, &status, 0); if (Debug == true) { - std::clog << "gpgv exited\n"; + ioprintf(std::clog, "gpgv exited with status %i\n", WEXITSTATUS(status)); } if (WEXITSTATUS(status) == 0) diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index ee8d20dbf..247334991 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -21,3 +21,4 @@ compressfile aptarchive/dists/unstable/main/binary-i386/Packages '+1hour' # ensure that a update will only succeed entirely or not at all testfailure aptget update testequal "partial" ls rootdir/var/lib/apt/lists + -- cgit v1.2.3 From 1f4dd8fd8489fbfd62c4d1667f159433a5f532b2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 22 Jul 2014 11:35:30 +0200 Subject: WIP cleanup pkgAcqMetaSig --- apt-pkg/acquire-item.cc | 119 ++++++++++++++++-------------------------------- apt-pkg/acquire-item.h | 5 +- apt-pkg/acquire.cc | 1 - 3 files changed, 42 insertions(+), 83 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index fa41bca7a..01e1841d6 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -135,7 +135,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); - UsedMirror = LookupTag(Message,"UsedMirror"); + UsedMirror = LookupTag(Message,"UsedMirror"); if (Complete == false && !Local && FileName == DestFile) { if (Owner->Log != 0) @@ -1322,55 +1322,30 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, /*{{{*/ indexRecords* MetaIndexParser) : Item(MetaOwner->GetOwner(), HashStringList()), RealURI(URI), MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), - IndexTargets(IndexTargets), AuthPass(false) + IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - TransactionID = (unsigned long)MetaOwner; - // remove any partial downloaded sig-file in partial/. // it may confuse proxies and is too small to warrant a // partial download anyway unlink(DestFile.c_str()); + // set the TransactionID + TransactionID = (unsigned long)MetaOwner; + // Create the item Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; Desc.URI = URI; -#if 0 - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(RealURI); - if (RealFileExists(Final) == true) - { - // File was already in place. It needs to be re-downloaded/verified - // because Release might have changed, we do give it a different - // name than DestFile because otherwise the http method will - // send If-Range requests and there are too many broken servers - // out there that do not understand them - LastGoodSig = DestFile+".reverify"; - Rename(Final,LastGoodSig); - } -#endif - // we expect the indextargets + one additional Release file - //ExpectedAdditionalItems = IndexTargets->size() + 1; - QueueURI(Desc); } /*}}}*/ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ { - // if the file was never queued undo file-changes done in the constructor - if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false && - LastGoodSig.empty() == false) - { - string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) - Rename(LastGoodSig, Final); - } - } /*}}}*/ // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1378,8 +1353,11 @@ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ /* The only header we use is the last-modified header. */ string pkgAcqMetaSig::Custom600Headers() const { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + struct stat Buf; - if (stat(LastGoodSig.c_str(),&Buf) != 0) + if (stat(FinalFile.c_str(),&Buf) != 0) return "\nIndex-File: true"; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -1407,6 +1385,18 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList return; } + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + IMSHit = true; + + // adjust paths if its a ims-hit + if(IMSHit) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + DestFile = PartialFile = FinalFile; + } + // queue for verify if(AuthPass == false) { @@ -1417,65 +1407,31 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList return; } - Complete = true; - - // put the last known good file back on i-m-s hit (it will - // be re-verified again) - // Else do nothing, we have the new file in DestFile then - if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) - Rename(LastGoodSig, DestFile); - - // queue for copy - PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; - PartialFile += URItoFileName(RealURI); - - DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI); + // queue to copy the file in place if it was not a ims hit, on ims + // hit the file is already at the right place + if(IMSHit == false) + { + PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; + PartialFile += URItoFileName(RealURI); + + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + } -#if 0 - // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved - pkgAcqMetaIndex *metaindex = new pkgAcqMetaIndex( - Owner, MetaIndexURI, MetaIndexURIDesc, - MetaIndexShortDesc, DestFile, IndexTargets, - MetaIndexParser); + Complete = true; - TransactionID = (unsigned long)metaindex; -#endif } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - // if we get a network error we fail gracefully - if(Status == StatTransientNetworkError) - { - Item::Failed(Message,Cnf); - // move the sigfile back on transient network failures - if(FileExists(LastGoodSig)) - Rename(LastGoodSig,Final); - // set the status back to , Item::Failed likes to reset it - Status = pkgAcquire::Item::StatTransientNetworkError; - return; - } + // this ensures that any file in the lists/ dir is removed by the + // transaction + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + PartialFile = ""; - // Delete any existing sigfile when the acquire failed - unlink(Final.c_str()); -#if 0 - // queue a pkgAcqMetaIndex with no sigfile - new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, - "", IndexTargets, MetaIndexParser); -#endif - if (Cnf->LocalOnly == true || - StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) - { - // Ignore this - Status = StatDone; - Complete = false; - Dequeue(); - return; - } - Item::Failed(Message,Cnf); } /*}}}*/ @@ -1582,6 +1538,7 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList FinalFile += URItoFileName(RealURI); if (SigFile == DestFile) SigFile = FinalFile; + // queue for copy in place PartialFile = DestFile; DestFile = FinalFile; } diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 6235c353b..11a596ad5 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -1031,9 +1031,12 @@ class pkgAcqMetaSig : public pkgAcquire::Item */ const std::vector* IndexTargets; - /** \brief if we are in fetching or download state */ + /** \brief If we are in fetching or download state */ bool AuthPass; + /** \brief Was this file already on disk */ + bool IMSHit; + public: // Specialized action members diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 4b82fa46d..b14a54f0f 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -222,7 +222,6 @@ void pkgAcquire::CommitTransaction(unsigned long TransactionID) std::clog << "mv " << (*I)->PartialFile << " -> " << (*I)->DestFile << std::endl; - Rename((*I)->PartialFile, (*I)->DestFile); chmod((*I)->DestFile.c_str(),0644); } else { -- cgit v1.2.3 From e05672e88678f520b2db59599e939345ad0b6e53 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 31 Jul 2014 09:53:13 +0200 Subject: Rework TransactionID stuff --- apt-pkg/acquire-item.cc | 140 +++++++++++++++-------- apt-pkg/acquire-item.h | 47 ++++---- apt-pkg/acquire.cc | 3 + apt-pkg/deb/debmetaindex.cc | 13 ++- test/integration/test-apt-update-rollback | 180 ++++++++++++++++++++++++++++++ 5 files changed, 305 insertions(+), 78 deletions(-) create mode 100755 test/integration/test-apt-update-rollback diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 01e1841d6..47cd9eab2 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -64,10 +64,12 @@ static void printHashSumComparision(std::string const &URI, HashStringList const /*}}}*/ // Acquire::Item::Item - Constructor /*{{{*/ -pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) : - Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0), TransactionID(0), ExpectedAdditionalItems(0), - ExpectedHashes(ExpectedHashes) +pkgAcquire::Item::Item(pkgAcquire *Owner, + HashStringList const &ExpectedHashes, + unsigned long TransactionID) + : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), + Local(false), QueueCounter(0), TransactionID(TransactionID), + ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes) { Owner->Add(this); Status = StatIdle; @@ -239,10 +241,12 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) // --------------------------------------------------------------------- /* Get a sub-index file based on checksums from a 'master' file and possibly query additional files */ -pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, - string const &URIDesc, string const &ShortDesc, - HashStringList const &ExpectedHashes) - : Item(Owner, ExpectedHashes) +pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, + unsigned long TransactionID, + string const &URI, + string const &URIDesc, string const &ShortDesc, + HashStringList const &ExpectedHashes) + : Item(Owner, ExpectedHashes, TransactionID) { /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); @@ -354,11 +358,12 @@ bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ * patches. If anything goes wrong in that process, it will fall back to * the original packages file */ -pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcqMetaIndex *MetaOwner, +pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, + unsigned long TransactionID, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(MetaOwner, Target, ExpectedHashes, + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser) { @@ -457,7 +462,8 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ std::clog << "Package file is up-to-date" << std::endl; // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(MetaOwner, Target, ExpectedHashes, MetaIndexParser, + new pkgAcqIndexDiffs(Owner, TransactionID, Target, + ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return true; } @@ -544,14 +550,17 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (pdiff_merge == false) { - new pkgAcqIndexDiffs(MetaOwner, Target, ExpectedHashes, MetaIndexParser, + new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes, + MetaIndexParser, ServerSha1, available_patches); } else { std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) - (*diffs)[i] = new pkgAcqIndexMergeDiffs(MetaOwner, Target, + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, + TransactionID, + Target, ExpectedHashes, MetaIndexParser, available_patches[i], @@ -579,7 +588,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; @@ -621,13 +630,14 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList /* The package diff is added to the queue. one object is constructed * for each diff and the index */ -pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcqMetaIndex *MetaOwner, +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { @@ -659,7 +669,7 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@ -799,7 +809,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(MetaOwner, Target, + new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return Finish(); @@ -809,13 +819,14 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi } /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ -pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcqMetaIndex *MetaOwner, +pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -858,7 +869,7 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*C // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -941,7 +952,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashStringList const &ExpectedHash, string comprExt) - : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI) + : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI) { if(comprExt.empty() == true) { @@ -969,18 +980,21 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, } #endif /*}}}*/ -pkgAcqIndex::pkgAcqIndex(pkgAcqMetaIndex *MetaOwner, +pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, + unsigned long TransactionID, IndexTarget const *Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(MetaOwner->GetOwner(), Target, ExpectedHash, + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHash, MetaIndexParser), RealURI(Target->URI) { // autoselect the compression method AutoSelectCompression(); Init(Target->URI, Target->Description, Target->ShortDesc); - TransactionID = (unsigned long)MetaOwner; + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgIndex with TransactionID " + << TransactionID << std::endl; } /*}}}*/ void pkgAcqIndex::AutoSelectCompression() @@ -1265,9 +1279,12 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, { } /*}}}*/ -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcqMetaIndex *MetaOwner, IndexTarget const * const Target, - HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqIndex(MetaOwner, Target, ExpectedHashes, MetaIndexParser) +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, + unsigned long TransactionID, + IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser) { // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); @@ -1302,6 +1319,7 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) return; } + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -1315,12 +1333,13 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, /*{{{*/ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ + unsigned long TransactionID, string URI,string URIDesc,string ShortDesc, string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(MetaOwner->GetOwner(), HashStringList()), RealURI(URI), + Item(Owner, HashStringList(), TransactionID), RealURI(URI), MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { @@ -1333,7 +1352,9 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, /*{{{*/ unlink(DestFile.c_str()); // set the TransactionID - TransactionID = (unsigned long)MetaOwner; + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaSig with TransactionID " + << TransactionID << std::endl; // Create the item Desc.Description = URIDesc; @@ -1432,35 +1453,56 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ DestFile += URItoFileName(RealURI); PartialFile = ""; + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) + { + // Ignore this + Status = StatDone; + Complete = false; + Dequeue(); + return; + } Item::Failed(Message,Cnf); } /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ + unsigned long TransactionID, string URI,string URIDesc,string ShortDesc, string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), IndexTargets(IndexTargets), + Item(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets), MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) { - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + if(TransactionID == 0) + this->TransactionID = (unsigned long)this; + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaIndex with TransactionID " + << TransactionID << std::endl; - TransactionID = (unsigned long)this; + Init(URIDesc, ShortDesc); +} + /*}}}*/ +// pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/ +void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) +{ + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); // Create the item Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; - Desc.URI = URI; + Desc.URI = RealURI; // we expect more item ExpectedAdditionalItems = IndexTargets->size(); QueueURI(Desc); } - /*}}}*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ @@ -1585,7 +1627,8 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ // queue a signature if(SigFile != DestFile) - new pkgAcqMetaSig(this, MetaIndexSigURI, MetaIndexSigURIDesc, + new pkgAcqMetaSig(Owner, TransactionID, + MetaIndexSigURI, MetaIndexSigURIDesc, MetaIndexSigShortDesc, DestFile, IndexTargets, MetaIndexParser); @@ -1695,15 +1738,16 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ if ((*Target)->IsOptional() == true) { if ((*Target)->IsSubIndex() == true) - new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, + new pkgAcqSubIndex(Owner, TransactionID, + (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, ExpectedIndexHashes); else if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(this, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(this, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndexTrans(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@ -1714,9 +1758,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(this, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(this, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@ -1872,8 +1916,9 @@ void pkgAcqMetaIndex::Finished() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "Finished: " << DestFile <TransactionHasError((unsigned long)this) == false) - Owner->CommitTransaction((unsigned long)this); + if(Owner->TransactionHasError(TransactionID) == false && + TransactionID > 0) + Owner->CommitTransaction(TransactionID); } @@ -1883,7 +1928,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), + pkgAcqMetaIndex(Owner, (unsigned long)this, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { @@ -1892,7 +1937,6 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ // index targets + (worst case:) Release/Release.gpg ExpectedAdditionalItems = IndexTargets->size() + 2; - // keep the old InRelease around in case of transistent network errors string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); if (RealFileExists(Final) == true) @@ -1947,7 +1991,7 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* if (FileExists(FinalFile)) unlink(FinalFile.c_str()); - new pkgAcqMetaIndex(Owner, + new pkgAcqMetaIndex(Owner, TransactionID, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 11a596ad5..3f7cca083 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -297,7 +297,8 @@ class pkgAcquire::Item : public WeakPointable * \param ExpectedHashes of the file represented by this item */ Item(pkgAcquire *Owner, - HashStringList const &ExpectedHashes=HashStringList()); + HashStringList const &ExpectedHashes=HashStringList(), + unsigned long TransactionID=0); /** \brief Remove this item from its owner's queue by invoking * pkgAcquire::Remove. @@ -370,7 +371,9 @@ class pkgAcqSubIndex : public pkgAcquire::Item * * \param ExpectedHashes The list file's hashsums which are expected. */ - pkgAcqSubIndex(pkgAcquire *Owner, std::string const &URI,std::string const &URIDesc, + pkgAcqSubIndex(pkgAcquire *Owner, + unsigned long TransactionID, + std::string const &URI,std::string const &URIDesc, std::string const &ShortDesc, HashStringList const &ExpectedHashes); }; /*}}}*/ @@ -459,6 +462,9 @@ class pkgAcqMetaIndex : public pkgAcquire::Item /** \brief A brief description of the meta-index file */ std::string MetaIndexSigShortDesc; + + /** \brief delayed constructor */ + void Init(std::string URIDesc, std::string ShortDesc); public: @@ -472,6 +478,7 @@ class pkgAcqMetaIndex : public pkgAcquire::Item /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, + unsigned long TransactionID, std::string URI,std::string URIDesc, std::string ShortDesc, std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc, const std::vector* IndexTargets, @@ -525,22 +532,14 @@ class pkgAcqBaseIndex : public pkgAcquire::Item */ const struct IndexTarget * Target; indexRecords *MetaIndexParser; - pkgAcqMetaIndex *MetaOwner; - - pkgAcqBaseIndex(pkgAcqMetaIndex *MetaOwner, - struct IndexTarget const * const Target, - HashStringList const &ExpectedHashes, - indexRecords *MetaIndexParser) - : Item(MetaOwner->GetOwner(), ExpectedHashes), Target(Target), - MetaIndexParser(MetaIndexParser), MetaOwner(MetaOwner) {}; pkgAcqBaseIndex(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) : Item(Owner, ExpectedHashes), Target(Target), - MetaIndexParser(MetaIndexParser), MetaOwner(0) {}; - + MetaIndexParser(MetaIndexParser) {}; }; /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ @@ -606,7 +605,8 @@ class pkgAcqDiffIndex : public pkgAcqBaseIndex * * \param ExpectedHashes The list file's hashsums which are expected. */ - pkgAcqDiffIndex(pkgAcqMetaIndex *MetaIndexOwner, + pkgAcqDiffIndex(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); @@ -694,7 +694,8 @@ class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex * \param allPatches contains all related items so that each item can * check if it was the last one to complete the download step */ - pkgAcqIndexMergeDiffs(pkgAcqMetaIndex *MetaIndexOwner, + pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser, @@ -822,7 +823,8 @@ class pkgAcqIndexDiffs : public pkgAcqBaseIndex * should be ordered so that each diff appears before any diff * that depends on it. */ - pkgAcqIndexDiffs(pkgAcqMetaIndex *MetaIndexOwner, + pkgAcqIndexDiffs(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser, @@ -899,13 +901,7 @@ class pkgAcqIndex : public pkgAcqBaseIndex pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, HashStringList const &ExpectedHashes, std::string compressExt=""); -#if 0 - pkgAcqIndex(pkgAcquire *Owner, - IndexTarget const * const Target, - HashStringList const &ExpectedHash, - indexRecords *MetaIndexParser); -#endif - pkgAcqIndex(pkgAcqMetaIndex *MetaIndexOwner, + pkgAcqIndex(pkgAcquire *Owner, unsigned long TransactionID, IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser); @@ -942,7 +938,9 @@ class pkgAcqIndexTrans : public pkgAcqIndex pkgAcqIndexTrans(pkgAcquire *Owner, std::string URI,std::string URIDesc, std::string ShortDesc); - pkgAcqIndexTrans(pkgAcqMetaIndex *Owner, IndexTarget const * const Target, + pkgAcqIndexTrans(pkgAcquire *Owner, + unsigned long TransactionID, + IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); }; @@ -1047,7 +1045,8 @@ class pkgAcqMetaSig : public pkgAcquire::Item virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaSig. */ - pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, + pkgAcqMetaSig(pkgAcquire *Owner, + unsigned long TransactionID, std::string URI,std::string URIDesc, std::string ShortDesc, std::string MetaIndexFile, const std::vector* IndexTargets, diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index b14a54f0f..33afd8f1f 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -198,6 +198,7 @@ bool pkgAcquire::TransactionHasError(unsigned long TransactionID) if((*I)->Status == pkgAcquire::Item::StatError || (*I)->Status == pkgAcquire::Item::StatAuthError) return true; + return false; } // Acquire::CommitTransaction - Commit a transaction /*{{{*/ @@ -230,6 +231,8 @@ void pkgAcquire::CommitTransaction(unsigned long TransactionID) << (*I)->DestFile << std::endl; unlink((*I)->DestFile.c_str()); } + // mark that this transaction is finished + (*I)->TransactionID = 0; } } /*}}}*/ diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 98f99e888..b1dc060fe 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -265,11 +265,12 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const // this is normally created in pkgAcqMetaSig, but if we run // in --print-uris mode, we add it here if (tryInRelease == false) - new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"), - MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", - ComputeIndexTargets(), - new indexRecords (Dist)); + new pkgAcqMetaIndex(Owner, 0, + MetaIndexURI("Release"), + MetaIndexInfo("Release"), "Release", + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", + ComputeIndexTargets(), + new indexRecords (Dist)); } if (tryInRelease == true) @@ -280,7 +281,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const ComputeIndexTargets(), new indexRecords (Dist)); else - new pkgAcqMetaIndex(Owner, + new pkgAcqMetaIndex(Owner, 0, MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback new file mode 100755 index 000000000..cd28f1f1f --- /dev/null +++ b/test/integration/test-apt-update-rollback @@ -0,0 +1,180 @@ +#!/bin/sh +# +# test that apt-get update is transactional +# +set -e + +avoid_ims_hit() { + touch -d '+1hour' aptarchive/dists/unstable/main/binary-i386/Packages* + touch -d '+1hour' aptarchive/dists/unstable/main/source/Sources* + touch -d '+1hour' aptarchive/dists/unstable/*Release* + + touch -d '-1hour' rootdir/var/lib/apt/lists/* +} + +create_fresh_archive() +{ + rm -rf aptarchive/* + rm -f rootdir/var/lib/apt/lists/_* rootdir/var/lib/apt/lists/partial/* + + insertpackage 'unstable' 'old' 'all' '1.0' + + setupaptarchive +} + +add_new_package() { + insertpackage "unstable" "new" "all" "1.0" + insertsource "unstable" "new" "all" "1.0" + + setupaptarchive --no-update + + avoid_ims_hit +} + +break_repository_sources_index() { + printf "xxx" > $APTARCHIVE/dists/unstable/main/source/Sources + gzip -c $APTARCHIVE/dists/unstable/main/source/Sources > \ + $APTARCHIVE/dists/unstable/main/source/Sources.gz + avoid_ims_hit +} + +test_inrelease_to_new_inrelease() { + msgmsg "Test InRelease to new InRelease works fine" + create_fresh_archive + testequal "old/unstable 1.0 all" apt list -q + + add_new_package + testsuccess aptget update + + testequal "new/unstable 1.0 all +old/unstable 1.0 all" apt list -q +} + +test_inrelease_to_broken_hash_reverts_all() { + msgmsg "Test InRelease to broken InRelease reverts everything" + create_fresh_archive + add_new_package + # break the Sources file + break_repository_sources_index + + # test the error condition + testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease + +W: Failed to fetch copy:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch + +W: Failed to fetch copy:${APTARCHIVE}/dists/unstable/main/binary-i386/Packages + +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq + # ensure that the Packages file is also rolled back + testequal "E: Unable to locate package new" aptget install new -s -qq +} + +test_inreleae_to_valid_release() { + msgmsg "Test InRelease to valid Release" + create_fresh_archive + add_new_package + # switch to a unsinged repo now + rm $APTARCHIVE/dists/unstable/InRelease + rm $APTARCHIVE/dists/unstable/Release.gpg + avoid_ims_hit + + # update works + testsuccess aptget update -o Debug::Acquire::Transaction=1 + + # test that we can install the new packages but do no longer have a sig + testsuccess aptget install old -s + testsuccess aptget install new -s + testfailure ls $ROOTDIR/var/lib/apt/lists/*_InRelease + testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release.gpg + testsuccess ls $ROOTDIR/var/lib/apt/lists/*_Release +} + +test_inreleae_to_release_reverts_all() { + msgmsg "Test InRelease to broken Release reverts everything" + create_fresh_archive + + # switch to a unsinged repo now + add_new_package + rm $APTARCHIVE/dists/unstable/InRelease + rm $APTARCHIVE/dists/unstable/Release.gpg + # break it + break_repository_sources_index + + # ensure error + testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease + +W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release + +W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release.gpg + +W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch + +W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/binary-i386/Packages + +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq # -o Debug::acquire::transaction=1 + + # ensure that the Packages file is also rolled back + testsuccess aptget install old -s + testfailure aptget install new -s + testsuccess ls $ROOTDIR/var/lib/apt/lists/*_InRelease + testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release +} + +test_unauthenticated_to_invalid_inrelease() { + msgmsg "Test UnAuthenticated to invalid InRelease reverts everything" + create_fresh_archive + rm $APTARCHIVE/dists/unstable/InRelease + rm $APTARCHIVE/dists/unstable/Release.gpg + avoid_ims_hit + + testsuccess aptget update -qq + testequal "WARNING: The following packages cannot be authenticated! + old +E: There are problems and -y was used without --force-yes" aptget install -qq -y old + + # go to authenticated but not correct + add_new_package + break_repository_sources_index + + testequal "W: Hashsum mismatch $ROOTDIR/var/lib/apt/lists/${APTARCHIVE_LISTS}_dists_unstable_main_source_Sources +W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease + +W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch + +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq + + testfailure ls rootdir/var/lib/apt/lists/*_InRelease + testequal "WARNING: The following packages cannot be authenticated! + old +E: There are problems and -y was used without --force-yes" aptget install -qq -y old +} + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +# setup the archive and ensure we have a single package that installs fine +setupaptarchive +APTARCHIVE=$(readlink -f ./aptarchive) +ROOTDIR=${TMPWORKINGDIRECTORY}/rootdir +APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" + +# test the following cases: +# - InRelease -> broken InRelease revert to previous state +# - empty lists dir and broken remote leaves nothing on the system +# - InRelease -> hashsum mismatch for one file reverts all files to previous state +# - Release/Release.gpg -> hashsum mismatch +# - InRelease -> Release with hashsum mismatch revert entire state and kills Release +# - Release -> InRelease with broken Sig/Hash removes InRelease +# going from Release/Release.gpg -> InRelease and vice versa +# - unauthenticated -> invalid InRelease + +test_inrelease_to_new_inrelease +test_inrelease_to_broken_hash_reverts_all + +test_inreleae_to_valid_release +test_inreleae_to_release_reverts_all + +#test_unauthenticated_to_invalid_inrelease -- cgit v1.2.3 From 7712d13bcc19c7b3c44d8fea121baaa587cb10a4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 31 Jul 2014 10:00:26 +0200 Subject: make test_inreleae_to_release_reverts_all work --- apt-pkg/acquire-item.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 47cd9eab2..31dc2073d 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1616,11 +1616,13 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ if (SigFile == DestFile) { SigFile = FinalFile; +#if 0 // constructor of pkgAcqMetaClearSig moved it out of the way, // now move it back in on IMS hit for the 'old' file string const OldClearSig = DestFile + ".reverify"; if (RealFileExists(OldClearSig) == true) Rename(OldClearSig, FinalFile); +#endif } DestFile = FinalFile; } @@ -1937,6 +1939,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ // index targets + (worst case:) Release/Release.gpg ExpectedAdditionalItems = IndexTargets->size() + 2; +#if 0 // keep the old InRelease around in case of transistent network errors string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); if (RealFileExists(Final) == true) @@ -1944,10 +1947,12 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const LastGoodSig = DestFile + ".reverify"; Rename(Final,LastGoodSig); } +#endif } /*}}}*/ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ { +#if 0 // if the file was never queued undo file-changes done in the constructor if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false) { @@ -1956,6 +1961,7 @@ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) Rename(LastGoodSig, Final); } +#endif } /*}}}*/ // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1969,7 +1975,6 @@ string pkgAcqMetaClearSig::Custom600Headers() const struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) { - Final = DestFile + ".reverify"; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true\nFail-Ignore: true\n"; } @@ -1984,12 +1989,13 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* if (AuthPass == false) { - // Remove the 'old' InRelease file if we try Release.gpg now as otherwise - // the file will stay around and gives a false-auth impression (CVE-2012-0214) + // Queue the 'old' InRelease file for removal if we try Release.gpg + // as otherwise the file will stay around and gives a false-auth + // impression (CVE-2012-0214) string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile.append(URItoFileName(RealURI)); - if (FileExists(FinalFile)) - unlink(FinalFile.c_str()); + PartialFile = ""; + DestFile = FinalFile; new pkgAcqMetaIndex(Owner, TransactionID, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, -- cgit v1.2.3 From 6d979490c13e9d8004942507c99d152c22184a27 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 31 Jul 2014 10:02:27 +0200 Subject: make test_unauthenticated_to_invalid_inrelease work --- test/integration/test-apt-update-rollback | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index cd28f1f1f..9771f0edc 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -136,11 +136,12 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y add_new_package break_repository_sources_index - testequal "W: Hashsum mismatch $ROOTDIR/var/lib/apt/lists/${APTARCHIVE_LISTS}_dists_unstable_main_source_Sources -W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease - + testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease + W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch +W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/binary-i386/Packages + E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq testfailure ls rootdir/var/lib/apt/lists/*_InRelease @@ -177,4 +178,4 @@ test_inrelease_to_broken_hash_reverts_all test_inreleae_to_valid_release test_inreleae_to_release_reverts_all -#test_unauthenticated_to_invalid_inrelease +test_unauthenticated_to_invalid_inrelease -- cgit v1.2.3 From c5fced388848b967f0ce076656cad5366517f981 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 31 Jul 2014 18:40:05 +0200 Subject: ensure InRelease->Release is transactional as well --- apt-pkg/acquire-item.cc | 5 +++-- test/integration/test-apt-update-rollback | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 31dc2073d..2d9328b6b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1888,6 +1888,7 @@ void pkgAcqMetaIndex::Failed(string /*Message*/, ReportMirrorFailure("GPGFailure"); } #endif + /* Always move the meta index, even if gpgv failed. This ensures * that PackageFile objects are correctly filled in */ if (FileExists(DestFile)) { @@ -1902,9 +1903,9 @@ void pkgAcqMetaIndex::Failed(string /*Message*/, "Release"); SigFile = FinalFile; } - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); + // Done, queue for rename on transaction finished + PartialFile = DestFile; DestFile = FinalFile; } diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index 9771f0edc..c16e4f480 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -150,6 +150,22 @@ E: Some index files failed to download. They have been ignored, or old ones used E: There are problems and -y was used without --force-yes" aptget install -qq -y old } +test_inrelease_to_unauth_inrelease() { + msgmsg "Test InRelease to InRelease without sig" + create_fresh_archive + signreleasefiles 'Marvin Paranoid' + avoid_ims_hit + + testsuccess aptget update -qq + + testequal "WARNING: The following packages cannot be authenticated! + old +E: There are problems and -y was used without --force-yes" aptget install -qq -y old + + testfailure ls rootdir/var/lib/apt/lists/*_InRelease + testsuccess ls rootdir/var/lib/apt/lists/*_Release +} + TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework @@ -179,3 +195,5 @@ test_inreleae_to_valid_release test_inreleae_to_release_reverts_all test_unauthenticated_to_invalid_inrelease + +test_inrelease_to_unauth_inrelease -- cgit v1.2.3 From 21638c3af355b3997fadd169495551568af6acfe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 31 Jul 2014 19:24:36 +0200 Subject: fail early (again) on gpg sig failures --- apt-pkg/acquire-item.cc | 45 +++++++++---------------------- apt-pkg/acquire.cc | 4 +-- test/integration/test-apt-update-rollback | 11 ++++---- 3 files changed, 20 insertions(+), 40 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2d9328b6b..da8402ffc 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -954,32 +954,16 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashStringList const &ExpectedHash, string comprExt) : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI) { - if(comprExt.empty() == true) - { - // autoselect the compression method - std::vector types = APT::Configuration::getCompressionTypes(); - for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - comprExt.append(*t).append(" "); - if (comprExt.empty() == false) - comprExt.erase(comprExt.end()-1); - } - CompressionExtension = comprExt; - - Init(URI, URIDesc, ShortDesc); -} -#if 0 -pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, - HashStringList const &ExpectedHash, - indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser), - RealURI(Target->URI) -{ - // autoselect the compression method AutoSelectCompression(); - Init(Target->URI, Target->Description, Target->ShortDesc); + Init(URI, URIDesc, ShortDesc); + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgIndex with TransactionID " + << TransactionID << std::endl; } -#endif /*}}}*/ +// AcqIndex::AcqIndex - Constructor /*{{{*/ +// --------------------------------------------------------------------- pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, unsigned long TransactionID, IndexTarget const *Target, @@ -997,6 +981,8 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, << TransactionID << std::endl; } /*}}}*/ +// AcqIndex::AutoSelectCompression - Select compression /*{{{*/ +// --------------------------------------------------------------------- void pkgAcqIndex::AutoSelectCompression() { std::vector types = APT::Configuration::getCompressionTypes(); @@ -1846,24 +1832,19 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/ // pkgAcqMetaIndex::Failed - no Release file present or no signature file present /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqMetaIndex::Failed(string /*Message*/, +void pkgAcqMetaIndex::Failed(string Message, pkgAcquire::MethodConfig * /*Cnf*/) { -#if 0 if (AuthPass == true) { // gpgv method failed, if we have a good signature - string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); + string LastGoodSigFile = _config->FindDir("Dir::State::lists"); + LastGoodSigFile += URItoFileName(RealURI); if (DestFile != SigFile) LastGoodSigFile.append(".gpg"); - LastGoodSigFile.append(".reverify"); if(FileExists(LastGoodSigFile)) { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - if (DestFile != SigFile) - VerifiedSigFile.append(".gpg"); - Rename(LastGoodSigFile, VerifiedSigFile); Status = StatTransientNetworkError; _error->Warning(_("An error occurred during the signature " "verification. The repository is not updated " @@ -1878,6 +1859,7 @@ void pkgAcqMetaIndex::Failed(string /*Message*/, _error->Error(_("GPG error: %s: %s"), Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); + Status = StatError; return; } else { _error->Warning(_("GPG error: %s: %s"), @@ -1887,7 +1869,6 @@ void pkgAcqMetaIndex::Failed(string /*Message*/, // gpgv method failed ReportMirrorFailure("GPGFailure"); } -#endif /* Always move the meta index, even if gpgv failed. This ensures * that PackageFile objects are correctly filled in */ diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 33afd8f1f..15af5d6bd 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -195,8 +195,8 @@ bool pkgAcquire::TransactionHasError(unsigned long TransactionID) std::vector Transaction; for (ItemIterator I = Items.begin(); I != Items.end(); ++I) if((*I)->TransactionID == TransactionID) - if((*I)->Status == pkgAcquire::Item::StatError || - (*I)->Status == pkgAcquire::Item::StatAuthError) + if((*I)->Status != pkgAcquire::Item::StatDone && + (*I)->Status != pkgAcquire::Item::StatIdle) return true; return false; diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index c16e4f480..a6297792e 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -156,14 +156,13 @@ test_inrelease_to_unauth_inrelease() { signreleasefiles 'Marvin Paranoid' avoid_ims_hit - testsuccess aptget update -qq + testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2 - testequal "WARNING: The following packages cannot be authenticated! - old -E: There are problems and -y was used without --force-yes" aptget install -qq -y old +W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease - testfailure ls rootdir/var/lib/apt/lists/*_InRelease - testsuccess ls rootdir/var/lib/apt/lists/*_Release +W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq + + testsuccess ls rootdir/var/lib/apt/lists/*_InRelease } TESTDIR=$(readlink -f $(dirname $0)) -- cgit v1.2.3 From 80976dd5452a9cfbe0c4f6229c729711ba685a5f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 1 Aug 2014 11:06:47 +0200 Subject: mve MetaKey into pkgAcqBaseIndex --- apt-pkg/acquire-item.cc | 1 - apt-pkg/acquire-item.h | 3 +++ test/integration/test-apt-update-rollback | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index da8402ffc..43c09e7b5 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1010,7 +1010,6 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S DestFile += URItoFileName(URI); std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - std::string MetaKey; if (comprExt == "uncompressed") { Desc.URI = URI; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3f7cca083..f5a308c58 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -532,6 +532,8 @@ class pkgAcqBaseIndex : public pkgAcquire::Item */ const struct IndexTarget * Target; indexRecords *MetaIndexParser; + /** \brief The MetaIndex Key */ + std::string MetaKey; pkgAcqBaseIndex(pkgAcquire *Owner, unsigned long TransactionID, @@ -851,6 +853,7 @@ class pkgAcqIndex : public pkgAcqBaseIndex */ bool Erase; + // FIXME: // Unused, used to be used to verify that "Packages: " header was there bool __DELME_ON_NEXT_ABI_BREAK_Verify; diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index a6297792e..d7078d217 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -44,7 +44,9 @@ test_inrelease_to_new_inrelease() { testequal "old/unstable 1.0 all" apt list -q add_new_package - testsuccess aptget update + aptget update -o Debug::Acquire::Transaction=1 + + testsuccess aptget update -o Debug::Acquire::Transaction=1 testequal "new/unstable 1.0 all old/unstable 1.0 all" apt list -q -- cgit v1.2.3 From 81273628cc3022641756b05e78256d59b7bd7c51 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 1 Aug 2014 11:46:16 +0200 Subject: fix transactionid passing --- apt-pkg/acquire-item.h | 2 +- test/integration/test-apt-update-rollback | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index f5a308c58..e191e2554 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -540,7 +540,7 @@ class pkgAcqBaseIndex : public pkgAcquire::Item struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : Item(Owner, ExpectedHashes), Target(Target), + : Item(Owner, ExpectedHashes, TransactionID), Target(Target), MetaIndexParser(MetaIndexParser) {}; }; /*}}}*/ diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index d7078d217..4eef2aecf 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -44,7 +44,6 @@ test_inrelease_to_new_inrelease() { testequal "old/unstable 1.0 all" apt list -q add_new_package - aptget update -o Debug::Acquire::Transaction=1 testsuccess aptget update -o Debug::Acquire::Transaction=1 -- cgit v1.2.3 From 7abcfdde365d2f1110b1f1189e3fce04abdac98c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 1 Aug 2014 17:13:15 +0200 Subject: check hashes of compressed files as well --- apt-pkg/acquire-item.cc | 14 ++++++++++++++ test/integration/test-apt-update-rollback | 29 +++++++++++++++++------------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 43c09e7b5..c75ef36a9 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1178,6 +1178,20 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con unlink(DestFile.c_str()); #endif return; + } else { + // FIXME: use the same method to find + // check the compressed hash too + if(MetaKey != "" && Hashes.size() > 0) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) + { + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, Record->Hashes, Hashes); + Failed(Message, Cfg); + return; + } + } } Erase = false; diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index 4eef2aecf..b8a2b0791 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -61,9 +61,7 @@ test_inrelease_to_broken_hash_reverts_all() { # test the error condition testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease -W: Failed to fetch copy:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch - -W: Failed to fetch copy:${APTARCHIVE}/dists/unstable/main/binary-i386/Packages +W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq # ensure that the Packages file is also rolled back @@ -108,9 +106,7 @@ W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release.gpg -W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch - -W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/binary-i386/Packages +W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq # -o Debug::acquire::transaction=1 @@ -139,9 +135,7 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease -W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch - -W: Failed to fetch copy:$APTARCHIVE/dists/unstable/main/binary-i386/Packages +W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq @@ -166,6 +160,19 @@ W: Some index files failed to download. They have been ignored, or old ones used testsuccess ls rootdir/var/lib/apt/lists/*_InRelease } +test_inrelease_to_broken_gzip() { + msgmsg "Test InRelease to broken gzip" + create_fresh_archive + # append junk at the end of the gzip, this + echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz + # remove uncompressed file, otherwise apt will just fallback fetching + # that + rm $APTARCHIVE/dists/unstable/main/source/Sources + avoid_ims_hit + + testfailure aptget update +} + TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework @@ -190,10 +197,8 @@ APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" test_inrelease_to_new_inrelease test_inrelease_to_broken_hash_reverts_all - test_inreleae_to_valid_release test_inreleae_to_release_reverts_all - test_unauthenticated_to_invalid_inrelease - test_inrelease_to_unauth_inrelease +test_inrelease_to_broken_gzip -- cgit v1.2.3 From 183160cb20cd4aa86e78657bf060bf688edce703 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 1 Aug 2014 17:15:53 +0200 Subject: make errors more consistent --- apt-pkg/acquire.cc | 5 +++-- test/integration/test-apt-update-rollback | 16 +++------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 15af5d6bd..be4e494e0 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -185,8 +185,9 @@ void pkgAcquire::AbortTransaction(unsigned long TransactionID) { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << " Cancel: " << (*I)->DestFile << std::endl; - //Dequeue(*I); - (*I)->Status = pkgAcquire::Item::StatError; + // the transaction will abort, so stop anything that is idle + if ((*I)->Status == pkgAcquire::Item::StatIdle) + (*I)->Status = pkgAcquire::Item::StatDone; } } /*}}}*/ diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index b8a2b0791..24027787e 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -59,9 +59,7 @@ test_inrelease_to_broken_hash_reverts_all() { break_repository_sources_index # test the error condition - testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease - -W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch + testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq # ensure that the Packages file is also rolled back @@ -100,13 +98,7 @@ test_inreleae_to_release_reverts_all() { break_repository_sources_index # ensure error - testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease - -W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release - -W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release.gpg - -W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch + testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq # -o Debug::acquire::transaction=1 @@ -133,9 +125,7 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y add_new_package break_repository_sources_index - testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease - -W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch + testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq -- cgit v1.2.3 From 67f2f9e2ed2f48833926abb7c31cca4a57ebfec1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 1 Aug 2014 17:20:19 +0200 Subject: add gzip test and todo --- test/integration/test-apt-update-rollback | 4 ++++ test/integration/test-hashsum-verification | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index 24027787e..ccd7f57ff 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -185,6 +185,10 @@ APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" # going from Release/Release.gpg -> InRelease and vice versa # - unauthenticated -> invalid InRelease +# stuff to do: +# - ims-hit +# - gzip-index tests + test_inrelease_to_new_inrelease test_inrelease_to_broken_hash_reverts_all test_inreleae_to_valid_release diff --git a/test/integration/test-hashsum-verification b/test/integration/test-hashsum-verification index e77efb46e..70bf1b476 100755 --- a/test/integration/test-hashsum-verification +++ b/test/integration/test-hashsum-verification @@ -75,5 +75,13 @@ runtest() { } -runtest - +for COMPRESSEDINDEXES in 'false' 'true'; do + echo "Acquire::GzipIndexes \"$COMPRESSEDINDEXES\";" > rootdir/etc/apt/apt.conf.d/compressindexes + if $COMPRESSEDINDEXES; then + msgmsg 'Run tests with GzipIndexes enabled' + else + msgmsg 'Run tests with GzipIndexes disabled' + fi + + runtest +done -- cgit v1.2.3 From 63d0f85391839a666957add1833e67f7638c8a83 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 1 Aug 2014 19:25:00 +0200 Subject: make i-m-s work again --- apt-pkg/acquire-item.cc | 11 +++++++---- test/integration/test-apt-update-ims | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) create mode 100755 test/integration/test-apt-update-ims diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c75ef36a9..a6f69944e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1164,10 +1164,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // Done, queue for rename on transaction finished PartialFile = DestFile; - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - DestFile = FinalFile; -#if 0 +#if 1 // FIXME: waaaay too complicated /* We restore the original name to DestFile so that the clean operation will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -1177,6 +1174,12 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con if (Erase == true) unlink(DestFile.c_str()); #endif + + // Done, queue for rename on transaction finished + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + DestFile = FinalFile; + return; } else { // FIXME: use the same method to find diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims new file mode 100755 index 000000000..cf2b28bb5 --- /dev/null +++ b/test/integration/test-apt-update-ims @@ -0,0 +1,22 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' + +buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable' + +setupaptarchive +changetowebserver + +testsuccess aptget update + +# check that I-M-S header is kept in redirections +testequal "Hit http://localhost:8080 unstable InRelease +Hit http://localhost:8080 unstable/main Sources +Hit http://localhost:8080 unstable/main amd64 Packages +Hit http://localhost:8080 unstable/main Translation-en +Reading package lists..." aptget update + -- cgit v1.2.3 From e84d3803ed3bdd55e20e3720b375769330966fa0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 2 Aug 2014 05:09:13 +0200 Subject: move clearsign check into pkgAcqMetaClearSig::Failed() --- apt-pkg/acquire-item.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index a6f69944e..bf6866f17 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1551,18 +1551,6 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList } else { - // FIXME: move this into pkgAcqMetaClearSig::Done on the next - // ABI break - - // if we expect a ClearTextSignature (InRelase), ensure that - // this is what we get and if not fail to queue a - // Release/Release.gpg, see #346386 - if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile)) - { - Failed(Message, Cfg); - return; - } - // There was a signature file, so pass it to gpgv for // verification if (_config->FindB("Debug::pkgAcquire::Auth", false)) @@ -1985,6 +1973,17 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* // we failed, we will not get additional items from this method ExpectedAdditionalItems = 0; + // if we expect a ClearTextSignature (InRelase), ensure that + // this is what we get and if not fail to queue a + // Release/Release.gpg, see #346386 + if (!StartsWithGPGClearTextSignature(DestFile)) + { + //_error->Error(_("Does not start with a clear sign signature")); + pkgAcquire::Item::Failed(Message, Cnf); + return; + } + + if (AuthPass == false) { // Queue the 'old' InRelease file for removal if we try Release.gpg -- cgit v1.2.3 From 63b7249e6930c1bcb69bac32f10108119eeacc2a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 2 Aug 2014 05:37:43 +0200 Subject: add ims check verify --- apt-pkg/acquire-item.cc | 50 +++++++++++++++++++++++++++++------- apt-pkg/acquire-item.h | 8 ++++++ test/integration/test-apt-update-ims | 3 +++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index bf6866f17..e98eaa445 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1094,6 +1094,9 @@ string pkgAcqIndex::Custom600Headers() const return msg; } /*}}}*/ +// pkgAcqIndex::Failed - getting the indexfile failed /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { size_t const nextExt = CompressionExtension.find(' '); @@ -1117,6 +1120,35 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ Owner->AbortTransaction(TransactionID); } /*}}}*/ +// pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ +// --------------------------------------------------------------------- +/* */ +std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, + std::string const &compExt) +{ + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(URI); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + FinalFile += ".gz"; + return FinalFile; +} + /*}}}*/ +// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) +{ + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + DestFile += ".gz"; + + // copy FinalFile into partial/ so that we check the hash again + string FinalFile = GetFinalFilename(RealURI, compExt); + Decompression = true; + Desc.URI = "copy:" + FinalFile; + QueueURI(Desc); +} + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -1128,6 +1160,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hashes,Cfg); + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (Decompression == true) { @@ -1176,9 +1210,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con #endif // Done, queue for rename on transaction finished - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - DestFile = FinalFile; + DestFile = GetFinalFilename(RealURI, compExt); return; } else { @@ -1206,7 +1238,10 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con { // The files timestamp matches if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) + { + ReverifyAfterIMS(FileName); return; + } Decompression = true; Local = true; DestFile += ".decomp"; @@ -1223,13 +1258,10 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con ErrorText = "Method gave a blank filename"; } - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - // The files timestamp matches - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile += ".gz"; + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + ReverifyAfterIMS(FileName); return; } diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index e191e2554..31279f7df 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -873,6 +873,14 @@ class pkgAcqIndex : public pkgAcqBaseIndex /** \brief Auto select the right compression to use */ void AutoSelectCompression(); + /** \brief Get the full pathname of the final file for the given URI + */ + std::string GetFinalFilename(std::string const &URI, + std::string const &compExt); + + /** \brief Schedule file for verification after a IMS hit */ + void ReverifyAfterIMS(std::string const &FileName); + public: // Specialized action members diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index cf2b28bb5..3bd6e843c 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -20,3 +20,6 @@ Hit http://localhost:8080 unstable/main amd64 Packages Hit http://localhost:8080 unstable/main Translation-en Reading package lists..." aptget update +# ensure that we still do a hash check on ims hit +msgtest 'Test I-M-S reverify' +aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail -- cgit v1.2.3 From a9bb651a04d1eae42164a2fb2cdf92bf4392a532 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 2 Aug 2014 05:44:50 +0200 Subject: really move clearsign check into pkgAcqMetaClearSig::Done() --- apt-pkg/acquire-item.cc | 19 +++++++++++++------ apt-pkg/acquire-item.h | 4 +++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index e98eaa445..7690ebb2e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -2000,21 +2000,28 @@ string pkgAcqMetaClearSig::Custom600Headers() const return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +// pkgAcqMetaClearSig::Done - We got a file /*{{{*/ +// --------------------------------------------------------------------- +void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cnf) { - // we failed, we will not get additional items from this method - ExpectedAdditionalItems = 0; - // if we expect a ClearTextSignature (InRelase), ensure that // this is what we get and if not fail to queue a // Release/Release.gpg, see #346386 - if (!StartsWithGPGClearTextSignature(DestFile)) + if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) { //_error->Error(_("Does not start with a clear sign signature")); pkgAcquire::Item::Failed(Message, Cnf); return; } - + pkgAcqMetaIndex::Done(Message, Size, Hashes, Cnf); +} + /*}}}*/ +void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +{ + // we failed, we will not get additional items from this method + ExpectedAdditionalItems = 0; if (AuthPass == false) { diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 31279f7df..28577e9b8 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -507,8 +507,10 @@ class pkgAcqMetaClearSig : public pkgAcqMetaIndex std::string MetaSigShortDesc; public: - void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); virtual std::string Custom600Headers() const; + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cnf); /** \brief Create a new pkgAcqMetaClearSig. */ pkgAcqMetaClearSig(pkgAcquire *Owner, -- cgit v1.2.3 From b6f0063caa281bfae8e6fe3d643769ea5ae19f40 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 24 Aug 2014 15:26:12 -0700 Subject: cleanup --- apt-pkg/acquire-item.cc | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7690ebb2e..30953cc7d 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1155,7 +1155,7 @@ void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) method could possibly return an alternate filename which points to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file - is decompressed with a gzip uri. */ + is decompressed with a compressed uri. */ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { @@ -1180,8 +1180,9 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con * have a Package field) (LP: #346386) (Closes: #627642) */ FileFd fd(DestFile, FileFd::ReadOnly); - // Only test for correctness if the file is not empty (empty is ok) - if (fd.FileSize() > 0) + // Only test for correctness if the content of the file is not empty + // (empty is ok) + if (fd.Size() > 0) { pkgTagSection sec; pkgTagFile tag(&fd); @@ -1195,21 +1196,15 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con } } - // Done, queue for rename on transaction finished - PartialFile = DestFile; - -#if 1 // FIXME: waaaay too complicated - /* We restore the original name to DestFile so that the clean operation - will work OK */ - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); - + // FIXME: can we void the "Erase" bool here as its very non-local? + std::string CompressedFile = _config->FindDir("Dir::State::lists") + "partial/"; + CompressedFile += URItoFileName(RealURI); // Remove the compressed version. if (Erase == true) - unlink(DestFile.c_str()); -#endif + unlink(CompressedFile.c_str()); // Done, queue for rename on transaction finished + PartialFile = DestFile; DestFile = GetFinalFilename(RealURI, compExt); return; -- cgit v1.2.3 From 5f6c6c6e085bcba021a29d88859e70b1e03ff153 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 24 Aug 2014 16:00:36 -0700 Subject: make compressed-indexes test pass again --- apt-pkg/acquire-item.cc | 38 ++++++++++++++++---------------------- methods/copy.cc | 32 +++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 30953cc7d..d5cce8c49 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1162,7 +1162,6 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con Item::Done(Message,Size,Hashes,Cfg); std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (Decompression == true) { if (ExpectedHashes.usable() && ExpectedHashes != Hashes) @@ -1179,7 +1178,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnly); + FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); // Only test for correctness if the content of the file is not empty // (empty is ok) if (fd.Size() > 0) @@ -1208,19 +1207,19 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con DestFile = GetFinalFilename(RealURI, compExt); return; - } else { - // FIXME: use the same method to find - // check the compressed hash too - if(MetaKey != "" && Hashes.size() > 0) + } + + // FIXME: use the same method to find + // check the compressed hash too + if(MetaKey != "" && Hashes.size() > 0) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) { - indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); - if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) - { - RenameOnError(HashSumMismatch); - printHashSumComparision(RealURI, Record->Hashes, Hashes); - Failed(Message, Cfg); - return; - } + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, Record->Hashes, Hashes); + Failed(Message, Cfg); + return; } } @@ -1269,14 +1268,9 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // If we enable compressed indexes and already have gzip, keep it if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI) + ".gz"; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI) + ".gz"; + // Done, queue for rename on transaction finished + PartialFile = DestFile; + DestFile = GetFinalFilename(RealURI, compExt); return; } diff --git a/methods/copy.cc b/methods/copy.cc index d59f032ff..8c797ff1f 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -27,12 +28,28 @@ class CopyMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + void CalculateHashes(FetchResult &Res); public: - CopyMethod() : pkgAcqMethod("1.0",SingleInstance) {}; + CopyMethod() : pkgAcqMethod("1.0",SingleInstance|SendConfig) {}; }; +void CopyMethod::CalculateHashes(FetchResult &Res) +{ + // For gzip indexes we need to look inside the gzip for the hash + // We can not use the extension here as its not used in partial + // on a IMS hit + FileFd::OpenMode OpenMode = FileFd::ReadOnly; + if (_config->FindB("Acquire::GzipIndexes", false) == true) + OpenMode = FileFd::ReadOnlyGzip; + + Hashes Hash; + FileFd Fd(Res.Filename, OpenMode); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); +} + // CopyMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -53,6 +70,14 @@ bool CopyMethod::Fetch(FetchItem *Itm) Res.LastModified = Buf.st_mtime; Res.IMSHit = false; URIStart(Res); + + // when the files are identical, just compute the hashes + if(File == Itm->DestFile) + { + CalculateHashes(Res); + URIDone(Res); + return true; + } // See if the file exists FileFd From(File,FileFd::ReadOnly); @@ -82,10 +107,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) if (utimes(Res.Filename.c_str(), times) != 0) return _error->Errno("utimes",_("Failed to set modification time")); - Hashes Hash; - FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd); - Res.TakeHashes(Hash); + CalculateHashes(Res); URIDone(Res); return true; -- cgit v1.2.3 From d4ab7e9c6ee81f4d81e4c3c54395fe1af4d1e7b4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 24 Aug 2014 16:16:49 -0700 Subject: make the test-pdiff-usage code work again --- apt-pkg/acquire-item.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d5cce8c49..fcfdd50ea 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -676,12 +676,21 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) // Finish - helper that cleans the item out of the fetcher queue /*{{{*/ void pkgAcqIndexDiffs::Finish(bool allDone) { + if(Debug) + std::clog << "pkgAcqIndexDiffs::Finish(): " + << allDone << " " + << Desc.URI << std::endl; + // we restore the original name, this is required, otherwise // the file will be cleaned if(allDone) { DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); + + // FIXME: we want the rred stuff to use the real transactional update + // this is just a workaround + PartialFile = DestFile; if(HashSums().usable() && !HashSums().VerifyFile(DestFile)) { @@ -929,6 +938,8 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri // otherwise lists cleanup will eat the file DestFile = FinalFile; + // FIXME: make the merged rred code really transactional + PartialFile = FinalFile; // ensure the ed's are gone regardless of list-cleanup for (std::vector::const_iterator I = allPatches->begin(); -- cgit v1.2.3 From 09475bebba5554481a7cb05995ded92cf30063fa Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 24 Aug 2014 21:30:44 -0700 Subject: all tests pass --- apt-pkg/acquire-item.cc | 42 +++++++++++++++++----- .../test-ubuntu-bug-346386-apt-get-update-paywall | 10 +++--- ...st-ubuntu-bug-784473-InRelease-one-message-only | 4 +-- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index fcfdd50ea..c3b6f0e6a 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1487,6 +1487,36 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ DestFile += URItoFileName(RealURI); PartialFile = ""; + // FIXME: duplicated code from pkgAcqMetaIndex + if (AuthPass == true) + { + if(FileExists(Final)) + { + Status = StatTransientNetworkError; + _error->Warning(_("An error occurred during the signature " + "verification. The repository is not updated " + "and the previous index files will be used. " + "GPG error: %s: %s\n"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + RunScripts("APT::Update::Auth-Failure"); + return; + } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { + /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ + _error->Error(_("GPG error: %s: %s"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + Status = StatError; + return; + } else { + _error->Warning(_("GPG error: %s: %s"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + } + // gpgv method failed + ReportMirrorFailure("GPGFailure"); + } + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) @@ -1871,15 +1901,11 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/ void pkgAcqMetaIndex::Failed(string Message, pkgAcquire::MethodConfig * /*Cnf*/) { + string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + if (AuthPass == true) { - // gpgv method failed, if we have a good signature - string LastGoodSigFile = _config->FindDir("Dir::State::lists"); - LastGoodSigFile += URItoFileName(RealURI); - if (DestFile != SigFile) - LastGoodSigFile.append(".gpg"); - - if(FileExists(LastGoodSigFile)) + if(FileExists(Final)) { Status = StatTransientNetworkError; _error->Warning(_("An error occurred during the signature " @@ -2011,8 +2037,8 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, // Release/Release.gpg, see #346386 if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) { - //_error->Error(_("Does not start with a clear sign signature")); pkgAcquire::Item::Failed(Message, Cnf); + ErrorText = _("Does not start with a cleartext signature"); return; } pkgAcqMetaIndex::Done(Message, Size, Hashes, Cnf); diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall index 8e50843f3..388c2bfdb 100755 --- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall +++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall @@ -37,8 +37,8 @@ ensure_n_canary_strings_in_dir() { LISTS='rootdir/var/lib/apt/lists' rm -rf rootdir/var/lib/apt/lists -msgtest 'Got expected NODATA failure in' 'apt-get update' -aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail +msgtest 'Got expected failure message' 'apt-get update' +aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0 testequal 'partial' ls $LISTS @@ -48,8 +48,8 @@ for f in Release Release.gpg main_binary-amd64_Packages main_source_Sources; do echo 'peng neee-wom' > $LISTS/localhost:8080_dists_stable_${f} done -msgtest 'Got expected NODATA failure in' 'apt-get update' -aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail +msgtest 'Got expected failure message in' 'apt-get update' +aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail ensure_n_canary_strings_in_dir $LISTS 'peng neee-wom' 4 ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0 @@ -58,7 +58,7 @@ ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0 echo 'peng neee-wom' > $LISTS/localhost:8080_dists_stable_InRelease rm -f $LISTS/localhost:8080_dists_stable_Release $LISTS/localhost:8080_dists_stable_Release.gpg msgtest 'excpected failure of' 'apt-get update' -aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail +aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail ensure_n_canary_strings_in_dir $LISTS 'peng neee-wom' 3 ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0 diff --git a/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only b/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only index 50ca2bf57..09315868b 100755 --- a/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only +++ b/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only @@ -28,12 +28,10 @@ MD5Sum: done msgtest 'The unsigned garbage before signed block is' 'ignored' -testsuccess --nomsg aptget update +aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail ROOTDIR="$(readlink -f .)" testequal "Package files: 100 ${ROOTDIR}/rootdir/var/lib/dpkg/status release a=now - 500 file:${ROOTDIR}/aptarchive/ unstable/main i386 Packages - release a=unstable,n=sid,c=main Pinned packages:" aptcache policy -- cgit v1.2.3 From e6e893903869635ab7ee3200f654129b08717ded Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 25 Aug 2014 20:58:02 -0700 Subject: add shared code into pkgAcqMetaSigBase::GenerateAuthWarning() --- apt-pkg/acquire-item.cc | 103 ++++++++++++++++++++++++------------------------ apt-pkg/acquire-item.h | 16 +++++++- 2 files changed, 65 insertions(+), 54 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c3b6f0e6a..0ec151050 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1367,13 +1367,56 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ + +pkgAcqMetaSigBase::pkgAcqMetaSigBase(pkgAcquire *Owner, + HashStringList const &ExpectedHashes, + unsigned long TransactionID) + : Item(Owner, ExpectedHashes, TransactionID) +{ +} + /*{{{*/ +bool pkgAcqMetaSigBase::GenerateAuthWarning(const std::string &RealURI, + const std::string &Message) +{ + string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + + if(FileExists(Final)) + { + Status = StatTransientNetworkError; + _error->Warning(_("An error occurred during the signature " + "verification. The repository is not updated " + "and the previous index files will be used. " + "GPG error: %s: %s\n"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + RunScripts("APT::Update::Auth-Failure"); + return true; + } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { + /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ + _error->Error(_("GPG error: %s: %s"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + Status = StatError; + return true; + } else { + _error->Warning(_("GPG error: %s: %s"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + } + // gpgv method failed + ReportMirrorFailure("GPGFailure"); + return false; +} + /*}}}*/ + + pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ unsigned long TransactionID, string URI,string URIDesc,string ShortDesc, string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList(), TransactionID), RealURI(URI), + pkgAcqMetaSigBase(Owner, HashStringList(), TransactionID), RealURI(URI), MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { @@ -1490,31 +1533,9 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // FIXME: duplicated code from pkgAcqMetaIndex if (AuthPass == true) { - if(FileExists(Final)) - { - Status = StatTransientNetworkError; - _error->Warning(_("An error occurred during the signature " - "verification. The repository is not updated " - "and the previous index files will be used. " - "GPG error: %s: %s\n"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - RunScripts("APT::Update::Auth-Failure"); - return; - } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { - /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ - _error->Error(_("GPG error: %s: %s"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - Status = StatError; - return; - } else { - _error->Warning(_("GPG error: %s: %s"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - } - // gpgv method failed - ReportMirrorFailure("GPGFailure"); + bool Stop = GenerateAuthWarning(RealURI, Message); + if(Stop) + return; } // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor @@ -1536,7 +1557,7 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets), + pkgAcqMetaSigBase(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets), MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) @@ -1905,31 +1926,9 @@ void pkgAcqMetaIndex::Failed(string Message, if (AuthPass == true) { - if(FileExists(Final)) - { - Status = StatTransientNetworkError; - _error->Warning(_("An error occurred during the signature " - "verification. The repository is not updated " - "and the previous index files will be used. " - "GPG error: %s: %s\n"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - RunScripts("APT::Update::Auth-Failure"); - return; - } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { - /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ - _error->Error(_("GPG error: %s: %s"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - Status = StatError; - return; - } else { - _error->Warning(_("GPG error: %s: %s"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - } - // gpgv method failed - ReportMirrorFailure("GPGFailure"); + bool Stop = GenerateAuthWarning(RealURI, Message); + if(Stop) + return; } /* Always move the meta index, even if gpgv failed. This ensures diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 28577e9b8..ae93ea311 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -378,6 +378,18 @@ class pkgAcqSubIndex : public pkgAcquire::Item }; /*}}}*/ +class pkgAcqMetaSigBase : public pkgAcquire::Item +{ + protected: + bool GenerateAuthWarning(const std::string &RealURI, + const std::string &Message); + + public: + pkgAcqMetaSigBase(pkgAcquire *Owner, + HashStringList const &ExpectedHashes=HashStringList(), + unsigned long TransactionID=0); +}; + /** \brief An item that is responsible for downloading the meta-index {{{ * file (i.e., Release) itself and verifying its signature. * @@ -388,7 +400,7 @@ class pkgAcqSubIndex : public pkgAcquire::Item * otherwise, the expected hashsums will be "" (causing the * authentication of the index files to be bypassed). */ -class pkgAcqMetaIndex : public pkgAcquire::Item +class pkgAcqMetaIndex : public pkgAcqMetaSigBase { protected: /** \brief The URI that is actually being downloaded; never @@ -1017,7 +1029,7 @@ class OptionalSubIndexTarget : public OptionalIndexTarget * * \sa pkgAcqMetaIndex */ -class pkgAcqMetaSig : public pkgAcquire::Item +class pkgAcqMetaSig : public pkgAcqMetaSigBase { protected: /** \brief The last good signature file */ -- cgit v1.2.3 From f456b60b7804c95810ccccdb87a76c4bc7f746d7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 16 Sep 2014 17:03:04 +0200 Subject: remove pkgAcqSubIndex --- apt-pkg/acquire-item.cc | 120 +----------------------------------------------- apt-pkg/acquire-item.h | 57 ----------------------- 2 files changed, 1 insertion(+), 176 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index eee1097e9..d005dce4c 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -237,120 +237,6 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) } } /*}}}*/ -// AcqSubIndex::AcqSubIndex - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* Get a sub-index file based on checksums from a 'master' file and - possibly query additional files */ -pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, - unsigned long TransactionID, - string const &URI, - string const &URIDesc, string const &ShortDesc, - HashStringList const &ExpectedHashes) - : Item(Owner, ExpectedHashes, TransactionID) -{ - /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ - Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); - - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); - - Desc.URI = URI; - Desc.Description = URIDesc; - Desc.Owner = this; - Desc.ShortDesc = ShortDesc; - - QueueURI(Desc); - - if(Debug) - std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl; -} - /*}}}*/ -// AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ -// --------------------------------------------------------------------- -/* The only header we use is the last-modified header. */ -string pkgAcqSubIndex::Custom600Headers() const -{ - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(Desc.URI); - - struct stat Buf; - if (stat(Final.c_str(),&Buf) != 0) - return "\nIndex-File: true\nFail-Ignore: true\n"; - return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); -} - /*}}}*/ -void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ -{ - if(Debug) - std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl; - - Complete = false; - Status = StatDone; - Dequeue(); - - // No good Index is provided -} - /*}}}*/ -void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ - pkgAcquire::MethodConfig *Cnf) -{ - if(Debug) - std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl; - - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) - { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } - - if (FileName != DestFile) - { - Local = true; - Desc.URI = "copy:" + FileName; - QueueURI(Desc); - return; - } - - Item::Done(Message, Size, Hashes, Cnf); - - string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); - - /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */ - indexRecords SubIndexParser; - if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) { - Status = StatError; - ErrorText = SubIndexParser.ErrorText; - return; - } - - // success in downloading the index - // rename the index - if(Debug) - std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - DestFile = FinalFile; - - if(ParseIndex(DestFile) == false) - return Failed("", NULL); - - Complete = true; - Status = StatDone; - Dequeue(); - return; -} - /*}}}*/ -bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ -{ - indexRecords SubIndexParser; - if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false) - return false; - // so something with the downloaded index - return true; -} - /*}}}*/ // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Get the DiffIndex file first and see if there are patches available @@ -1812,11 +1698,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ if ((*Target)->IsOptional() == true) { - if ((*Target)->IsSubIndex() == true) - new pkgAcqSubIndex(Owner, TransactionID, - (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHashes); - else if (transInRelease == false || Record != NULL || compressedAvailable == true) + if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index ae93ea311..90eccdd16 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -338,44 +338,6 @@ struct DiffInfo { unsigned long size; }; /*}}}*/ -/** \brief An item that is responsible for fetching a SubIndex {{{ - * - * The MetaIndex file includes only records for important indexes - * and records for these SubIndex files so these can carry records - * for addition files like PDiffs and Translations - */ -class pkgAcqSubIndex : public pkgAcquire::Item -{ - protected: - /** \brief If \b true, debugging information will be written to std::clog. */ - bool Debug; - - public: - // Specialized action members - virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, - pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() const {return Desc.URI;}; - virtual std::string Custom600Headers() const; - virtual bool ParseIndex(std::string const &IndexFile); - - /** \brief Create a new pkgAcqSubIndex. - * - * \param Owner The Acquire object that owns this item. - * - * \param URI The URI of the list file to download. - * - * \param URIDesc A long description of the list file to download. - * - * \param ShortDesc A short description of the list file to download. - * - * \param ExpectedHashes The list file's hashsums which are expected. - */ - pkgAcqSubIndex(pkgAcquire *Owner, - unsigned long TransactionID, - std::string const &URI,std::string const &URIDesc, - std::string const &ShortDesc, HashStringList const &ExpectedHashes); -}; /*}}}*/ class pkgAcqMetaSigBase : public pkgAcquire::Item @@ -991,9 +953,6 @@ class IndexTarget virtual bool IsOptional() const { return false; } - virtual bool IsSubIndex() const { - return false; - } }; /*}}}*/ /** \brief Information about an optional index file. */ /*{{{*/ @@ -1004,22 +963,6 @@ class OptionalIndexTarget : public IndexTarget } }; /*}}}*/ -/** \brief Information about an subindex index file. */ /*{{{*/ -class SubIndexTarget : public IndexTarget -{ - virtual bool IsSubIndex() const { - return true; - } -}; - /*}}}*/ -/** \brief Information about an subindex index file. */ /*{{{*/ -class OptionalSubIndexTarget : public OptionalIndexTarget -{ - virtual bool IsSubIndex() const { - return true; - } -}; - /*}}}*/ /** \brief An acquire item that downloads the detached signature {{{ * of a meta-index (Release) file, then queues up the release -- cgit v1.2.3 From 60323ed7fde01e8c013616e2d428bf9718f2fbc7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 16 Sep 2014 17:13:09 +0200 Subject: add a bunch of dpointers --- apt-pkg/acquire-item.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 90eccdd16..c6fd05d6d 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -63,6 +63,8 @@ class pkgAcqMetaIndex; */ class pkgAcquire::Item : public WeakPointable { + void *d; + protected: /** \brief The acquire object with which this item is associated. */ @@ -342,6 +344,8 @@ struct DiffInfo { class pkgAcqMetaSigBase : public pkgAcquire::Item { + void *d; + protected: bool GenerateAuthWarning(const std::string &RealURI, const std::string &Message); @@ -364,6 +368,8 @@ class pkgAcqMetaSigBase : public pkgAcquire::Item */ class pkgAcqMetaIndex : public pkgAcqMetaSigBase { + void *d; + protected: /** \brief The URI that is actually being downloaded; never * modified by pkgAcqMetaIndex. @@ -462,6 +468,8 @@ class pkgAcqMetaIndex : public pkgAcqMetaSigBase /** \brief An item repsonsible for downloading clearsigned metaindexes {{{*/ class pkgAcqMetaClearSig : public pkgAcqMetaIndex { + void *d; + /** \brief The URI of the meta-index file for the detached signature */ std::string MetaIndexURI; @@ -503,6 +511,8 @@ public: */ class pkgAcqBaseIndex : public pkgAcquire::Item { + void *d; + protected: /** \brief Pointer to the IndexTarget data */ @@ -531,6 +541,8 @@ class pkgAcqBaseIndex : public pkgAcquire::Item */ class pkgAcqDiffIndex : public pkgAcqBaseIndex { + void *d; + protected: /** \brief If \b true, debugging information will be written to std::clog. */ bool Debug; @@ -603,6 +615,8 @@ class pkgAcqDiffIndex : public pkgAcqBaseIndex */ class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex { + void *d; + protected: /** \brief If \b true, debugging output will be written to @@ -694,6 +708,8 @@ class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex */ class pkgAcqIndexDiffs : public pkgAcqBaseIndex { + void *d; + private: /** \brief Queue up the next diff download. @@ -819,6 +835,8 @@ class pkgAcqIndexDiffs : public pkgAcqBaseIndex */ class pkgAcqIndex : public pkgAcqBaseIndex { + void *d; + protected: /** \brief If \b true, the index file has been decompressed. */ @@ -906,6 +924,8 @@ class pkgAcqIndex : public pkgAcqBaseIndex */ class pkgAcqIndexTrans : public pkgAcqIndex { + void *d; + public: virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); @@ -935,6 +955,8 @@ class pkgAcqIndexTrans : public pkgAcqIndex /** \brief Information about an index file. */ /*{{{*/ class IndexTarget { + void *d; + public: /** \brief A URI from which the index file can be downloaded. */ std::string URI; @@ -958,6 +980,8 @@ class IndexTarget /** \brief Information about an optional index file. */ /*{{{*/ class OptionalIndexTarget : public IndexTarget { + void *d; + virtual bool IsOptional() const { return true; } @@ -974,6 +998,8 @@ class OptionalIndexTarget : public IndexTarget */ class pkgAcqMetaSig : public pkgAcqMetaSigBase { + void *d; + protected: /** \brief The last good signature file */ std::string LastGoodSig; @@ -1029,6 +1055,8 @@ class pkgAcqMetaSig : public pkgAcqMetaSigBase */ class pkgAcqArchive : public pkgAcquire::Item { + void *d; + protected: /** \brief The package version being fetched. */ pkgCache::VerIterator Version; @@ -1107,6 +1135,8 @@ class pkgAcqArchive : public pkgAcquire::Item */ class pkgAcqFile : public pkgAcquire::Item { + void *d; + /** \brief How many times to retry the download, set from * Acquire::Retries. */ -- cgit v1.2.3 From 715c65de1f132aff9f040f0640e985018e4b564e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Sep 2014 09:17:49 +0200 Subject: use pkgAcqMetaBase as the transactionManager --- apt-pkg/acquire-item.cc | 163 ++++++++++++++++++++++++++++++-------------- apt-pkg/acquire-item.h | 52 ++++++++------ apt-pkg/acquire.cc | 69 ------------------- apt-pkg/acquire.h | 5 -- apt-pkg/deb/debmetaindex.cc | 5 +- 5 files changed, 146 insertions(+), 148 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d005dce4c..d36c47a49 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -66,13 +66,15 @@ static void printHashSumComparision(std::string const &URI, HashStringList const // Acquire::Item::Item - Constructor /*{{{*/ pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes, - unsigned long TransactionID) + pkgAcqMetaBase *TransactionManager) : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0), TransactionID(TransactionID), + Local(false), QueueCounter(0), TransactionManager(TransactionManager), ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes) { Owner->Add(this); Status = StatIdle; + if(TransactionManager != NULL) + TransactionManager->Add(this); } /*}}}*/ // Acquire::Item::~Item - Destructor /*{{{*/ @@ -245,11 +247,11 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser) { @@ -348,7 +350,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ std::clog << "Package file is up-to-date" << std::endl; // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(Owner, TransactionID, Target, + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return true; @@ -436,7 +438,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (pdiff_merge == false) { - new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes, + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); } @@ -445,7 +447,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, - TransactionID, + TransactionManager, Target, ExpectedHashes, MetaIndexParser, @@ -474,7 +476,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; @@ -517,13 +519,13 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { @@ -555,7 +557,7 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@ -704,7 +706,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, TransactionID, Target, + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return Finish(); @@ -715,13 +717,13 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -764,7 +766,7 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*C // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -855,18 +857,18 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, Init(URI, URIDesc, ShortDesc); if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgIndex with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgIndex with TransactionManager " + << TransactionManager << std::endl; } /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const *Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHash, + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, MetaIndexParser), RealURI(Target->URI) { // autoselect the compression method @@ -874,8 +876,8 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, Init(Target->URI, Target->Description, Target->ShortDesc); if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgIndex with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgIndex with TransactionManager " + << TransactionManager << std::endl; } /*}}}*/ // AcqIndex::AutoSelectCompression - Select compression /*{{{*/ @@ -1014,7 +1016,7 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ Item::Failed(Message,Cnf); /// cancel the entire transaction - Owner->AbortTransaction(TransactionID); + TransactionManager->AbortTransaction(); } /*}}}*/ // pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ @@ -1200,11 +1202,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, } /*}}}*/ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser) + : pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser) { // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); @@ -1254,15 +1256,70 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } /*}}}*/ -pkgAcqMetaSigBase::pkgAcqMetaSigBase(pkgAcquire *Owner, - HashStringList const &ExpectedHashes, - unsigned long TransactionID) - : Item(Owner, ExpectedHashes, TransactionID) +void pkgAcqMetaBase::Add(Item *I) { + Transaction.push_back(I); } + +void pkgAcqMetaBase::AbortTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "AbortTransaction: " << TransactionManager << std::endl; + + for (std::vector::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << " Cancel: " << (*I)->DestFile << std::endl; + // the transaction will abort, so stop anything that is idle + if ((*I)->Status == pkgAcquire::Item::StatIdle) + (*I)->Status = pkgAcquire::Item::StatDone; + } +} + /*}}}*/ +bool pkgAcqMetaBase::TransactionHasError() +{ + for (pkgAcquire::ItemIterator I = Transaction.begin(); + I != Transaction.end(); ++I) + if((*I)->Status != pkgAcquire::Item::StatDone && + (*I)->Status != pkgAcquire::Item::StatIdle) + return true; + + return false; +} +// Acquire::CommitTransaction - Commit a transaction /*{{{*/ +void pkgAcqMetaBase::CommitTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "CommitTransaction: " << this << std::endl; + + // move new files into place *and* remove files that are not + // part of the transaction but are still on disk + for (std::vector::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + if((*I)->PartialFile != "") + { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "mv " + << (*I)->PartialFile << " -> " + << (*I)->DestFile << std::endl; + Rename((*I)->PartialFile, (*I)->DestFile); + chmod((*I)->DestFile.c_str(),0644); + } else { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "rm " + << (*I)->DestFile << std::endl; + unlink((*I)->DestFile.c_str()); + } + // mark that this transaction is finished + (*I)->TransactionManager = 0; + } +} + /*{{{*/ -bool pkgAcqMetaSigBase::GenerateAuthWarning(const std::string &RealURI, - const std::string &Message) +bool pkgAcqMetaBase::GenerateAuthWarning(const std::string &RealURI, + const std::string &Message) { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@ -1297,12 +1354,12 @@ bool pkgAcqMetaSigBase::GenerateAuthWarning(const std::string &RealURI, pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, string URI,string URIDesc,string ShortDesc, string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaSigBase(Owner, HashStringList(), TransactionID), RealURI(URI), + pkgAcqMetaBase(Owner, HashStringList(), TransactionManager), RealURI(URI), MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { @@ -1314,10 +1371,10 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ // partial download anyway unlink(DestFile.c_str()); - // set the TransactionID + // set the TransactionManager if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgAcqMetaSig with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgAcqMetaSig with TransactionManager " + << TransactionManager << std::endl; // Create the item Desc.Description = URIDesc; @@ -1438,22 +1495,26 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, string URI,string URIDesc,string ShortDesc, string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaSigBase(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets), + pkgAcqMetaBase(Owner, HashStringList(), TransactionManager), RealURI(URI), IndexTargets(IndexTargets), MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) { - if(TransactionID == 0) - this->TransactionID = (unsigned long)this; + if(TransactionManager == NULL) + { + this->TransactionManager = this; + this->TransactionManager->Add(this); + } if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgAcqMetaIndex with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgAcqMetaIndex with TransactionManager " + << this->TransactionManager << std::endl; + Init(URIDesc, ShortDesc); } @@ -1588,7 +1649,7 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ // queue a signature if(SigFile != DestFile) - new pkgAcqMetaSig(Owner, TransactionID, + new pkgAcqMetaSig(Owner, TransactionManager, MetaIndexSigURI, MetaIndexSigURIDesc, MetaIndexSigShortDesc, DestFile, IndexTargets, MetaIndexParser); @@ -1702,9 +1763,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndexTrans(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@ -1715,9 +1776,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@ -1843,9 +1904,9 @@ void pkgAcqMetaIndex::Finished() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "Finished: " << DestFile <TransactionHasError(TransactionID) == false && - TransactionID > 0) - Owner->CommitTransaction(TransactionID); + if(TransactionManager != NULL && + TransactionManager->TransactionHasError() == false) + TransactionManager->CommitTransaction(); } @@ -1855,7 +1916,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaIndex(Owner, (unsigned long)this, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), + pkgAcqMetaIndex(Owner, NULL, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { @@ -1940,7 +2001,7 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* PartialFile = ""; DestFile = FinalFile; - new pkgAcqMetaIndex(Owner, TransactionID, + new pkgAcqMetaIndex(Owner, TransactionManager, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index c6fd05d6d..622324347 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -47,7 +47,7 @@ class indexRecords; class pkgRecords; class pkgSourceList; class IndexTarget; -class pkgAcqMetaIndex; +class pkgAcqMetaBase; /** \brief Represents the process by which a pkgAcquire object should {{{ * retrieve a file or a collection of files. @@ -171,8 +171,8 @@ class pkgAcquire::Item : public WeakPointable */ unsigned int QueueCounter; - /** \brief TransactionID */ - unsigned long TransactionID; + /** \brief TransactionManager */ + pkgAcqMetaBase *TransactionManager; /** \brief The number of additional fetch items that are expected * once this item is done. @@ -300,7 +300,7 @@ class pkgAcquire::Item : public WeakPointable */ Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes=HashStringList(), - unsigned long TransactionID=0); + pkgAcqMetaBase *TransactionManager=NULL); /** \brief Remove this item from its owner's queue by invoking * pkgAcquire::Remove. @@ -342,20 +342,32 @@ struct DiffInfo { /*}}}*/ /*}}}*/ -class pkgAcqMetaSigBase : public pkgAcquire::Item +class pkgAcqMetaBase : public pkgAcquire::Item { void *d; protected: + std::vector Transaction; + + public: + // transaction code + void Add(Item *I); + void AbortTransaction(); + bool TransactionHasError(); + void CommitTransaction(); + + // helper for the signature warning bool GenerateAuthWarning(const std::string &RealURI, const std::string &Message); - public: - pkgAcqMetaSigBase(pkgAcquire *Owner, - HashStringList const &ExpectedHashes=HashStringList(), - unsigned long TransactionID=0); + + pkgAcqMetaBase(pkgAcquire *Owner, + HashStringList const &ExpectedHashes=HashStringList(), + pkgAcqMetaBase *TransactionManager=NULL) + : Item(Owner, ExpectedHashes, TransactionManager) {}; }; + /** \brief An item that is responsible for downloading the meta-index {{{ * file (i.e., Release) itself and verifying its signature. * @@ -366,7 +378,7 @@ class pkgAcqMetaSigBase : public pkgAcquire::Item * otherwise, the expected hashsums will be "" (causing the * authentication of the index files to be bypassed). */ -class pkgAcqMetaIndex : public pkgAcqMetaSigBase +class pkgAcqMetaIndex : public pkgAcqMetaBase { void *d; @@ -458,7 +470,7 @@ class pkgAcqMetaIndex : public pkgAcqMetaSigBase /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, std::string URI,std::string URIDesc, std::string ShortDesc, std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc, const std::vector* IndexTargets, @@ -522,11 +534,11 @@ class pkgAcqBaseIndex : public pkgAcquire::Item std::string MetaKey; pkgAcqBaseIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : Item(Owner, ExpectedHashes, TransactionID), Target(Target), + : Item(Owner, ExpectedHashes, TransactionManager), Target(Target), MetaIndexParser(MetaIndexParser) {}; }; /*}}}*/ @@ -596,7 +608,7 @@ class pkgAcqDiffIndex : public pkgAcqBaseIndex * \param ExpectedHashes The list file's hashsums which are expected. */ pkgAcqDiffIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); @@ -687,7 +699,7 @@ class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex * check if it was the last one to complete the download step */ pkgAcqIndexMergeDiffs(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser, @@ -818,7 +830,7 @@ class pkgAcqIndexDiffs : public pkgAcqBaseIndex * that depends on it. */ pkgAcqIndexDiffs(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser, @@ -906,7 +918,7 @@ class pkgAcqIndex : public pkgAcqBaseIndex pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, HashStringList const &ExpectedHashes, std::string compressExt=""); - pkgAcqIndex(pkgAcquire *Owner, unsigned long TransactionID, + pkgAcqIndex(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser); @@ -946,7 +958,7 @@ class pkgAcqIndexTrans : public pkgAcqIndex std::string URI,std::string URIDesc, std::string ShortDesc); pkgAcqIndexTrans(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); @@ -996,7 +1008,7 @@ class OptionalIndexTarget : public IndexTarget * * \sa pkgAcqMetaIndex */ -class pkgAcqMetaSig : public pkgAcqMetaSigBase +class pkgAcqMetaSig : public pkgAcqMetaBase { void *d; @@ -1040,7 +1052,7 @@ class pkgAcqMetaSig : public pkgAcqMetaSigBase /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, std::string URI,std::string URIDesc, std::string ShortDesc, std::string MetaIndexFile, const std::vector* IndexTargets, diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index be4e494e0..9060d492b 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -169,75 +169,6 @@ void pkgAcquire::Remove(Item *Itm) } } /*}}}*/ -// Acquire::AbortTransaction - Remove a transaction /*{{{*/ -void pkgAcquire::AbortTransaction(unsigned long TransactionID) -{ - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "AbortTransaction: " << TransactionID << std::endl; - - std::vector Transaction; - for (ItemIterator I = Items.begin(); I != Items.end(); ++I) - if((*I)->TransactionID == TransactionID) - Transaction.push_back(*I); - - for (std::vector::iterator I = Transaction.begin(); - I != Transaction.end(); ++I) - { - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << " Cancel: " << (*I)->DestFile << std::endl; - // the transaction will abort, so stop anything that is idle - if ((*I)->Status == pkgAcquire::Item::StatIdle) - (*I)->Status = pkgAcquire::Item::StatDone; - } -} - /*}}}*/ -bool pkgAcquire::TransactionHasError(unsigned long TransactionID) -{ - std::vector Transaction; - for (ItemIterator I = Items.begin(); I != Items.end(); ++I) - if((*I)->TransactionID == TransactionID) - if((*I)->Status != pkgAcquire::Item::StatDone && - (*I)->Status != pkgAcquire::Item::StatIdle) - return true; - - return false; -} -// Acquire::CommitTransaction - Commit a transaction /*{{{*/ -void pkgAcquire::CommitTransaction(unsigned long TransactionID) -{ - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "CommitTransaction: " << TransactionID << std::endl; - - std::vector Transaction; - for (ItemIterator I = Items.begin(); I != Items.end(); ++I) - if((*I)->TransactionID == TransactionID) - Transaction.push_back(*I); - - // move new files into place *and* remove files that are not - // part of the transaction but are still on disk - for (std::vector::iterator I = Transaction.begin(); - I != Transaction.end(); ++I) - { - if((*I)->PartialFile != "") - { - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "mv " - << (*I)->PartialFile << " -> " - << (*I)->DestFile << std::endl; - Rename((*I)->PartialFile, (*I)->DestFile); - chmod((*I)->DestFile.c_str(),0644); - } else { - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "rm " - << (*I)->DestFile << std::endl; - unlink((*I)->DestFile.c_str()); - } - // mark that this transaction is finished - (*I)->TransactionID = 0; - } -} - /*}}}*/ - // Acquire::Add - Add a worker /*{{{*/ // --------------------------------------------------------------------- /* A list of workers is kept so that the select loop can direct their FD diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 7f51dd8f5..0113021b2 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -248,11 +248,6 @@ class pkgAcquire public: - /** \brief Abort a given transaction */ - void AbortTransaction(unsigned long TransactionID); - void CommitTransaction(unsigned long TransactionID); - bool TransactionHasError(unsigned long TransactionID); - /** \brief Retrieve information about a fetch method by name. * * \param Access The name of the method to look up. diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index b1dc060fe..f2d637676 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -265,14 +265,13 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const // this is normally created in pkgAcqMetaSig, but if we run // in --print-uris mode, we add it here if (tryInRelease == false) - new pkgAcqMetaIndex(Owner, 0, + new pkgAcqMetaIndex(Owner, NULL, MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), new indexRecords (Dist)); } - if (tryInRelease == true) new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"), MetaIndexInfo("InRelease"), "InRelease", @@ -281,7 +280,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const ComputeIndexTargets(), new indexRecords (Dist)); else - new pkgAcqMetaIndex(Owner, 0, + new pkgAcqMetaIndex(Owner, NULL, MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), -- cgit v1.2.3 From 916b89109cd77728004819d4705778e3dc489b2e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Sep 2014 09:24:19 +0200 Subject: fix gcc warnings --- apt-pkg/acquire-item.cc | 10 +++++----- apt-pkg/acquire-item.h | 7 +++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d36c47a49..1fa7971c5 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -850,7 +850,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - HashStringList const &ExpectedHash, string comprExt) + HashStringList const &ExpectedHash) : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI) { AutoSelectCompression(); @@ -1035,7 +1035,7 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) +void pkgAcqIndex::ReverifyAfterIMS() { std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") @@ -1132,7 +1132,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // The files timestamp matches if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) { - ReverifyAfterIMS(FileName); + ReverifyAfterIMS(); return; } Decompression = true; @@ -1154,7 +1154,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // The files timestamp matches if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - ReverifyAfterIMS(FileName); + ReverifyAfterIMS(); return; } @@ -1197,7 +1197,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con /* The Translation file is added to the queue */ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) - : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") + : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList()) { } /*}}}*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 622324347..0741ae60d 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -353,7 +353,7 @@ class pkgAcqMetaBase : public pkgAcquire::Item // transaction code void Add(Item *I); void AbortTransaction(); - bool TransactionHasError(); + bool TransactionHasError() APT_PURE; void CommitTransaction(); // helper for the signature warning @@ -885,7 +885,7 @@ class pkgAcqIndex : public pkgAcqBaseIndex std::string const &compExt); /** \brief Schedule file for verification after a IMS hit */ - void ReverifyAfterIMS(std::string const &FileName); + void ReverifyAfterIMS(); public: @@ -916,8 +916,7 @@ class pkgAcqIndex : public pkgAcqBaseIndex * fallback is ".gz" or none. */ pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, - std::string ShortDesc, HashStringList const &ExpectedHashes, - std::string compressExt=""); + std::string ShortDesc, HashStringList const &ExpectedHashes); pkgAcqIndex(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHash, -- cgit v1.2.3 From 6f5ccfde13b99be593a5cfce1ebc68a7ee995c88 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 23 Sep 2014 17:00:05 +0200 Subject: fix one broken test --- apt-pkg/acquire-item.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index e8b0e25de..7185a5efa 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1158,11 +1158,10 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) return; - // The files timestamp matches, for non-local URLs reverify the local - // file, for local file, uncompress again to ensure the hashsum is still - // matching the Release file - if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + // The files timestamp matches, reverify by copy into partial/ + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { + Erase = false; ReverifyAfterIMS(); return; } -- cgit v1.2.3 From 3f073d44c2d6cecb807f05a5f95c1c35ab23fb3c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 23 Sep 2014 17:18:30 +0200 Subject: cleanup --- apt-pkg/acquire-item.cc | 30 +++++++++++------------------- apt-pkg/acquire-item.h | 5 ++--- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7185a5efa..0cae5308e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -901,7 +901,10 @@ void pkgAcqIndex::AutoSelectCompression() CompressionExtension.erase(CompressionExtension.end()-1); } // AcqIndex::Init - defered Constructor /*{{{*/ -void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { +// --------------------------------------------------------------------- +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, + string const &ShortDesc) +{ Decompression = false; Erase = false; @@ -976,16 +979,9 @@ void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) /* The only header we use is the last-modified header. */ string pkgAcqIndex::Custom600Headers() const { - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + string Final = GetFinalFilename(); string msg = "\nIndex-File: true"; - // FIXME: this really should use "IndexTarget::IsOptional()" but that - // seems to be difficult without breaking ABI - if (ShortDesc().find("Translation") != 0) - msg += "\nFail-Ignore: true"; struct stat Buf; if (stat(Final.c_str(),&Buf) == 0) msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -1022,11 +1018,11 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ // pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ // --------------------------------------------------------------------- /* */ -std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, - std::string const &compExt) +std::string pkgAcqIndex::GetFinalFilename() const { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); std::string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(URI); + FinalFile += URItoFileName(RealURI); if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") FinalFile += ".gz"; return FinalFile; @@ -1042,7 +1038,7 @@ void pkgAcqIndex::ReverifyAfterIMS() DestFile += ".gz"; // copy FinalFile into partial/ so that we check the hash again - string FinalFile = GetFinalFilename(RealURI, compExt); + string FinalFile = GetFinalFilename(); Decompression = true; Desc.URI = "copy:" + FinalFile; QueueURI(Desc); @@ -1105,7 +1101,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // Done, queue for rename on transaction finished PartialFile = DestFile; - DestFile = GetFinalFilename(RealURI, compExt); + DestFile = GetFinalFilename(); return; } @@ -1225,11 +1221,7 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, // --------------------------------------------------------------------- string pkgAcqIndexTrans::Custom600Headers() const { - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(RealURI); - - if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + string Final = GetFinalFilename(); struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3c522f66e..ae53fc6ac 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -880,10 +880,9 @@ class pkgAcqIndex : public pkgAcqBaseIndex /** \brief Auto select the right compression to use */ void AutoSelectCompression(); - /** \brief Get the full pathname of the final file for the given URI + /** \brief Get the full pathname of the final file for the current URI */ - std::string GetFinalFilename(std::string const &URI, - std::string const &compExt); + std::string GetFinalFilename() const; /** \brief Schedule file for verification after a IMS hit */ void ReverifyAfterIMS(); -- cgit v1.2.3 From c8aa88aa2c3139584cfabb1ce4619c773e9f2b99 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 23 Sep 2014 18:08:53 +0200 Subject: cleanup, fix test-apt-update-unauth as the behavior of apt changed --- apt-pkg/acquire-item.cc | 9 ++++++--- apt-pkg/acquire-item.h | 4 ---- test/integration/test-apt-update-unauth | 28 +++++++++++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 0cae5308e..c819afd9b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1051,7 +1051,8 @@ void pkgAcqIndex::ReverifyAfterIMS() to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a compressed uri. */ -void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, +void pkgAcqIndex::Done(string Message, unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hashes,Cfg); @@ -1164,7 +1165,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con string decompProg; // If we enable compressed indexes, queue for hash verification - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") { DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI) + ".gz"; @@ -1682,6 +1683,8 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ std::cerr << "Signature verification succeeded: " << DestFile << std::endl; +// we ensure this by other means +#if 0 // do not trust any previously unverified content that we may have string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); if (DestFile != SigFile) @@ -1702,7 +1705,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ unlink(index.c_str()); } } - +#endif // Download further indexes with verification QueueIndexes(true); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index ae53fc6ac..49f057b43 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -859,10 +859,6 @@ class pkgAcqIndex : public pkgAcqBaseIndex */ bool Erase; - // FIXME: - // Unused, used to be used to verify that "Packages: " header was there - bool __DELME_ON_NEXT_ABI_BREAK_Verify; - /** \brief The object that is actually being fetched (minus any * compression-related extensions). */ diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index 13487603c..4e08b5e35 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -17,23 +17,41 @@ insertsource 'unstable' 'foo' 'all' '1.0' setupaptarchive changetowebserver +# FIXME: +# - also check the unauth -> auth success case, i.e. that all files are +# reverified runtest() { # start unauthenticated find rootdir/var/lib/apt/lists/ -type f | xargs rm -f rm -f aptarchive/dists/unstable/*Release* + # remove uncompressed version + find aptarchive/ -name Packages | xargs rm -f aptget update -qq # become authenticated generatereleasefiles signreleasefiles - # and ensure we do download the data again - msgtest "Check that the data is check when going to authenticated" - if aptget update |grep -q Hit; then - msgfail - else + # and ensure we re-check the downloaded data + msgtest "Check rollback on going from unauth -> auth" + + # change the local packages file + PKGS=$(ls rootdir/var/lib/apt/lists/*Packages*) + echo "meep" > $PKGS + ls -l rootdir/var/lib/apt/lists > lists.before + + # update and ensure all is reverted on the hashsum failure + aptget update -o Debug::Acquire::Transaction=1 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 > output.log 2>&1 || true + + # ensure we have before what we have after + ls -l rootdir/var/lib/apt/lists > lists.after + if diff -u lists.before lists.after; then msgpass + else + #cat output.log + msgfail fi + } for COMPRESSEDINDEXES in 'false' 'true'; do -- cgit v1.2.3 From 03bfbc965443393b92b2d6d82613472fa3a5067f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 23 Sep 2014 23:48:19 +0200 Subject: make pdiff transactional (but at the cost of a CopyFile() --- apt-pkg/acquire-item.cc | 101 +++++++++++++++++++++++++++----------- apt-pkg/acquire-item.h | 8 ++- test/integration/test-pdiff-usage | 1 + 3 files changed, 79 insertions(+), 31 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c819afd9b..5bfc72adf 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -157,7 +157,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis // --------------------------------------------------------------------- /* This helper function is used by a lot of item methods as their final step */ -void pkgAcquire::Item::Rename(string From,string To) +bool pkgAcquire::Item::Rename(string From,string To) { if (rename(From.c_str(),To.c_str()) != 0) { @@ -165,8 +165,10 @@ void pkgAcquire::Item::Rename(string From,string To) snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno), From.c_str(),To.c_str()); Status = StatError; - ErrorText = S; + ErrorText += S; + return false; } + return true; } /*}}}*/ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/ @@ -252,7 +254,7 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, - MetaIndexParser) + MetaIndexParser), PackagesFileReadyInPartial(false) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -348,6 +350,10 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ // we have the same sha1 as the server so we are done here if(Debug) std::clog << "Package file is up-to-date" << std::endl; + // ensure we have no leftovers from previous runs + std::string Partial = _config->FindDir("Dir::State::lists"); + Partial += "partial/" + URItoFileName(RealURI); + unlink(Partial.c_str()); // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us new pkgAcqIndexDiffs(Owner, TransactionManager, Target, @@ -418,6 +424,21 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ // we have something, queue the next diff if(found) { + // FIXME: make this use the method + PackagesFileReadyInPartial = true; + std::string Partial = _config->FindDir("Dir::State::lists"); + Partial += "partial/" + URItoFileName(RealURI); + + FileFd From(CurrentPackagesFile, FileFd::ReadOnly); + FileFd To(Partial, FileFd::WriteEmpty); + if(CopyFile(From, To) == false) + return _error->Errno("CopyFile", "failed to copy"); + + if(Debug) + std::cerr << "Done copying " << CurrentPackagesFile + << " -> " << Partial + << std::endl; + // queue the diffs string::size_type const last_space = Description.rfind(" "); if(last_space != string::npos) @@ -455,10 +476,10 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ diffs); } - Complete = false; - Status = StatDone; - Dequeue(); - return true; + Complete = false; + Status = StatDone; + Dequeue(); + return true; } } @@ -541,7 +562,9 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, if(available_patches.empty() == true) { - // we are done (yeah!) + // we are done (yeah!), check hashes against the final file + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(Target->URI); Finish(true); } else @@ -573,13 +596,6 @@ void pkgAcqIndexDiffs::Finish(bool allDone) // the file will be cleaned if(allDone) { - DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI); - - // FIXME: we want the rred stuff to use the real transactional update - // this is just a workaround - PartialFile = DestFile; - if(HashSums().usable() && !HashSums().VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); @@ -587,6 +603,16 @@ void pkgAcqIndexDiffs::Finish(bool allDone) return; } + // queue for copy + PartialFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); + + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + + // this happens if we have a up-to-date indexfile + if(!FileExists(PartialFile)) + PartialFile = DestFile; + // this is for the "real" finish Complete = true; Status = StatDone; @@ -606,10 +632,15 @@ void pkgAcqIndexDiffs::Finish(bool allDone) /*}}}*/ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ { - // calc sha1 of the just patched file string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); + FinalFile += "partial/" + URItoFileName(RealURI); + + if(!FileExists(FinalFile)) + { + Failed("No FinalFile " + FinalFile + " available", NULL); + return false; + } FileFd fd(FinalFile, FileFd::ReadOnly); SHA1Summation SHA1; @@ -619,6 +650,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ std::clog << "QueueNextDiff: " << FinalFile << " (" << local_sha1 << ")"<FindDir("Dir::State::lists")+URItoFileName(RealURI); + FinalFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); // success in downloading a diff, enter ApplyDiff state if(State == StateFetchDiff) @@ -711,6 +743,8 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi ServerSha1, available_patches); return Finish(); } else + // update + DestFile = FinalFile; return Finish(true); } } @@ -777,7 +811,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Item::Done(Message,Size,Hashes,Cnf); - string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + string const FinalFile = _config->FindDir("Dir::State::lists") + "partial/" + URItoFileName(RealURI); if (State == StateFetchDiff) { @@ -817,23 +851,27 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri return; } + + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + // move the result into place if(Debug) - std::clog << "Moving patched file in place: " << std::endl + std::clog << "Queue patched file in place: " << std::endl << DestFile << " -> " << FinalFile << std::endl; - Rename(DestFile, FinalFile); - chmod(FinalFile.c_str(), 0644); - // otherwise lists cleanup will eat the file + // queue for copy by the transaction manager + PartialFile = DestFile; DestFile = FinalFile; - // FIXME: make the merged rred code really transactional - PartialFile = FinalFile; // ensure the ed's are gone regardless of list-cleanup for (std::vector::const_iterator I = allPatches->begin(); I != allPatches->end(); ++I) { - std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz"; + std::string PartialFile = _config->FindDir("Dir::State::lists"); + PartialFile += "partial/" + URItoFileName(RealURI); + std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz"; + std::cerr << patch << std::endl; unlink(patch.c_str()); } @@ -1306,13 +1344,18 @@ void pkgAcqMetaBase::CommitTransaction() if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "mv " << (*I)->PartialFile << " -> " - << (*I)->DestFile << std::endl; + << (*I)->DestFile << " " + << (*I)->DescURI() + << std::endl; Rename((*I)->PartialFile, (*I)->DestFile); chmod((*I)->DestFile.c_str(),0644); } else { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "rm " - << (*I)->DestFile << std::endl; + << (*I)->DestFile + << " " + << (*I)->DescURI() + << std::endl; unlink((*I)->DestFile.c_str()); } // mark that this transaction is finished diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 49f057b43..15b566069 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -90,7 +90,7 @@ class pkgAcquire::Item : public WeakPointable * \param To The new name of \a From. If \a To exists it will be * overwritten. */ - void Rename(std::string From,std::string To); + bool Rename(std::string From,std::string To); public: @@ -574,6 +574,10 @@ class pkgAcqDiffIndex : public pkgAcqBaseIndex */ std::string Description; + /** \brief If the copy step of the packages file is done + */ + bool PackagesFileReadyInPartial; + public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); @@ -803,7 +807,7 @@ class pkgAcqIndexDiffs : public pkgAcqBaseIndex virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() const {return RealURI + "Index";}; + virtual std::string DescURI() const {return RealURI + "IndexDiffs";}; /** \brief Create an index diff item. * diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 74749d6ab..e86963f28 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -159,6 +159,7 @@ SHA1-Patches: " aptcache show apt newstuff } echo 'Debug::pkgAcquire::Diffs "true"; +Debug::Acquire::Transaction "true"; Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 -- cgit v1.2.3 From e7d37fac6328bba86fa183edc284ac629f7c03e4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 26 Sep 2014 18:13:48 +0200 Subject: Do not download Packages/Sources files on I-M-S hit of the Release file With this branch we know that the data in the lists directory is always what the release file says, so if the Release file is unchanged, then there is no need to queue the download of the other indexfiles as they will be unchanged too (or broken :) --- apt-pkg/acquire-item.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 5bfc72adf..9d9aec4d0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1750,8 +1750,14 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ } #endif - // Download further indexes with verification - QueueIndexes(true); + // Download further indexes with verification + // + // we do not need to download indexfiles if the Release file has not + // changed because without a changed release file there are no new hashes + // and we ensure that the repository is always "complete" (i.e. all + // that is in the release file is downloaded) + if(IMSHit == false) + QueueIndexes(true); #if 0 // is it a clearsigned MetaIndex file? -- cgit v1.2.3 From c4ffa0428b617cd844f0f9dfd5d16ae0553675ac Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 26 Sep 2014 20:59:56 +0200 Subject: Print warning for unauthenticated repositories --- apt-pkg/acquire-item.cc | 4 +++ .../integration/test-apt-get-update-unauth-warning | 30 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 test/integration/test-apt-get-update-unauth-warning diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 9d9aec4d0..4e843ecaf 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1970,6 +1970,10 @@ void pkgAcqMetaIndex::Failed(string Message, DestFile = FinalFile; } + // warn if the repository is unsinged + _error->Warning(_("The data from '%s' is not signed. Packages " + "from that repository can not be authenticated."), + URIDesc.c_str()); // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification QueueIndexes(false); diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning new file mode 100755 index 000000000..4411a7430 --- /dev/null +++ b/test/integration/test-apt-get-update-unauth-warning @@ -0,0 +1,30 @@ +#!/bin/sh +# +# ensure we print warnings for unauthenticated repositories +# +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +# a "normal" package with source and binary +buildsimplenativepackage 'foo' 'all' '2.0' + +setupaptarchive --no-update + +APTARCHIVE=$(readlink -f ./aptarchive) +rm -f $APTARCHIVE/dists/unstable/*Release* + +# update without authenticated InRelease file +testequal "Ign file: unstable InRelease +Ign file: unstable Release +Reading package lists... +W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated." aptget update + +# ensure we can not install the package +testequal "WARNING: The following packages cannot be authenticated! + foo +E: There are problems and -y was used without --force-yes" aptget install -qq -y foo -- cgit v1.2.3 From 631a7dc7906a10ccd5f14dcfe42224e6107e11f6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 26 Sep 2014 20:59:31 +0200 Subject: Do not allow going from authenticated to unauthenticated repo Also rework the way we load the Release file, so it only after Release.gpg verified the Release file. The rational is that we never want to load untrusted data into our parsers. Only stuff verified with gpg or by its hashes get loaded. To load untrusted data you now need to use apt-get update --allow-unauthenticated. --- apt-pkg/acquire-item.cc | 79 +++++++++-- apt-pkg/acquire-item.h | 148 ++++++++++---------- test/integration/test-apt-update-nofallback | 207 ++++++++++++++++++++++++++++ 3 files changed, 348 insertions(+), 86 deletions(-) create mode 100755 test/integration/test-apt-update-nofallback diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 4e843ecaf..7fad5605d 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -193,6 +193,14 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const Status = StatError; // do not report as usually its not the mirrors fault, but Portal/Proxy break; + case SignatureError: + ErrorText = _("Signature error"); + Status = StatError; + break; + case NotClearsigned: + ErrorText = _("Does not start with a cleartext signature"); + Status = StatError; + break; } return false; } @@ -1307,6 +1315,8 @@ void pkgAcqMetaBase::AbortTransaction() if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "AbortTransaction: " << TransactionManager << std::endl; + // ensure the toplevel is in error state too + Status = pkgAcquire::Item::StatError; for (std::vector::iterator I = Transaction.begin(); I != Transaction.end(); ++I) { @@ -1407,6 +1417,7 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ indexRecords* MetaIndexParser) : pkgAcqMetaBase(Owner, HashStringList(), TransactionManager), RealURI(URI), MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), + URIDesc(URIDesc), ShortDesc(ShortDesc), IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -1505,13 +1516,35 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList DestFile += URItoFileName(RealURI); } - Complete = true; + // we parse the MetaIndexFile here because at this point we can + // trust the data + if(AuthPass == true) + { + // load indexes and queue further downloads + MetaIndexParser->Load(MetaIndexFile); + ((pkgAcqMetaIndex*)TransactionManager)->QueueIndexes(true); + } + Complete = true; } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + + // FIXME: meh, this is not really elegant + string InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12, + "InRelease"); + string FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI); + + if(RealFileExists(Final) || RealFileExists(FinalInRelease)) + { + _error->Error("The repository '%s' is no longer signed.", + URIDesc.c_str()); + Rename(MetaIndexFile, MetaIndexFile+".FAILED"); + TransactionManager->AbortTransaction(); + return; + } // this ensures that any file in the lists/ dir is removed by the // transaction @@ -1527,6 +1560,19 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ return; } + // only allow going further if the users explicitely wants it + if(_config->FindB("APT::Get::AllowUnauthenticated", false)) + { + _error->Warning("Please use --allow-unauthenticated"); + } + else + { + // we parse the indexes here because at this point the user wanted + // a repository that may potentially harm him + MetaIndexParser->Load(MetaIndexFile); + ((pkgAcqMetaIndex*)TransactionManager)->QueueIndexes(true); + } + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) @@ -1547,6 +1593,7 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ const vector* IndexTargets, indexRecords* MetaIndexParser) : pkgAcqMetaBase(Owner, HashStringList(), TransactionManager), RealURI(URI), IndexTargets(IndexTargets), + URIDesc(URIDesc), ShortDesc(ShortDesc), MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) @@ -1619,13 +1666,7 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList // Still more retrieving to do return; - if (SigFile == "") - { - // load indexes, the signature will downloaded afterwards - MetaIndexParser->Load(DestFile); - QueueIndexes(true); - } - else + if (SigFile != "") { // There was a signature file, so pass it to gpgv for // verification @@ -1952,7 +1993,8 @@ void pkgAcqMetaIndex::Failed(string Message, /* Always move the meta index, even if gpgv failed. This ensures * that PackageFile objects are correctly filled in */ - if (FileExists(DestFile)) { + if (FileExists(DestFile)) + { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); /* InRelease files become Release files, otherwise @@ -1970,13 +2012,19 @@ void pkgAcqMetaIndex::Failed(string Message, DestFile = FinalFile; } - // warn if the repository is unsinged - _error->Warning(_("The data from '%s' is not signed. Packages " - "from that repository can not be authenticated."), - URIDesc.c_str()); // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification - QueueIndexes(false); + // only allow going further if the users explicitely wants it + if(_config->FindB("APT::Get::AllowUnauthenticated", false)) + { + // warn if the repository is unsinged + _error->Warning(_("The data from '%s' is not signed. Packages " + "from that repository can not be authenticated."), + URIDesc.c_str()); + _error->Warning("Please use --allow-unauthenticated"); + } else { + QueueIndexes(false); + } } /*}}}*/ @@ -2060,7 +2108,8 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) { pkgAcquire::Item::Failed(Message, Cnf); - ErrorText = _("Does not start with a cleartext signature"); + RenameOnError(NotClearsigned); + TransactionManager->AbortTransaction(); return; } pkgAcqMetaIndex::Done(Message, Size, Hashes, Cnf); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 15b566069..cc156cf17 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -312,7 +312,9 @@ class pkgAcquire::Item : public WeakPointable enum RenameOnErrorState { HashSumMismatch, SizeMismatch, - InvalidFormat + InvalidFormat, + SignatureError, + NotClearsigned, }; /** \brief Rename failed file and set error @@ -367,6 +369,67 @@ class pkgAcqMetaBase : public pkgAcquire::Item : Item(Owner, ExpectedHashes, TransactionManager) {}; }; +/** \brief An acquire item that downloads the detached signature {{{ + * of a meta-index (Release) file, then queues up the release + * file itself. + * + * \todo Why protected members? + * + * \sa pkgAcqMetaIndex + */ +class pkgAcqMetaSig : public pkgAcqMetaBase +{ + void *d; + + protected: + + /** \brief The URI of the signature file. Unlike Desc.URI, this is + * never modified; it is used to determine the file that is being + * downloaded. + */ + std::string RealURI; + + std::string URIDesc; + std::string ShortDesc; + + /** \brief A package-system-specific parser for the meta-index file. */ + indexRecords* MetaIndexParser; + + /** \brief The file we need to verify */ + std::string MetaIndexFile; + + /** \brief The index files which should be looked up in the meta-index + * and then downloaded. + * + * \todo Why a list of pointers instead of a list of structs? + */ + const std::vector* IndexTargets; + + /** \brief If we are in fetching or download state */ + bool AuthPass; + + /** \brief Was this file already on disk */ + bool IMSHit; + + public: + + // Specialized action members + virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cnf); + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return RealURI; }; + + /** \brief Create a new pkgAcqMetaSig. */ + pkgAcqMetaSig(pkgAcquire *Owner, + pkgAcqMetaBase *TransactionManager, + std::string URI,std::string URIDesc, std::string ShortDesc, + std::string MetaIndexFile, + const std::vector* IndexTargets, + indexRecords* MetaIndexParser); + virtual ~pkgAcqMetaSig(); +}; + /*}}}*/ /** \brief An item that is responsible for downloading the meta-index {{{ * file (i.e., Release) itself and verifying its signature. @@ -436,15 +499,8 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase */ void AuthDone(std::string Message); - /** \brief Starts downloading the individual index files. - * - * \param verify If \b true, only indices whose expected hashsum - * can be determined from the meta-index will be downloaded, and - * the hashsums of indices will be checked (reporting - * #StatAuthError if there is a mismatch). If verify is \b false, - * no hashsum checking will be performed. - */ - void QueueIndexes(bool verify); + std::string URIDesc; + std::string ShortDesc; /** \brief The URI of the meta-index file for the detached signature */ std::string MetaIndexSigURI; @@ -459,7 +515,17 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase void Init(std::string URIDesc, std::string ShortDesc); public: - + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected hashsum + * can be determined from the meta-index will be downloaded, and + * the hashsums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no hashsum checking will be performed. + */ + void QueueIndexes(bool verify); + // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, @@ -999,66 +1065,6 @@ class OptionalIndexTarget : public IndexTarget }; /*}}}*/ -/** \brief An acquire item that downloads the detached signature {{{ - * of a meta-index (Release) file, then queues up the release - * file itself. - * - * \todo Why protected members? - * - * \sa pkgAcqMetaIndex - */ -class pkgAcqMetaSig : public pkgAcqMetaBase -{ - void *d; - - protected: - /** \brief The last good signature file */ - std::string LastGoodSig; - - /** \brief The URI of the signature file. Unlike Desc.URI, this is - * never modified; it is used to determine the file that is being - * downloaded. - */ - std::string RealURI; - - /** \brief A package-system-specific parser for the meta-index file. */ - indexRecords* MetaIndexParser; - - /** \brief The file we need to verify */ - std::string MetaIndexFile; - - /** \brief The index files which should be looked up in the meta-index - * and then downloaded. - * - * \todo Why a list of pointers instead of a list of structs? - */ - const std::vector* IndexTargets; - - /** \brief If we are in fetching or download state */ - bool AuthPass; - - /** \brief Was this file already on disk */ - bool IMSHit; - - public: - - // Specialized action members - virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, - pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers() const; - virtual std::string DescURI() const {return RealURI; }; - - /** \brief Create a new pkgAcqMetaSig. */ - pkgAcqMetaSig(pkgAcquire *Owner, - pkgAcqMetaBase *TransactionManager, - std::string URI,std::string URIDesc, std::string ShortDesc, - std::string MetaIndexFile, - const std::vector* IndexTargets, - indexRecords* MetaIndexParser); - virtual ~pkgAcqMetaSig(); -}; - /*}}}*/ /** \brief An item that is responsible for fetching a package file. {{{ * * If the package file already exists in the cache, nothing will be diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback new file mode 100755 index 000000000..4e8ea9916 --- /dev/null +++ b/test/integration/test-apt-update-nofallback @@ -0,0 +1,207 @@ +#!/bin/sh +# +# ensure we never fallback from a signed to a unsigned repo +# +# hash checks are done in +# +set -e + +simulate_mitm_and_inject_evil_package() +{ + rm -f $APTARCHIVE/dists/unstable/InRelease + rm -f $APTARCHIVE/dists/unstable/Release.gpg + inject_evil_package +} + +inject_evil_package() +{ + cat > $APTARCHIVE/dists/unstable/main/binary-i386/Packages < +Architecture: all +Version: 1.0 +Filename: pool/evil_1.0_all.deb +Size: 1270 +Description: an autogenerated evil package +EOF + # avoid ims hit + touch -d '+1hour' aptarchive/dists/unstable/main/binary-i386/Packages +} + +assert_update_is_refused_and_last_good_state_used() +{ + testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq + + assert_repo_is_intact +} + +assert_repo_is_intact() +{ + testequal "foo/unstable 2.0 all" apt list -q + testsuccess "" aptget install -y -s foo + testfailure "" aptget install -y evil + + LISTDIR=rootdir/var/lib/apt/lists + if ! ( ls $LISTDIR/*InRelease >/dev/null 2>&1 || + ls $LISTDIR/*Release.gpg >/dev/null 2>&1 ); then + echo "Can not find InRelease/Release.gpg in $(ls $LISTDIR)" + msgfail + fi +} + +setupaptarchive_with_lists_clean() +{ + setupaptarchive --no-update + rm -f rootdir/var/lib/apt/lists/_* + #rm -rf rootdir/var/lib/apt/lists +} + +test_from_inrelease_to_unsigned() +{ + # setup archive with InRelease file + setupaptarchive_with_lists_clean + testsuccess aptget update + + simulate_mitm_and_inject_evil_package + assert_update_is_refused_and_last_good_state_used +} + +test_from_release_gpg_to_unsigned() +{ + # setup archive with Release/Release.gpg (but no InRelease) + setupaptarchive_with_lists_clean + rm $APTARCHIVE/dists/unstable/InRelease + testsuccess aptget update + + simulate_mitm_and_inject_evil_package + assert_update_is_refused_and_last_good_state_used +} + +test_cve_2012_0214() +{ + # see https://bugs.launchpad.net/ubuntu/+source/apt/+bug/947108 + # + # it was possible to MITM the download so that InRelease/Release.gpg + # are not delivered (404) and a altered Release file was send + # + # apt left the old InRelease file in /var/lib/apt/lists and downloaded + # the unauthenticated Release file too giving the false impression that + # Release was authenticated + # + # Note that this is pretty much impossible nowdays because: + # a) InRelease is left as is, not split to InRelease/Release as it was + # in the old days + # b) we refuse to go from signed->unsigned + # + # Still worth having a regression test the simulates the condition + + # setup archive with InRelease + setupaptarchive_with_lists_clean + testsuccess aptget update + + # do what CVE-2012-0214 did + rm $APTARCHIVE/dists/unstable/InRelease + rm $APTARCHIVE/dists/unstable/Release.gpg + inject_evil_package + # build valid Release file + aptftparchive -qq release ./aptarchive > aptarchive/dists/unstable/Release + + assert_update_is_refused_and_last_good_state_used + + # ensure there is no _Release file downloaded + testfailure ls rootdir/var/lib/apt/lists/*_Release +} + +test_subvert_inrelease() +{ + # setup archive with InRelease + setupaptarchive_with_lists_clean + testsuccess aptget update + + # replace InRelease with something else + mv $APTARCHIVE/dists/unstable/Release $APTARCHIVE/dists/unstable/InRelease + + testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease Does not start with a cleartext signature + +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq + + # ensure we keep the repo + assert_repo_is_intact +} + +test_inrelease_to_invalid_inrelease() +{ + # setup archive with InRelease + setupaptarchive_with_lists_clean + testsuccess aptget update + + # now remove InRelease and subvert Release do no longer verify + sed -i 's/Codename.*/Codename: evil!'/ $APTARCHIVE/dists/unstable/InRelease + inject_evil_package + + testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) + +W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease + +W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq + + # ensure we keep the repo + assert_repo_is_intact + testfailure grep "evil" rootdir/var/lib/apt/lists/*InRelease +} + +test_release_gpg_to_invalid_release_release_gpg() +{ + # setup archive with InRelease + setupaptarchive_with_lists_clean + rm $APTARCHIVE/dists/unstable/InRelease + testsuccess aptget update + + # now subvert Release do no longer verify + echo "Some evil data" >> $APTARCHIVE/dists/unstable/Release + inject_evil_package + + testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq + + assert_repo_is_intact + testfailure grep "evil" rootdir/var/lib/apt/lists/*Release +} + + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +# a "normal" package with source and binary +buildsimplenativepackage 'foo' 'all' '2.0' + +# setup the archive and ensure we have a single package that installs fine +setupaptarchive +APTARCHIVE=$(readlink -f ./aptarchive) +assert_repo_is_intact + +# test the various cases where a repo may go from signed->unsigned +msgmsg "test_from_inrelease_to_unsigned" +test_from_inrelease_to_unsigned + +msgmsg "test_from_release_gpg_to_unsigned" +test_from_release_gpg_to_unsigned + +# ensure we do not regress on CVE-2012-0214 +msgmsg "test_cve_2012_0214" +test_cve_2012_0214 + +# ensure InRelase can not be subverted +msgmsg "test_subvert_inrelease" +test_subvert_inrelease + +# ensure we revert to last good state if InRelease does not verify +msgmsg "test_inrelease_to_invalid_inrelease" +test_inrelease_to_invalid_inrelease + +# ensure we revert to last good state if Release/Release.gpg does not verify +msgmsg "test_release_gpg_to_invalid_release_release_gpg" +test_release_gpg_to_invalid_release_release_gpg -- cgit v1.2.3 From bca84917c326fa3158e120147c8aecebe0789b47 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 26 Sep 2014 22:45:18 +0200 Subject: test fixes --- apt-pkg/acquire-item.cc | 41 +++++++++++----------- test/integration/test-apt-get-source-authenticated | 2 +- .../integration/test-apt-get-update-unauth-warning | 14 ++++++-- test/integration/test-apt-update-rollback | 16 ++++----- test/integration/test-apt-update-unauth | 2 +- .../test-bug-717891-abolute-uris-for-proxies | 2 +- test/integration/test-bug-738785-switch-protocol | 2 +- test/integration/test-policy-pinning | 3 +- 8 files changed, 46 insertions(+), 36 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7fad5605d..b8317b13d 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1316,7 +1316,6 @@ void pkgAcqMetaBase::AbortTransaction() std::clog << "AbortTransaction: " << TransactionManager << std::endl; // ensure the toplevel is in error state too - Status = pkgAcquire::Item::StatError; for (std::vector::iterator I = Transaction.begin(); I != Transaction.end(); ++I) { @@ -1542,6 +1541,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ _error->Error("The repository '%s' is no longer signed.", URIDesc.c_str()); Rename(MetaIndexFile, MetaIndexFile+".FAILED"); + Status = pkgAcquire::Item::StatError; TransactionManager->AbortTransaction(); return; } @@ -1561,16 +1561,16 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } // only allow going further if the users explicitely wants it - if(_config->FindB("APT::Get::AllowUnauthenticated", false)) - { - _error->Warning("Please use --allow-unauthenticated"); - } - else + if(_config->FindB("APT::Get::AllowUnauthenticated", false) == true) { // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him MetaIndexParser->Load(MetaIndexFile); ((pkgAcqMetaIndex*)TransactionManager)->QueueIndexes(true); + } + else + { + _error->Warning("Use --allow-unauthenticated to force the update"); } // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor @@ -1793,12 +1793,12 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ // Download further indexes with verification // - // we do not need to download indexfiles if the Release file has not - // changed because without a changed release file there are no new hashes - // and we ensure that the repository is always "complete" (i.e. all - // that is in the release file is downloaded) - if(IMSHit == false) - QueueIndexes(true); + // it would be really nice if we could simply do + // if (IMSHit == false) QueueIndexes(true) + // and skip the download if the Release file has not changed + // - but right now the list cleaner will needs to be tricked + // to not delete all our packages/source indexes in this case + QueueIndexes(true); #if 0 // is it a clearsigned MetaIndex file? @@ -2012,19 +2012,20 @@ void pkgAcqMetaIndex::Failed(string Message, DestFile = FinalFile; } + _error->Warning(_("The data from '%s' is not signed. Packages " + "from that repository can not be authenticated."), + URIDesc.c_str()); + // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification // only allow going further if the users explicitely wants it - if(_config->FindB("APT::Get::AllowUnauthenticated", false)) + if(_config->FindB("APT::Get::AllowUnauthenticated", false) == true) { - // warn if the repository is unsinged - _error->Warning(_("The data from '%s' is not signed. Packages " - "from that repository can not be authenticated."), - URIDesc.c_str()); - _error->Warning("Please use --allow-unauthenticated"); - } else { QueueIndexes(false); - } + } else { + // warn if the repository is unsinged + _error->Warning("Use --allow-unauthenticated to force the update"); + } } /*}}}*/ diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated index 2cee13923..d73097b54 100755 --- a/test/integration/test-apt-get-source-authenticated +++ b/test/integration/test-apt-get-source-authenticated @@ -21,7 +21,7 @@ APTARCHIVE=$(readlink -f ./aptarchive) rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated InRelease file -testsuccess aptget update +testsuccess aptget update --allow-unauthenticated # this all should fail testfailure aptget install -y foo diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 4411a7430..b1c676738 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -18,11 +18,21 @@ setupaptarchive --no-update APTARCHIVE=$(readlink -f ./aptarchive) rm -f $APTARCHIVE/dists/unstable/*Release* -# update without authenticated InRelease file +# update without authenticated files leads to warning testequal "Ign file: unstable InRelease Ign file: unstable Release Reading package lists... -W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated." aptget update +W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated. +W: Use --allow-unauthenticated to force the update" aptget update + +# no package foo +testequal "Listing..." apt list foo + +# allow override +testequal "Ign file: unstable InRelease +Ign file: unstable Release +Reading package lists... +W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated." aptget update --allow-unauthenticated # ensure we can not install the package testequal "WARNING: The following packages cannot be authenticated! diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index ccd7f57ff..a88b0042b 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -75,15 +75,14 @@ test_inreleae_to_valid_release() { rm $APTARCHIVE/dists/unstable/Release.gpg avoid_ims_hit - # update works - testsuccess aptget update -o Debug::Acquire::Transaction=1 + # update fails + testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # test that we can install the new packages but do no longer have a sig testsuccess aptget install old -s - testsuccess aptget install new -s - testfailure ls $ROOTDIR/var/lib/apt/lists/*_InRelease - testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release.gpg - testsuccess ls $ROOTDIR/var/lib/apt/lists/*_Release + testfailure aptget install new -s + testsuccess ls $ROOTDIR/var/lib/apt/lists/*_InRelease + testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release } test_inreleae_to_release_reverts_all() { @@ -98,9 +97,7 @@ test_inreleae_to_release_reverts_all() { break_repository_sources_index # ensure error - testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch - -E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq # -o Debug::acquire::transaction=1 + testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1 # ensure that the Packages file is also rolled back testsuccess aptget install old -s @@ -112,6 +109,7 @@ E: Some index files failed to download. They have been ignored, or old ones used test_unauthenticated_to_invalid_inrelease() { msgmsg "Test UnAuthenticated to invalid InRelease reverts everything" create_fresh_archive + rm -rf rootdir/var/lib/apt/lists/* rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg avoid_ims_hit diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index 4e08b5e35..2e46e3ace 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -26,7 +26,7 @@ runtest() { rm -f aptarchive/dists/unstable/*Release* # remove uncompressed version find aptarchive/ -name Packages | xargs rm -f - aptget update -qq + aptget update -qq --allow-unauthenticated # become authenticated generatereleasefiles diff --git a/test/integration/test-bug-717891-abolute-uris-for-proxies b/test/integration/test-bug-717891-abolute-uris-for-proxies index ac1d6ec11..a8947b5e2 100755 --- a/test/integration/test-bug-717891-abolute-uris-for-proxies +++ b/test/integration/test-bug-717891-abolute-uris-for-proxies @@ -12,7 +12,7 @@ setupaptarchive changetowebserver --request-absolute='uri' msgtest 'Check that absolute paths are' 'not accepted' -testfailure --nomsg aptget update +testfailure --nomsg aptget update --allow-unauthenticated echo 'Acquire::http::Proxy "http://localhost:8080";' > rootdir/etc/apt/apt.conf.d/99proxy diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol index 1e5748eae..4ff044515 100755 --- a/test/integration/test-bug-738785-switch-protocol +++ b/test/integration/test-bug-738785-switch-protocol @@ -60,4 +60,4 @@ mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS} # check that downgrades from https to http are not allowed webserverconfig 'aptwebserver::support::http' 'true' sed -i -e 's#:8080/redirectme#:4433/downgrademe#' -e 's# http:# https:#' rootdir/etc/apt/sources.list.d/* -testfailure aptget update +testfailure aptget update --allow-unauthenticated diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning index 8eb4bcbad..2281d7a1d 100755 --- a/test/integration/test-policy-pinning +++ b/test/integration/test-policy-pinning @@ -28,7 +28,7 @@ Pinned packages:" aptcache policy $* aptgetupdate() { # just to be sure that no old files are used rm -rf rootdir/var/lib/apt - if aptget update -qq 2>&1 | grep '^E: '; then + if aptget update --allow-unauthenticated -qq 2>&1 | grep '^E: '; then msgwarn 'apt-get update failed with an error' fi } @@ -36,6 +36,7 @@ aptgetupdate() { ### not signed archive aptgetupdate + testequalpolicy 100 500 testequalpolicy 990 500 -t now -- cgit v1.2.3 From c36db2b56a08bb7542d0c1be900ee90808efa770 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 10:41:31 +0200 Subject: Adjust DestFile in ReverifyAfterIMS() to not include compr Extension --- apt-pkg/acquire-item.cc | 7 +++++++ apt-pkg/contrib/fileutl.cc | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 923a153a7..d6b00f9b7 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1104,6 +1104,13 @@ std::string pkgAcqIndex::GetFinalFilename() const void pkgAcqIndex::ReverifyAfterIMS() { std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + + // update destfile to *not* include the compression extension when doing + // a reverify (as its uncompressed on disk already) + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + + // adjust DestFile if its compressed on disk if (_config->FindB("Acquire::GzipIndexes",false) == true) DestFile += compExt; diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index df409fa36..c5eb56f0e 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1525,7 +1525,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) int err; char const * const errmsg = BZ2_bzerror(d->bz2, &err); if (err != BZ_IO_ERROR) - return FileFdError("BZ2_bzread: %s (%d: %s)", _("Read error"), err, errmsg); + return FileFdError("BZ2_bzread: %s %s (%d: %s)", FileName.c_str(), _("Read error"), err, errmsg); } #endif #ifdef HAVE_LZMA -- cgit v1.2.3 From c045cc0268d70eb3b7c41076ade7381f73e740f2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 10:43:00 +0200 Subject: refactor --- apt-pkg/acquire-item.cc | 21 +++++++++++---------- apt-pkg/acquire-item.h | 49 ++++++++++++++++++++++--------------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d6b00f9b7..bfa0eb4fc 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1470,10 +1470,10 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaBase(Owner, HashStringList(), TransactionManager), RealURI(URI), - MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), - URIDesc(URIDesc), ShortDesc(ShortDesc), - IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) + pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, + HashStringList(), TransactionManager), + RealURI(URI), MetaIndexFile(MetaIndexFile), URIDesc(URIDesc), + ShortDesc(ShortDesc), AuthPass(false), IMSHit(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -1577,7 +1577,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList { // load indexes and queue further downloads MetaIndexParser->Load(MetaIndexFile); - ((pkgAcqMetaIndex*)TransactionManager)->QueueIndexes(true); + QueueIndexes(true); } Complete = true; @@ -1622,7 +1622,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him MetaIndexParser->Load(MetaIndexFile); - ((pkgAcqMetaIndex*)TransactionManager)->QueueIndexes(true); + QueueIndexes(true); } else { @@ -1648,9 +1648,10 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaBase(Owner, HashStringList(), TransactionManager), RealURI(URI), IndexTargets(IndexTargets), - URIDesc(URIDesc), ShortDesc(ShortDesc), - MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), + pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, HashStringList(), + TransactionManager), + RealURI(URI), URIDesc(URIDesc), ShortDesc(ShortDesc), + AuthPass(false), IMSHit(false), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) { @@ -1881,7 +1882,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ #endif } /*}}}*/ -void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ +void pkgAcqMetaBase::QueueIndexes(bool verify) /*{{{*/ { bool transInRelease = false; { diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index e6a22ce7b..3c81f77a9 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -356,6 +356,24 @@ class pkgAcqMetaBase : public pkgAcquire::Item protected: std::vector Transaction; + /** \brief A package-system-specific parser for the meta-index file. */ + indexRecords *MetaIndexParser; + + /** \brief The index files which should be looked up in the meta-index + * and then downloaded. + */ + const std::vector* IndexTargets; + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected hashsum + * can be determined from the meta-index will be downloaded, and + * the hashsums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no hashsum checking will be performed. + */ + void QueueIndexes(bool verify); + public: // transaction code void Add(Item *I); @@ -369,9 +387,12 @@ class pkgAcqMetaBase : public pkgAcquire::Item pkgAcqMetaBase(pkgAcquire *Owner, + const std::vector* IndexTargets, + indexRecords* MetaIndexParser, HashStringList const &ExpectedHashes=HashStringList(), pkgAcqMetaBase *TransactionManager=NULL) - : Item(Owner, ExpectedHashes, TransactionManager) {}; + : Item(Owner, ExpectedHashes, TransactionManager), + MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) {}; }; /** \brief An acquire item that downloads the detached signature {{{ @@ -397,19 +418,9 @@ class pkgAcqMetaSig : public pkgAcqMetaBase std::string URIDesc; std::string ShortDesc; - /** \brief A package-system-specific parser for the meta-index file. */ - indexRecords* MetaIndexParser; - /** \brief The file we need to verify */ std::string MetaIndexFile; - /** \brief The index files which should be looked up in the meta-index - * and then downloaded. - * - * \todo Why a list of pointers instead of a list of structs? - */ - const std::vector* IndexTargets; - /** \brief If we are in fetching or download state */ bool AuthPass; @@ -463,12 +474,6 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase */ std::string SigFile; - /** \brief The index files to download. */ - const std::vector* IndexTargets; - - /** \brief The parser for the meta-index file. */ - indexRecords* MetaIndexParser; - /** \brief If \b true, the index's signature is currently being verified. */ bool AuthPass; @@ -521,16 +526,6 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase public: - /** \brief Starts downloading the individual index files. - * - * \param verify If \b true, only indices whose expected hashsum - * can be determined from the meta-index will be downloaded, and - * the hashsums of indices will be checked (reporting - * #StatAuthError if there is a mismatch). If verify is \b false, - * no hashsum checking will be performed. - */ - void QueueIndexes(bool verify); - // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, -- cgit v1.2.3 From e1bd768b762bd74221f9089133883723a7307f9b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 11:03:02 +0200 Subject: test fixes --- test/integration/test-apt-update-rollback | 2 +- test/integration/test-apt-update-unauth | 2 ++ .../test-bug-617690-allow-unauthenticated-makes-all-untrusted | 5 ++++- test/integration/test-bug-728500-tempdir | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index a88b0042b..e37be9554 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -114,7 +114,7 @@ test_unauthenticated_to_invalid_inrelease() { rm $APTARCHIVE/dists/unstable/Release.gpg avoid_ims_hit - testsuccess aptget update -qq + testsuccess aptget update -qq --allow-unauthenticated testequal "WARNING: The following packages cannot be authenticated! old E: There are problems and -y was used without --force-yes" aptget install -qq -y old diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index 2e46e3ace..5db8a3c16 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -8,6 +8,8 @@ set -e TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework +umask 022 + setupenvironment configarchitecture "i386" diff --git a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted index f93510fd7..276e10564 100755 --- a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted +++ b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted @@ -24,15 +24,18 @@ testfilemissing() { testrun() { rm -rf rootdir/var/lib/apt - testsuccess aptget update if [ "$1" = 'trusted' ]; then + testsuccess aptget update + testsuccess aptget download cool testfileexists 'cool_1.0_i386.deb' testsuccess aptget download cool --allow-unauthenticated testfileexists 'cool_1.0_i386.deb' else + testsuccess aptget update --allow-unauthenticated + testfailure aptget download cool testfilemissing 'cool_1.0_i386.deb' diff --git a/test/integration/test-bug-728500-tempdir b/test/integration/test-bug-728500-tempdir index 0451fc1ed..e9df0a709 100755 --- a/test/integration/test-bug-728500-tempdir +++ b/test/integration/test-bug-728500-tempdir @@ -17,7 +17,7 @@ msgtest 'Test apt-get update with incorrect' 'TMPDIR' OUTPUT=$(mktemp) addtrap "rm $OUTPUT;" export TMPDIR=/does-not-exists -if aptget update >${OUTPUT} 2>&1; then +if aptget update -o Debug::Acquire::gpg=1 >${OUTPUT} 2>&1; then msgpass else echo -- cgit v1.2.3 From ed6fa9754dba590e92604e09013c1478ecd46443 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 11:26:30 +0200 Subject: fix DestFile ext --- apt-pkg/acquire-item.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index bfa0eb4fc..3d6924847 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1112,7 +1112,7 @@ void pkgAcqIndex::ReverifyAfterIMS() // adjust DestFile if its compressed on disk if (_config->FindB("Acquire::GzipIndexes",false) == true) - DestFile += compExt; + DestFile += '.' + compExt; // copy FinalFile into partial/ so that we check the hash again string FinalFile = GetFinalFilename(); -- cgit v1.2.3 From c292cc32b3ab0d70c63e68f7c94446932217c0ec Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 11:47:03 +0200 Subject: more test fixes --- test/integration/test-bug-596498-trusted-unsigned-repo | 2 +- test/integration/test-hashsum-verification | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/integration/test-bug-596498-trusted-unsigned-repo b/test/integration/test-bug-596498-trusted-unsigned-repo index 06c9c8285..973520a97 100755 --- a/test/integration/test-bug-596498-trusted-unsigned-repo +++ b/test/integration/test-bug-596498-trusted-unsigned-repo @@ -12,7 +12,7 @@ setupaptarchive aptgetupdate() { rm -rf rootdir/var/lib/apt/ rootdir/var/cache/apt/*.bin - aptget update -qq + aptget update -qq --allow-unauthenticated } PKGTEXT="$(aptget install cool --assume-no -d | head -n 7)" diff --git a/test/integration/test-hashsum-verification b/test/integration/test-hashsum-verification index 2a400dcb4..2db2bab0f 100755 --- a/test/integration/test-hashsum-verification +++ b/test/integration/test-hashsum-verification @@ -70,9 +70,13 @@ runtest() { rm -rf rootdir/var/lib/apt/lists rm aptarchive/InRelease aptarchive/Release.gpg msgtest 'unsigned apt-get update gets the expected hashsum mismatch' - aptget update 2>&1 | grep "Hash Sum mismatch" > /dev/null && msgpass || msgfail - - + aptget update --allow-unauthenticated >output.log 2>&1 || true + if grep -q "Hash Sum mismatch" output.log; then + msgpass + else + cat output.log + msgfail + fi } for COMPRESSEDINDEXES in 'false' 'true'; do -- cgit v1.2.3 From fa3a96a1051d65e5aa8cd9e9e0bb854ccc67a4b6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 17:38:23 +0200 Subject: cleanup --- apt-pkg/acquire-item.cc | 49 ++++++++++++++++++++++++++++++++----------------- apt-pkg/acquire-item.h | 36 ++++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 3d6924847..5beb7e190 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -627,7 +627,9 @@ void pkgAcqIndexDiffs::Finish(bool allDone) // this happens if we have a up-to-date indexfile if(!FileExists(PartialFile)) PartialFile = DestFile; - + + TransactionManager->TransactionStageCopy(this, PartialFile, DestFile); + // this is for the "real" finish Complete = true; Status = StatDone; @@ -892,8 +894,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri << DestFile << " -> " << FinalFile << std::endl; // queue for copy by the transaction manager - PartialFile = DestFile; - DestFile = FinalFile; + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); // ensure the ed's are gone regardless of list-cleanup for (std::vector::const_iterator I = allPatches->begin(); @@ -1178,8 +1179,7 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, unlink(CompressedFile.c_str()); // Done, queue for rename on transaction finished - PartialFile = DestFile; - DestFile = GetFinalFilename(); + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); return; } @@ -1428,6 +1428,22 @@ void pkgAcqMetaBase::CommitTransaction() } } +void pkgAcqMetaBase::TransactionStageCopy(Item *I, + const std::string &From, + const std::string &To) +{ + I->PartialFile = From; + I->DestFile = To; +} + +void pkgAcqMetaBase::TransactionStageRemoval(Item *I, + const std::string &FinalFile) +{ + I->PartialFile = ""; + I->DestFile = FinalFile; +} + + /*{{{*/ bool pkgAcqMetaBase::GenerateAuthWarning(const std::string &RealURI, const std::string &Message) @@ -1473,7 +1489,7 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, HashStringList(), TransactionManager), RealURI(URI), MetaIndexFile(MetaIndexFile), URIDesc(URIDesc), - ShortDesc(ShortDesc), AuthPass(false), IMSHit(false) + ShortDesc(ShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -1547,7 +1563,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); - DestFile = PartialFile = FinalFile; + TransactionManager->TransactionStageCopy(this, FinalFile, FinalFile); } // queue for verify @@ -1567,8 +1583,10 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; PartialFile += URItoFileName(RealURI); - DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI); + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + TransactionManager->TransactionStageCopy(this, PartialFile, FinalFile); } // we parse the MetaIndexFile here because at this point we can @@ -1606,7 +1624,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // transaction DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - PartialFile = ""; + TransactionManager->TransactionStageRemoval(this, DestFile); // FIXME: duplicated code from pkgAcqMetaIndex if (AuthPass == true) @@ -1651,7 +1669,6 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, HashStringList(), TransactionManager), RealURI(URI), URIDesc(URIDesc), ShortDesc(ShortDesc), - AuthPass(false), IMSHit(false), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) { @@ -1752,9 +1769,9 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList FinalFile += URItoFileName(RealURI); if (SigFile == DestFile) SigFile = FinalFile; + // queue for copy in place - PartialFile = DestFile; - DestFile = FinalFile; + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); } } /*}}}*/ @@ -2077,8 +2094,7 @@ void pkgAcqMetaIndex::Failed(string Message, } // Done, queue for rename on transaction finished - PartialFile = DestFile; - DestFile = FinalFile; + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); } _error->Warning(_("The data from '%s' is not signed. Packages " @@ -2197,8 +2213,7 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* // impression (CVE-2012-0214) string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile.append(URItoFileName(RealURI)); - PartialFile = ""; - DestFile = FinalFile; + TransactionManager->TransactionStageRemoval(this, FinalFile); new pkgAcqMetaIndex(Owner, TransactionManager, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3c81f77a9..e560da956 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -364,6 +364,13 @@ class pkgAcqMetaBase : public pkgAcquire::Item */ const std::vector* IndexTargets; + /** \brief If \b true, the index's signature is currently being verified. + */ + bool AuthPass; + + // required to deal gracefully with problems caused by incorrect ims hits + bool IMSHit; + /** \brief Starts downloading the individual index files. * * \param verify If \b true, only indices whose expected hashsum @@ -381,6 +388,15 @@ class pkgAcqMetaBase : public pkgAcquire::Item bool TransactionHasError() APT_PURE; void CommitTransaction(); + /** \brief Stage (queue) a copy action when the transaction is commited + */ + void TransactionStageCopy(Item *I, + const std::string &From, + const std::string &To); + /** \brief Stage (queue) a removal action when the transaction is commited + */ + void TransactionStageRemoval(Item *I, const std::string &FinalFile); + // helper for the signature warning bool GenerateAuthWarning(const std::string &RealURI, const std::string &Message); @@ -392,7 +408,8 @@ class pkgAcqMetaBase : public pkgAcquire::Item HashStringList const &ExpectedHashes=HashStringList(), pkgAcqMetaBase *TransactionManager=NULL) : Item(Owner, ExpectedHashes, TransactionManager), - MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) {}; + MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets), + AuthPass(false), IMSHit(false) {}; }; /** \brief An acquire item that downloads the detached signature {{{ @@ -415,17 +432,14 @@ class pkgAcqMetaSig : public pkgAcqMetaBase */ std::string RealURI; - std::string URIDesc; - std::string ShortDesc; - /** \brief The file we need to verify */ std::string MetaIndexFile; - /** \brief If we are in fetching or download state */ - bool AuthPass; + /** \brief Long URI description used in the acquire system */ + std::string URIDesc; - /** \brief Was this file already on disk */ - bool IMSHit; + /** \brief Short URI description used in the acquire system */ + std::string ShortDesc; public: @@ -474,12 +488,6 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase */ std::string SigFile; - /** \brief If \b true, the index's signature is currently being verified. - */ - bool AuthPass; - // required to deal gracefully with problems caused by incorrect ims hits - bool IMSHit; - /** \brief Check that the release file is a release file for the * correct distribution. * -- cgit v1.2.3 From 8d266656767f6c7c3946700c7052d0b8b6212742 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 1 Oct 2014 11:20:08 +0200 Subject: add verify for the .diff/Index download and add FIXME for pkgAcqIndexDiffs/pkgAcqMergeDiffs --- apt-pkg/acquire-item.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 5beb7e190..4a684f40c 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -527,6 +527,21 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList Item::Done(Message, Size, Hashes, Cnf); + // verify the index target + if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.size() > 0) + { + std::string IndexMetaKey = Target->MetaKey + ".diff/Index"; + indexRecords::checkSum *Record = MetaIndexParser->Lookup(IndexMetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) + { + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, Record->Hashes, Hashes); + Failed(Message, Cnf); + return; + } + + } + string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@ -715,6 +730,8 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi Item::Done(Message, Size, Hashes, Cnf); + // FIXME: verify this download too before feeding it to rred + string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); @@ -836,6 +853,8 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Item::Done(Message,Size,Hashes,Cnf); + // FIXME: verify download before feeding it to rred + string const FinalFile = _config->FindDir("Dir::State::lists") + "partial/" + URItoFileName(RealURI); if (State == StateFetchDiff) -- cgit v1.2.3 From c99fe2e169243fc6e1a3278ce3768f0f521e260b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 1 Oct 2014 12:21:55 +0200 Subject: Use Acquire::Allow{InsecureRepositories,DowngradeToInsecureRepositories} The configuration key Acquire::AllowInsecureRepositories controls if apt allows loading of unsigned repositories at all. The configuration Acquire::AllowDowngradeToInsecureRepositories controls if a signed repository can ever become unsigned. This should really never be needed but we provide it to avoid having to mess around in /var/lib/apt/lists if there is a use-case for this (which I can't think of right now). --- apt-pkg/acquire-item.cc | 35 +++++++++++++++------- apt-pkg/init.cc | 4 +++ apt-private/private-cmndline.cc | 1 + doc/examples/configure-index | 2 +- test/integration/test-apt-get-source-authenticated | 2 +- .../integration/test-apt-get-update-unauth-warning | 4 +-- test/integration/test-apt-update-nofallback | 23 ++++++++++++++ test/integration/test-apt-update-rollback | 2 +- test/integration/test-apt-update-unauth | 2 +- .../test-bug-596498-trusted-unsigned-repo | 2 +- ...17690-allow-unauthenticated-makes-all-untrusted | 4 +-- .../test-bug-717891-abolute-uris-for-proxies | 2 +- test/integration/test-bug-738785-switch-protocol | 2 +- test/integration/test-hashsum-verification | 2 +- test/integration/test-policy-pinning | 2 +- 15 files changed, 65 insertions(+), 24 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 4a684f40c..93ba098ee 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1629,14 +1629,27 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ "InRelease"); string FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI); - if(RealFileExists(Final) || RealFileExists(FinalInRelease)) + if (RealFileExists(Final) || RealFileExists(FinalInRelease)) { - _error->Error("The repository '%s' is no longer signed.", - URIDesc.c_str()); - Rename(MetaIndexFile, MetaIndexFile+".FAILED"); - Status = pkgAcquire::Item::StatError; - TransactionManager->AbortTransaction(); - return; + std::string downgrade_msg; + strprintf(downgrade_msg, _("The repository '%s' is no longer signed."), + URIDesc.c_str()); + if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories")) + { + // meh, the users wants to take risks (we still mark the packages + // from this repository as unauthenticated) + _error->Warning("%s", downgrade_msg.c_str()); + _error->Warning(_("This is normally not allowed, but the option " + "Acquire::AllowDowngradeToInsecureRepositories was " + "given to override it.")); + + } else { + _error->Error("%s", downgrade_msg.c_str()); + Rename(MetaIndexFile, MetaIndexFile+".FAILED"); + Status = pkgAcquire::Item::StatError; + TransactionManager->AbortTransaction(); + return; + } } // this ensures that any file in the lists/ dir is removed by the @@ -1654,7 +1667,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } // only allow going further if the users explicitely wants it - if(_config->FindB("APT::Get::AllowUnauthenticated", false) == true) + if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him @@ -1663,7 +1676,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } else { - _error->Warning("Use --allow-unauthenticated to force the update"); + _error->Warning("Use --allow-insecure-repositories to force the update"); } // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor @@ -2123,12 +2136,12 @@ void pkgAcqMetaIndex::Failed(string Message, // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification // only allow going further if the users explicitely wants it - if(_config->FindB("APT::Get::AllowUnauthenticated", false) == true) + if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { QueueIndexes(false); } else { // warn if the repository is unsinged - _error->Warning("Use --allow-unauthenticated to force the update"); + _error->Warning("Use --allow-insecure-repositories to force the update"); } } /*}}}*/ diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 241628632..82dff4ee8 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -88,6 +88,10 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.Set("Dir::Ignore-Files-Silently::", "\\.orig$"); Cnf.Set("Dir::Ignore-Files-Silently::", "\\.distUpgrade$"); + // Repository security + Cnf.CndSet("Acquire::AllowInsecureRepositories", false); + Cnf.CndSet("Acquire::AllowDowngradeToInsecureRepositories", false); + // Default cdrom mount point Cnf.CndSet("Acquire::cdrom::mount", "/media/cdrom/"); diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc index a4490f5b4..079f81ee3 100644 --- a/apt-private/private-cmndline.cc +++ b/apt-private/private-cmndline.cc @@ -198,6 +198,7 @@ static bool addArgumentsAPTGet(std::vector &Args, char const addArg(0,"only-source","APT::Get::Only-Source",0); addArg(0,"arch-only","APT::Get::Arch-Only",0); addArg(0,"allow-unauthenticated","APT::Get::AllowUnauthenticated",0); + addArg(0,"allow-insecure-repositories","Acquire::AllowInsecureRepositories",0); addArg(0,"install-recommends","APT::Install-Recommends",CommandLine::Boolean); addArg(0,"install-suggests","APT::Install-Suggests",CommandLine::Boolean); addArg(0,"fix-policy","APT::Get::Fix-Policy-Broken",0); diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 2d9f829ba..2925733d7 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -34,7 +34,7 @@ APT { Host-Architecture "armel"; Arch-Only "false"; - AllowUnauthenticated "false"; + AllowUnauthenticated "false"; // packages from unauthenticated AutomaticRemove "false"; HideAutoRemove "false"; Download-Only "false"; diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated index d73097b54..d833ddd85 100755 --- a/test/integration/test-apt-get-source-authenticated +++ b/test/integration/test-apt-get-source-authenticated @@ -21,7 +21,7 @@ APTARCHIVE=$(readlink -f ./aptarchive) rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated InRelease file -testsuccess aptget update --allow-unauthenticated +testsuccess aptget update --allow-insecure-repositories # this all should fail testfailure aptget install -y foo diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index b1c676738..510249747 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -23,7 +23,7 @@ testequal "Ign file: unstable InRelease Ign file: unstable Release Reading package lists... W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated. -W: Use --allow-unauthenticated to force the update" aptget update +W: Use --allow-insecure-repositories to force the update" aptget update # no package foo testequal "Listing..." apt list foo @@ -32,7 +32,7 @@ testequal "Listing..." apt list foo testequal "Ign file: unstable InRelease Ign file: unstable Release Reading package lists... -W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated." aptget update --allow-unauthenticated +W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated." aptget update --allow-insecure-repositories # ensure we can not install the package testequal "WARNING: The following packages cannot be authenticated! diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback index 4e8ea9916..a53226e18 100755 --- a/test/integration/test-apt-update-nofallback +++ b/test/integration/test-apt-update-nofallback @@ -78,6 +78,25 @@ test_from_release_gpg_to_unsigned() assert_update_is_refused_and_last_good_state_used } +test_from_inrelease_to_unsigned_with_override() +{ + # setup archive with InRelease file + setupaptarchive_with_lists_clean + testsuccess aptget update + + # simulate moving to a unsigned but otherwise valid repo + simulate_mitm_and_inject_evil_package + generatereleasefiles + + # and ensure we can update to it (with enough force) + testsuccess aptget update --allow-insecure-repositories \ + -o Acquire::AllowDowngradeToInsecureRepositories=1 + # but that the individual packages are still considered untrusted + testequal "WARNING: The following packages cannot be authenticated! + evil +E: There are problems and -y was used without --force-yes" aptget install -qq -y evil +} + test_cve_2012_0214() { # see https://bugs.launchpad.net/ubuntu/+source/apt/+bug/947108 @@ -205,3 +224,7 @@ test_inrelease_to_invalid_inrelease # ensure we revert to last good state if Release/Release.gpg does not verify msgmsg "test_release_gpg_to_invalid_release_release_gpg" test_release_gpg_to_invalid_release_release_gpg + +# ensure we can ovveride the downgrade error +msgmsg "test_from_inrelease_to_unsigned" +test_from_inrelease_to_unsigned_with_override diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index e37be9554..ee8bc6926 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -114,7 +114,7 @@ test_unauthenticated_to_invalid_inrelease() { rm $APTARCHIVE/dists/unstable/Release.gpg avoid_ims_hit - testsuccess aptget update -qq --allow-unauthenticated + testsuccess aptget update -qq --allow-insecure-repositories testequal "WARNING: The following packages cannot be authenticated! old E: There are problems and -y was used without --force-yes" aptget install -qq -y old diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index 5db8a3c16..ade523ea7 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -28,7 +28,7 @@ runtest() { rm -f aptarchive/dists/unstable/*Release* # remove uncompressed version find aptarchive/ -name Packages | xargs rm -f - aptget update -qq --allow-unauthenticated + aptget update -qq --allow-insecure-repositories # become authenticated generatereleasefiles diff --git a/test/integration/test-bug-596498-trusted-unsigned-repo b/test/integration/test-bug-596498-trusted-unsigned-repo index 973520a97..3104a70c2 100755 --- a/test/integration/test-bug-596498-trusted-unsigned-repo +++ b/test/integration/test-bug-596498-trusted-unsigned-repo @@ -12,7 +12,7 @@ setupaptarchive aptgetupdate() { rm -rf rootdir/var/lib/apt/ rootdir/var/cache/apt/*.bin - aptget update -qq --allow-unauthenticated + aptget update -qq --allow-insecure-repositories } PKGTEXT="$(aptget install cool --assume-no -d | head -n 7)" diff --git a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted index 276e10564..0736bb6dc 100755 --- a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted +++ b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted @@ -26,7 +26,7 @@ testrun() { rm -rf rootdir/var/lib/apt if [ "$1" = 'trusted' ]; then - testsuccess aptget update + testsuccess aptget update testsuccess aptget download cool testfileexists 'cool_1.0_i386.deb' @@ -34,7 +34,7 @@ testrun() { testsuccess aptget download cool --allow-unauthenticated testfileexists 'cool_1.0_i386.deb' else - testsuccess aptget update --allow-unauthenticated + testsuccess aptget update --allow-insecure-repositories testfailure aptget download cool testfilemissing 'cool_1.0_i386.deb' diff --git a/test/integration/test-bug-717891-abolute-uris-for-proxies b/test/integration/test-bug-717891-abolute-uris-for-proxies index a8947b5e2..54a616686 100755 --- a/test/integration/test-bug-717891-abolute-uris-for-proxies +++ b/test/integration/test-bug-717891-abolute-uris-for-proxies @@ -12,7 +12,7 @@ setupaptarchive changetowebserver --request-absolute='uri' msgtest 'Check that absolute paths are' 'not accepted' -testfailure --nomsg aptget update --allow-unauthenticated +testfailure --nomsg aptget update --allow-insecure-repositories echo 'Acquire::http::Proxy "http://localhost:8080";' > rootdir/etc/apt/apt.conf.d/99proxy diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol index 4ff044515..f81bba4b9 100755 --- a/test/integration/test-bug-738785-switch-protocol +++ b/test/integration/test-bug-738785-switch-protocol @@ -60,4 +60,4 @@ mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS} # check that downgrades from https to http are not allowed webserverconfig 'aptwebserver::support::http' 'true' sed -i -e 's#:8080/redirectme#:4433/downgrademe#' -e 's# http:# https:#' rootdir/etc/apt/sources.list.d/* -testfailure aptget update --allow-unauthenticated +testfailure aptget update --allow-insecure-repositories diff --git a/test/integration/test-hashsum-verification b/test/integration/test-hashsum-verification index 2db2bab0f..5f88110b3 100755 --- a/test/integration/test-hashsum-verification +++ b/test/integration/test-hashsum-verification @@ -70,7 +70,7 @@ runtest() { rm -rf rootdir/var/lib/apt/lists rm aptarchive/InRelease aptarchive/Release.gpg msgtest 'unsigned apt-get update gets the expected hashsum mismatch' - aptget update --allow-unauthenticated >output.log 2>&1 || true + aptget update --allow-insecure-repositories >output.log 2>&1 || true if grep -q "Hash Sum mismatch" output.log; then msgpass else diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning index 2281d7a1d..c08a2f103 100755 --- a/test/integration/test-policy-pinning +++ b/test/integration/test-policy-pinning @@ -28,7 +28,7 @@ Pinned packages:" aptcache policy $* aptgetupdate() { # just to be sure that no old files are used rm -rf rootdir/var/lib/apt - if aptget update --allow-unauthenticated -qq 2>&1 | grep '^E: '; then + if aptget update --allow-insecure-repositories -qq 2>&1 | grep '^E: '; then msgwarn 'apt-get update failed with an error' fi } -- cgit v1.2.3 From e8b1db38cca29cbdc0116e567f0aa7a28034287b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 1 Oct 2014 14:06:01 +0200 Subject: update test/integration/test-releasefile-verification --- apt-pkg/acquire-item.cc | 18 +++++++++--------- test/integration/test-releasefile-verification | 13 ++++++++++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 93ba098ee..4ab4ef6a1 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1623,7 +1623,15 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - + + // FIXME: duplicated code from pkgAcqMetaIndex + if (AuthPass == true) + { + bool Stop = GenerateAuthWarning(RealURI, Message); + if(Stop) + return; + } + // FIXME: meh, this is not really elegant string InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12, "InRelease"); @@ -1658,14 +1666,6 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ DestFile += URItoFileName(RealURI); TransactionManager->TransactionStageRemoval(this, DestFile); - // FIXME: duplicated code from pkgAcqMetaIndex - if (AuthPass == true) - { - bool Stop = GenerateAuthWarning(RealURI, Message); - if(Stop) - return; - } - // only allow going further if the users explicitely wants it if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification index e558b83e8..3765a4b1f 100755 --- a/test/integration/test-releasefile-verification +++ b/test/integration/test-releasefile-verification @@ -235,10 +235,21 @@ runtest2() { " aptcache show apt failaptnew } -runtest2 +# diable some protection by default and ensure we still do the verification +# correctly +cat > rootdir/etc/apt/apt.conf.d/weaken-security < Date: Wed, 1 Oct 2014 14:22:46 +0200 Subject: fix test-apt-update-nofallback test --- test/integration/test-apt-update-nofallback | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback index a53226e18..c400dcc36 100755 --- a/test/integration/test-apt-update-nofallback +++ b/test/integration/test-apt-update-nofallback @@ -181,7 +181,11 @@ test_release_gpg_to_invalid_release_release_gpg() echo "Some evil data" >> $APTARCHIVE/dists/unstable/Release inject_evil_package - testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq + testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) + +W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg + +W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq assert_repo_is_intact testfailure grep "evil" rootdir/var/lib/apt/lists/*Release -- cgit v1.2.3 From 0b844e23f014bd3ce95e27fe5fa81138e9ae4879 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 1 Oct 2014 17:13:33 +0200 Subject: hack around test-apt-update-unauth failure --- apt-pkg/acquire-item.cc | 9 +++++++++ test/integration/test-apt-update-unauth | 21 +++++++++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 4ab4ef6a1..fd2ea08f8 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1192,6 +1192,8 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, // FIXME: can we void the "Erase" bool here as its very non-local? std::string CompressedFile = _config->FindDir("Dir::State::lists") + "partial/"; CompressedFile += URItoFileName(RealURI); + if(_config->FindB("Acquire::GzipIndexes",false) == false) + CompressedFile += '.' + compExt; // Remove the compressed version. if (Erase == true) @@ -1399,6 +1401,13 @@ void pkgAcqMetaBase::AbortTransaction() // the transaction will abort, so stop anything that is idle if ((*I)->Status == pkgAcquire::Item::StatIdle) (*I)->Status = pkgAcquire::Item::StatDone; + + // kill files in partial + string PartialFile = _config->FindDir("Dir::State::lists"); + PartialFile += "partial/"; + PartialFile += flNotDir((*I)->DestFile); + if(FileExists(PartialFile)) + Rename(PartialFile, PartialFile + ".FAILED"); } } /*}}}*/ diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index ade523ea7..cf5195024 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -26,34 +26,43 @@ runtest() { # start unauthenticated find rootdir/var/lib/apt/lists/ -type f | xargs rm -f rm -f aptarchive/dists/unstable/*Release* - # remove uncompressed version - find aptarchive/ -name Packages | xargs rm -f + aptget update -qq --allow-insecure-repositories + # FIXME: this really shouldn't be needed + rm -f rootdir/var/lib/apt/lists/partial/* + # become authenticated generatereleasefiles signreleasefiles + # move uncompressed away + mv aptarchive/dists/unstable/main/binary-i386/Packages \ + aptarchive/dists/unstable/main/binary-i386/Packages.uncompressed + # and ensure we re-check the downloaded data msgtest "Check rollback on going from unauth -> auth" # change the local packages file PKGS=$(ls rootdir/var/lib/apt/lists/*Packages*) echo "meep" > $PKGS - ls -l rootdir/var/lib/apt/lists > lists.before + ls rootdir/var/lib/apt/lists/ > lists.before # update and ensure all is reverted on the hashsum failure - aptget update -o Debug::Acquire::Transaction=1 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 > output.log 2>&1 || true + aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0 > output.log 2>&1 || true # ensure we have before what we have after - ls -l rootdir/var/lib/apt/lists > lists.after + ls rootdir/var/lib/apt/lists/ > lists.after if diff -u lists.before lists.after; then msgpass else - #cat output.log + cat output.log msgfail fi + # move uncompressed back for release file + mv aptarchive/dists/unstable/main/binary-i386/Packages.uncompressed \ + aptarchive/dists/unstable/main/binary-i386/Packages } for COMPRESSEDINDEXES in 'false' 'true'; do -- cgit v1.2.3 From 47450dea0904298c8d5ea06b15ea26368da5a4ee Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 1 Oct 2014 18:01:14 +0200 Subject: fix leftover files from Acquire::GzipIndex --- apt-pkg/acquire-item.cc | 2 +- test/integration/test-apt-update-ims | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index fd2ea08f8..dbc1dfbe7 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1283,7 +1283,7 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, // If we enable compressed indexes, queue for hash verification if (_config->FindB("Acquire::GzipIndexes",false)) { - DestFile = _config->FindDir("Dir::State::lists"); + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI) + '.' + compExt; Decompression = true; diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 3bd6e843c..946dfc7af 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -11,15 +11,30 @@ buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable' setupaptarchive changetowebserver -testsuccess aptget update +runtest() { + rm -f rootdir/var/lib/apt/lists/localhost* -# check that I-M-S header is kept in redirections -testequal "Hit http://localhost:8080 unstable InRelease + testsuccess aptget update + + # ensure no leftovers in partial + testfailure ls "rootdir/var/lib/apt/lists/partial/*" + + # check that I-M-S header is kept in redirections + testequal "Hit http://localhost:8080 unstable InRelease Hit http://localhost:8080 unstable/main Sources Hit http://localhost:8080 unstable/main amd64 Packages Hit http://localhost:8080 unstable/main Translation-en -Reading package lists..." aptget update +Reading package lists..." aptget update -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0 + + # ensure that we still do a hash check on ims hit + msgtest 'Test I-M-S reverify' + aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail + + # ensure no leftovers in partial + testfailure ls "rootdir/var/lib/apt/lists/partial/*" +} + +runtest -# ensure that we still do a hash check on ims hit -msgtest 'Test I-M-S reverify' -aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail +echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex +runtest -- cgit v1.2.3 From 899e4deda1d0b2e9ad405f7f0b44c5b9be05fce7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 1 Oct 2014 22:41:52 +0200 Subject: refactor and add pkgAcqIndex::ValidateFile() --- apt-pkg/acquire-item.cc | 55 ++++++++++++++++++++++++++++++------------------- apt-pkg/acquire-item.h | 9 ++++---- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index dbc1dfbe7..675a7ed41 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1141,6 +1141,35 @@ void pkgAcqIndex::ReverifyAfterIMS() QueueURI(Desc); } /*}}}*/ + +// pkgAcqIndex::ValidateFile - Validate the downloaded file /*{{{*/ +// --------------------------------------------------------------------- +bool pkgAcqIndex::ValidateFile(const std::string &FileName) +{ + // FIXME: this can go away once we only ever download stuff that + // has a valid hash and we never do GET based probing + // FIXME2: this also leaks debian-isms into the code and should go therefore + + /* Always validate the index file for correctness (all indexes must + * have a Package field) (LP: #346386) (Closes: #627642) + */ + FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); + // Only test for correctness if the content of the file is not empty + // (empty is ok) + if (fd.Size() > 0) + { + pkgTagSection sec; + pkgTagFile tag(&fd); + + // all our current indexes have a field 'Package' in each section + if (_error->PendingError() == true || + tag.Step(sec) == false || + sec.Exists("Package") == false) + return false; + } + return true; +} + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -1166,29 +1195,13 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, return; } - // FIXME: this can go away once we only ever download stuff that - // has a valid hash and we never do GET based probing - // - /* Always verify the index file for correctness (all indexes must - * have a Package field) (LP: #346386) (Closes: #627642) - */ - FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); - // Only test for correctness if the content of the file is not empty - // (empty is ok) - if (fd.Size() > 0) + if(!ValidateFile(DestFile)) { - pkgTagSection sec; - pkgTagFile tag(&fd); - - // all our current indexes have a field 'Package' in each section - if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false) - { - RenameOnError(InvalidFormat); - Failed(Message, Cfg); - return; - } + RenameOnError(InvalidFormat); + Failed(Message, Cfg); + return; } - + // FIXME: can we void the "Erase" bool here as its very non-local? std::string CompressedFile = _config->FindDir("Dir::State::lists") + "partial/"; CompressedFile += URItoFileName(RealURI); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index e560da956..30a8850e4 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -283,7 +283,6 @@ class pkgAcquire::Item : public WeakPointable /** \return \b true if this object is being fetched from a trusted source. */ virtual bool IsTrusted() const {return false;}; - // report mirror problems /** \brief Report mirror problem * * This allows reporting mirror failures back to a centralized @@ -293,7 +292,6 @@ class pkgAcquire::Item : public WeakPointable */ void ReportMirrorFailure(std::string FailCode); - /** \brief Initialize an item. * * Adds the item to the list of items known to the acquire @@ -947,7 +945,6 @@ class pkgAcqIndex : public pkgAcqBaseIndex */ std::string CompressionExtension; - /** \brief Do the changes needed to fetch via AptByHash (if needed) */ void InitByHashIfNeeded(const std::string MetaKey); @@ -961,11 +958,15 @@ class pkgAcqIndex : public pkgAcqBaseIndex /** \brief Schedule file for verification after a IMS hit */ void ReverifyAfterIMS(); + /** \brief Validate the downloaded index file */ + bool ValidateFile(const std::string &FileName); + public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + virtual void Done(std::string Message,unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); virtual std::string Custom600Headers() const; virtual std::string DescURI() {return Desc.URI;}; -- cgit v1.2.3 From 651bddadd0558a3912833c7d51bb20625b318af9 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 2 Oct 2014 00:05:44 +0200 Subject: Cleanup pkgAcqIndex --- apt-pkg/acquire-item.cc | 248 +++++++++++++++++++++++++++--------------------- apt-pkg/acquire-item.h | 58 ++++++----- 2 files changed, 171 insertions(+), 135 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 675a7ed41..b3c41a178 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -933,6 +933,23 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri } } /*}}}*/ + +// AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/ +bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) +{ + if(MetaKey != "" && Hashes.size() > 0) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) + { + printHashSumComparision(RealURI, Record->Hashes, Hashes); + return false; + } + } + return true; +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is @@ -940,8 +957,10 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashStringList const &ExpectedHash) - : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI) + : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL) { + RealURI = URI; + AutoSelectCompression(); Init(URI, URIDesc, ShortDesc); @@ -958,8 +977,10 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, - MetaIndexParser), RealURI(Target->URI) + MetaIndexParser) { + RealURI = Target->URI; + // autoselect the compression method AutoSelectCompression(); Init(Target->URI, Target->Description, Target->ShortDesc); @@ -974,34 +995,38 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, void pkgAcqIndex::AutoSelectCompression() { std::vector types = APT::Configuration::getCompressionTypes(); - CompressionExtension = ""; + CompressionExtensions = ""; if (ExpectedHashes.usable()) { - for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) - CompressionExtension.append(*t).append(" "); + for (std::vector::const_iterator t = types.begin(); + t != types.end(); ++t) + { + std::string CompressedMetaKey = string(Target->MetaKey).append(".").append(*t); + if (*t == "uncompressed" || + MetaIndexParser->Exists(CompressedMetaKey) == true) + CompressionExtensions.append(*t).append(" "); + } } else { for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - CompressionExtension.append(*t).append(" "); + CompressionExtensions.append(*t).append(" "); } - if (CompressionExtension.empty() == false) - CompressionExtension.erase(CompressionExtension.end()-1); + if (CompressionExtensions.empty() == false) + CompressionExtensions.erase(CompressionExtensions.end()-1); } // AcqIndex::Init - defered Constructor /*{{{*/ // --------------------------------------------------------------------- void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { - Decompression = false; - Erase = false; + Stage = STAGE_DOWNLOAD; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (comprExt == "uncompressed") + ComprExt = CompressionExtensions.substr(0, CompressionExtensions.find(' ')); + if (ComprExt == "uncompressed") { Desc.URI = URI; if(Target) @@ -1009,10 +1034,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, } else { - Desc.URI = URI + '.' + comprExt; - DestFile = DestFile + '.' + comprExt; + Desc.URI = URI + '.' + ComprExt; + DestFile = DestFile + '.' + ComprExt; if(Target) - MetaKey = string(Target->MetaKey) + '.' + comprExt; + MetaKey = string(Target->MetaKey) + '.' + ComprExt; } // load the filesize @@ -1084,18 +1109,19 @@ string pkgAcqIndex::Custom600Headers() const /* */ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - size_t const nextExt = CompressionExtension.find(' '); + size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) { - CompressionExtension = CompressionExtension.substr(nextExt+1); + CompressionExtensions = CompressionExtensions.substr(nextExt+1); Init(RealURI, Desc.Description, Desc.ShortDesc); return; } // on decompression failure, remove bad versions in partial/ - if (Decompression && Erase) { + if (Stage == STAGE_DECOMPRESS_AND_VERIFY) + { string s = _config->FindDir("Dir::State::lists") + "partial/"; - s.append(URItoFileName(RealURI)); + s += URItoFileName(RealURI) + '.' + ComprExt; unlink(s.c_str()); } @@ -1110,11 +1136,10 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ /* */ std::string pkgAcqIndex::GetFinalFilename() const { - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); if (_config->FindB("Acquire::GzipIndexes",false) == true) - FinalFile += '.' + compExt; + FinalFile += '.' + ComprExt; return FinalFile; } /*}}}*/ @@ -1123,8 +1148,6 @@ std::string pkgAcqIndex::GetFinalFilename() const /* */ void pkgAcqIndex::ReverifyAfterIMS() { - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - // update destfile to *not* include the compression extension when doing // a reverify (as its uncompressed on disk already) DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -1132,18 +1155,18 @@ void pkgAcqIndex::ReverifyAfterIMS() // adjust DestFile if its compressed on disk if (_config->FindB("Acquire::GzipIndexes",false) == true) - DestFile += '.' + compExt; + DestFile += '.' + ComprExt; // copy FinalFile into partial/ so that we check the hash again string FinalFile = GetFinalFilename(); - Decompression = true; + Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FinalFile; QueueURI(Desc); } /*}}}*/ -// pkgAcqIndex::ValidateFile - Validate the downloaded file /*{{{*/ -// --------------------------------------------------------------------- +// AcqIndex::ValidateFile - Validate the content of the downloaded file /*{{{*/ +// -------------------------------------------------------------------------- bool pkgAcqIndex::ValidateFile(const std::string &FileName) { // FIXME: this can go away once we only ever download stuff that @@ -1153,7 +1176,7 @@ bool pkgAcqIndex::ValidateFile(const std::string &FileName) /* Always validate the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); + FileFd fd(FileName, FileFd::ReadOnly, FileFd::Extension); // Only test for correctness if the content of the file is not empty // (empty is ok) if (fd.Size() > 0) @@ -1177,69 +1200,45 @@ bool pkgAcqIndex::ValidateFile(const std::string &FileName) to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a compressed uri. */ -void pkgAcqIndex::Done(string Message, unsigned long long Size, +void pkgAcqIndex::Done(string Message, + unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hashes,Cfg); - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (Decompression == true) + switch(Stage) { - if (ExpectedHashes.usable() && ExpectedHashes != Hashes) - { - Desc.URI = RealURI; - RenameOnError(HashSumMismatch); - printHashSumComparision(RealURI, ExpectedHashes, Hashes); - Failed(Message, Cfg); - return; - } - - if(!ValidateFile(DestFile)) - { - RenameOnError(InvalidFormat); - Failed(Message, Cfg); - return; - } - - // FIXME: can we void the "Erase" bool here as its very non-local? - std::string CompressedFile = _config->FindDir("Dir::State::lists") + "partial/"; - CompressedFile += URItoFileName(RealURI); - if(_config->FindB("Acquire::GzipIndexes",false) == false) - CompressedFile += '.' + compExt; - - // Remove the compressed version. - if (Erase == true) - unlink(CompressedFile.c_str()); - - // Done, queue for rename on transaction finished - TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); - - return; + case STAGE_DOWNLOAD: + StageDownloadDone(Message, Hashes, Cfg); + break; + case STAGE_DECOMPRESS_AND_VERIFY: + StageDecompressDone(Message, Hashes, Cfg); + break; } - - // FIXME: use the same method to find - // check the compressed hash too - if(MetaKey != "" && Hashes.size() > 0) +} + +// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ +void pkgAcqIndex::StageDownloadDone(string Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cfg) +{ + // First check if the calculcated Hash of the (compressed) downloaded + // file matches the hash we have in the MetaIndexRecords for this file + if(VerifyHashByMetaKey(Hashes) == false) { - indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); - if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) - { - RenameOnError(HashSumMismatch); - printHashSumComparision(RealURI, Record->Hashes, Hashes); - Failed(Message, Cfg); - return; - } + RenameOnError(HashSumMismatch); + Failed(Message, Cfg); + return; } - Erase = false; Complete = true; // Handle the unzipd case string FileName = LookupTag(Message,"Alt-Filename"); if (FileName.empty() == false) { - Decompression = true; + Stage = STAGE_DECOMPRESS_AND_VERIFY; Local = true; DestFile += ".decomp"; Desc.URI = "copy:" + FileName; @@ -1263,43 +1262,32 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, ErrorText = "Method gave a blank filename"; } - if (FileName == DestFile) - Erase = true; - else + // Methods like e.g. "file:" will give us a (compressed) FileName that is + // not the "DestFile" we set, in this case we uncompress from the local file + if (FileName != DestFile) Local = true; - // do not reverify cdrom sources as apt-cdrom may rewrite the Packages - // file when its doing the indexcopy - if (RealURI.substr(0,6) == "cdrom:" && - StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) - return; - - // The files timestamp matches, reverify by copy into partial/ - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + // we need to verify the file against the current Release file again + // on if-modfied-since hit to avoid a stale attack against us + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - Erase = false; - ReverifyAfterIMS(); -#if 0 // ??? - // set destfile to the final destfile - if(_config->FindB("Acquire::GzipIndexes",false) == false) - { - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); - } + // do not reverify cdrom sources as apt-cdrom may rewrite the Packages + // file when its doing the indexcopy + if (RealURI.substr(0,6) == "cdrom:") + return; - ReverifyAfterIMS(FileName); -#endif + // The files timestamp matches, reverify by copy into partial/ + ReverifyAfterIMS(); return; } - string decompProg; - // If we enable compressed indexes, queue for hash verification + // If we have compressed indexes enabled, queue for hash verification if (_config->FindB("Acquire::GzipIndexes",false)) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI) + '.' + compExt; + DestFile += URItoFileName(RealURI) + '.' + ComprExt; - Decompression = true; + Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FileName; QueueURI(Desc); @@ -1307,16 +1295,19 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, } // get the binary name for your used compression type - decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),""); - if(decompProg.empty() == false); - else if(compExt == "uncompressed") + string decompProg; + if(ComprExt == "uncompressed") decompProg = "copy"; - else { - _error->Error("Unsupported extension: %s", compExt.c_str()); + else + decompProg = _config->Find(string("Acquire::CompressionTypes::").append(ComprExt),""); + if(decompProg.empty() == true) + { + _error->Error("Unsupported extension: %s", ComprExt.c_str()); return; } - Decompression = true; + // queue uri for the next stage + Stage = STAGE_DECOMPRESS_AND_VERIFY; DestFile += ".decomp"; Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); @@ -1331,6 +1322,43 @@ void pkgAcqIndex::Done(string Message, unsigned long long Size, #pragma GCC diagnostic pop #endif } + /*}}}*/ +// pkgAcqIndex::StageDecompressDone - Final verification /*{{{*/ +void pkgAcqIndex::StageDecompressDone(string Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cfg) +{ + if (ExpectedHashes.usable() && ExpectedHashes != Hashes) + { + Desc.URI = RealURI; + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, ExpectedHashes, Hashes); + Failed(Message, Cfg); + return; + } + + if(!ValidateFile(DestFile)) + { + RenameOnError(InvalidFormat); + Failed(Message, Cfg); + return; + } + + // remove the compressed version of the file (if the file got uncompressed) + URI Get = LookupTag(Message, "URI"); + if (Get.Access != "copy") + { + // To account for relative paths + std::string CompressedFile = Get.Host + Get.Path; + unlink(CompressedFile.c_str()); + } + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + + return; +} + /*}}}*/ /*}}}*/ // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -1371,10 +1399,10 @@ string pkgAcqIndexTrans::Custom600Headers() const /* */ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - size_t const nextExt = CompressionExtension.find(' '); + size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) { - CompressionExtension = CompressionExtension.substr(nextExt+1); + CompressionExtensions = CompressionExtensions.substr(nextExt+1); Init(RealURI, Desc.Description, Desc.ShortDesc); Status = StatIdle; return; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 30a8850e4..6518c9a98 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -601,10 +601,20 @@ class pkgAcqBaseIndex : public pkgAcquire::Item /** \brief Pointer to the IndexTarget data */ const struct IndexTarget * Target; + + /** \brief Pointer to the indexRecords parser */ indexRecords *MetaIndexParser; + /** \brief The MetaIndex Key */ std::string MetaKey; + /** \brief The URI of the index file to recreate at our end (either + * by downloading it or by applying partial patches). + */ + std::string RealURI; + + bool VerifyHashByMetaKey(HashStringList const &Hashes); + pkgAcqBaseIndex(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, @@ -631,11 +641,6 @@ class pkgAcqDiffIndex : public pkgAcqBaseIndex /** \brief If \b true, debugging information will be written to std::clog. */ bool Debug; - /** \brief The URI of the index file to recreate at our end (either - * by downloading it or by applying partial patches). - */ - std::string RealURI; - /** \brief The index file which will be patched to generate the new * file. */ @@ -712,11 +717,6 @@ class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex */ bool Debug; - /** \brief URI of the package index file that is being - * reconstructed. - */ - std::string RealURI; - /** \brief description of the file being downloaded. */ std::string Description; @@ -831,11 +831,6 @@ class pkgAcqIndexDiffs : public pkgAcqBaseIndex */ bool Debug; - /** \brief The URI of the package index file that is being - * reconstructed. - */ - std::string RealURI; - /** A description of the file being downloaded. */ std::string Description; @@ -927,23 +922,36 @@ class pkgAcqIndex : public pkgAcqBaseIndex protected: - /** \brief If \b true, the index file has been decompressed. */ - bool Decompression; - - /** \brief If \b true, the partially downloaded file will be - * removed when the download completes. + /** \brief The stages the method goes through + * + * The method first downloads the indexfile, then its decompressed (or + * copied) and verified */ - bool Erase; + enum AllStages { + STAGE_DOWNLOAD, + STAGE_DECOMPRESS_AND_VERIFY, + }; + AllStages Stage; - /** \brief The object that is actually being fetched (minus any - * compression-related extensions). + /** \brief Handle what needs to be done when the download is done */ + void StageDownloadDone(std::string Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cfg); + + /** \brief Handle what needs to be done when the decompression/copy is + * done */ - std::string RealURI; + void StageDecompressDone(std::string Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cfg); /** \brief The compression-related file extensions that are being * added to the downloaded file one by one if first fails (e.g., "gz bz2"). */ - std::string CompressionExtension; + std::string CompressionExtensions; + + /** \brief The actual compression extension currently used */ + std::string ComprExt; /** \brief Do the changes needed to fetch via AptByHash (if needed) */ void InitByHashIfNeeded(const std::string MetaKey); -- cgit v1.2.3 From 1e8ba0d4087f72a930a588ce5fbf0c22dddb9403 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 2 Oct 2014 00:38:35 +0200 Subject: donkults fixes --- apt-pkg/acquire-item.cc | 65 ++++++++++++++------------------ apt-pkg/acquire-item.h | 12 ++++-- test/integration/test-apt-by-hash-update | 2 +- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b3c41a178..54e993458 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -269,12 +269,12 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, - MetaIndexParser), PackagesFileReadyInPartial(false) + MetaIndexParser, Target->URI), + PackagesFileReadyInPartial(false) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - RealURI = Target->URI; Desc.Owner = this; Desc.Description = Target->Description + "/DiffIndex"; Desc.ShortDesc = Target->ShortDesc; @@ -528,7 +528,7 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList Item::Done(Message, Size, Hashes, Cnf); // verify the index target - if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.size() > 0) + if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.usable()) { std::string IndexMetaKey = Target->MetaKey + ".diff/Index"; indexRecords::checkSum *Record = MetaIndexParser->Lookup(IndexMetaKey); @@ -576,7 +576,8 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, + MetaIndexParser, Target->URI), available_patches(diffs), ServerSha1(ServerSha1) { @@ -585,7 +586,6 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - RealURI = Target->URI; Desc.Owner = this; Description = Target->Description; Desc.ShortDesc = Target->ShortDesc; @@ -799,7 +799,8 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, + MetaIndexParser, Target->URI), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -808,7 +809,6 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - RealURI = Target->URI; Desc.Owner = this; Description = Target->Description; Desc.ShortDesc = Target->ShortDesc; @@ -937,7 +937,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri // AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/ bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) { - if(MetaKey != "" && Hashes.size() > 0) + if(MetaKey != "" && Hashes.usable()) { indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) @@ -957,10 +957,8 @@ bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashStringList const &ExpectedHash) - : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL) + : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL, RealURI) { - RealURI = URI; - AutoSelectCompression(); Init(URI, URIDesc, ShortDesc); @@ -977,10 +975,8 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, - MetaIndexParser) + MetaIndexParser, Target->URI) { - RealURI = Target->URI; - // autoselect the compression method AutoSelectCompression(); Init(Target->URI, Target->Description, Target->ShortDesc); @@ -1025,8 +1021,8 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - ComprExt = CompressionExtensions.substr(0, CompressionExtensions.find(' ')); - if (ComprExt == "uncompressed") + CurrentCompressionExtension = CompressionExtensions.substr(0, CompressionExtensions.find(' ')); + if (CurrentCompressionExtension == "uncompressed") { Desc.URI = URI; if(Target) @@ -1034,10 +1030,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, } else { - Desc.URI = URI + '.' + ComprExt; - DestFile = DestFile + '.' + ComprExt; + Desc.URI = URI + '.' + CurrentCompressionExtension; + DestFile = DestFile + '.' + CurrentCompressionExtension; if(Target) - MetaKey = string(Target->MetaKey) + '.' + ComprExt; + MetaKey = string(Target->MetaKey) + '.' + CurrentCompressionExtension; } // load the filesize @@ -1120,9 +1116,7 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ // on decompression failure, remove bad versions in partial/ if (Stage == STAGE_DECOMPRESS_AND_VERIFY) { - string s = _config->FindDir("Dir::State::lists") + "partial/"; - s += URItoFileName(RealURI) + '.' + ComprExt; - unlink(s.c_str()); + unlink(EraseFileName.c_str()); } Item::Failed(Message,Cnf); @@ -1139,7 +1133,7 @@ std::string pkgAcqIndex::GetFinalFilename() const std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); if (_config->FindB("Acquire::GzipIndexes",false) == true) - FinalFile += '.' + ComprExt; + FinalFile += '.' + CurrentCompressionExtension; return FinalFile; } /*}}}*/ @@ -1155,7 +1149,7 @@ void pkgAcqIndex::ReverifyAfterIMS() // adjust DestFile if its compressed on disk if (_config->FindB("Acquire::GzipIndexes",false) == true) - DestFile += '.' + ComprExt; + DestFile += '.' + CurrentCompressionExtension; // copy FinalFile into partial/ so that we check the hash again string FinalFile = GetFinalFilename(); @@ -1266,6 +1260,8 @@ void pkgAcqIndex::StageDownloadDone(string Message, // not the "DestFile" we set, in this case we uncompress from the local file if (FileName != DestFile) Local = true; + else + EraseFileName = FileName; // we need to verify the file against the current Release file again // on if-modfied-since hit to avoid a stale attack against us @@ -1277,6 +1273,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, return; // The files timestamp matches, reverify by copy into partial/ + EraseFileName = ""; ReverifyAfterIMS(); return; } @@ -1285,8 +1282,8 @@ void pkgAcqIndex::StageDownloadDone(string Message, if (_config->FindB("Acquire::GzipIndexes",false)) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI) + '.' + ComprExt; - + DestFile += URItoFileName(RealURI) + '.' + CurrentCompressionExtension; + EraseFileName = ""; Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FileName; QueueURI(Desc); @@ -1296,13 +1293,13 @@ void pkgAcqIndex::StageDownloadDone(string Message, // get the binary name for your used compression type string decompProg; - if(ComprExt == "uncompressed") + if(CurrentCompressionExtension == "uncompressed") decompProg = "copy"; else - decompProg = _config->Find(string("Acquire::CompressionTypes::").append(ComprExt),""); + decompProg = _config->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension),""); if(decompProg.empty() == true) { - _error->Error("Unsupported extension: %s", ComprExt.c_str()); + _error->Error("Unsupported extension: %s", CurrentCompressionExtension.c_str()); return; } @@ -1344,14 +1341,8 @@ void pkgAcqIndex::StageDecompressDone(string Message, return; } - // remove the compressed version of the file (if the file got uncompressed) - URI Get = LookupTag(Message, "URI"); - if (Get.Access != "copy") - { - // To account for relative paths - std::string CompressedFile = Get.Host + Get.Path; - unlink(CompressedFile.c_str()); - } + // remove the compressed version of the file + unlink(EraseFileName.c_str()); // Done, queue for rename on transaction finished TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 6518c9a98..5d5d6efb9 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -619,9 +619,10 @@ class pkgAcqBaseIndex : public pkgAcquire::Item pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, - indexRecords *MetaIndexParser) + indexRecords *MetaIndexParser, + std::string RealURI) : Item(Owner, ExpectedHashes, TransactionManager), Target(Target), - MetaIndexParser(MetaIndexParser) {}; + MetaIndexParser(MetaIndexParser), RealURI(RealURI) {}; }; /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ @@ -945,13 +946,18 @@ class pkgAcqIndex : public pkgAcqBaseIndex HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg); + /** \brief If \b set, this partially downloaded file will be + * removed when the download completes. + */ + std::string EraseFileName; + /** \brief The compression-related file extensions that are being * added to the downloaded file one by one if first fails (e.g., "gz bz2"). */ std::string CompressionExtensions; /** \brief The actual compression extension currently used */ - std::string ComprExt; + std::string CurrentCompressionExtension; /** \brief Do the changes needed to fetch via AptByHash (if needed) */ void InitByHashIfNeeded(const std::string MetaKey); diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 23282bf86..6e1ecdaff 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -34,7 +34,7 @@ Building dependency tree... E: Unable to locate package foo" aptget install -q -s foo # ensure we can apt-get update by hash -testsuccess aptget update -o APT::Acquire::By-Hash=1 +testsuccess aptget update -o APT::Acquire::By-Hash=1 # ensure it works testequal "Inst foo (1.0 unstable [all]) -- cgit v1.2.3 From a64bf0eb7a4e0a6fbb19d19efabecd709a19b917 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 2 Oct 2014 00:47:33 +0200 Subject: fix crash --- apt-pkg/acquire-item.cc | 20 ++++++++++++-------- apt-pkg/acquire-item.h | 5 ++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 54e993458..afc7ae20b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -269,12 +269,12 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, - MetaIndexParser, Target->URI), - PackagesFileReadyInPartial(false) + MetaIndexParser), PackagesFileReadyInPartial(false) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + RealURI = Target->URI; Desc.Owner = this; Desc.Description = Target->Description + "/DiffIndex"; Desc.ShortDesc = Target->ShortDesc; @@ -576,8 +576,7 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, - MetaIndexParser, Target->URI), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { @@ -586,6 +585,7 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + RealURI = Target->URI; Desc.Owner = this; Description = Target->Description; Desc.ShortDesc = Target->ShortDesc; @@ -799,8 +799,7 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, - MetaIndexParser, Target->URI), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -809,6 +808,7 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + RealURI = Target->URI; Desc.Owner = this; Description = Target->Description; Desc.ShortDesc = Target->ShortDesc; @@ -957,8 +957,10 @@ bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashStringList const &ExpectedHash) - : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL, RealURI) + : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL) { + RealURI = URI; + AutoSelectCompression(); Init(URI, URIDesc, ShortDesc); @@ -975,8 +977,10 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, - MetaIndexParser, Target->URI) + MetaIndexParser) { + RealURI = Target->URI; + // autoselect the compression method AutoSelectCompression(); Init(Target->URI, Target->Description, Target->ShortDesc); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 5d5d6efb9..fbbca8bf2 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -619,10 +619,9 @@ class pkgAcqBaseIndex : public pkgAcquire::Item pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, - indexRecords *MetaIndexParser, - std::string RealURI) + indexRecords *MetaIndexParser) : Item(Owner, ExpectedHashes, TransactionManager), Target(Target), - MetaIndexParser(MetaIndexParser), RealURI(RealURI) {}; + MetaIndexParser(MetaIndexParser) {}; }; /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ -- cgit v1.2.3 From 61aea84df918a4257ea1233e48e5860529ecfc9b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 2 Oct 2014 17:28:20 +0200 Subject: add a bunch of docstrings etc --- apt-pkg/acquire-item.cc | 48 ++++++++++++++++++++++++++++++++++++------------ apt-pkg/acquire-item.h | 6 ++++-- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index afc7ae20b..b61a1d833 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1417,12 +1417,17 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ - +// AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqMetaBase::Add(Item *I) { Transaction.push_back(I); } - + /*}}}*/ +// AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqMetaBase::AbortTransaction() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1447,6 +1452,9 @@ void pkgAcqMetaBase::AbortTransaction() } } /*}}}*/ +// AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ bool pkgAcqMetaBase::TransactionHasError() { for (pkgAcquire::ItemIterator I = Transaction.begin(); @@ -1457,7 +1465,10 @@ bool pkgAcqMetaBase::TransactionHasError() return false; } -// Acquire::CommitTransaction - Commit a transaction /*{{{*/ + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqMetaBase::CommitTransaction() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1491,7 +1502,10 @@ void pkgAcqMetaBase::CommitTransaction() (*I)->TransactionManager = 0; } } - + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqMetaBase::TransactionStageCopy(Item *I, const std::string &From, const std::string &To) @@ -1499,16 +1513,21 @@ void pkgAcqMetaBase::TransactionStageCopy(Item *I, I->PartialFile = From; I->DestFile = To; } - + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqMetaBase::TransactionStageRemoval(Item *I, const std::string &FinalFile) { I->PartialFile = ""; I->DestFile = FinalFile; } - - + /*}}}*/ /*{{{*/ +// AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ +// --------------------------------------------------------------------- +/* */ bool pkgAcqMetaBase::GenerateAuthWarning(const std::string &RealURI, const std::string &Message) { @@ -1542,9 +1561,10 @@ bool pkgAcqMetaBase::GenerateAuthWarning(const std::string &RealURI, return false; } /*}}}*/ - - -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ +// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager, string URI,string URIDesc,string ShortDesc, string MetaIndexFile, @@ -1595,8 +1615,12 @@ string pkgAcqMetaSig::Custom600Headers() const return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } - -void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes, + /*}}}*/ +// pkgAcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +void pkgAcqMetaSig::Done(string Message,unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message, Size, Hashes, Cfg); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index fbbca8bf2..97236f90a 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -443,7 +443,8 @@ class pkgAcqMetaSig : public pkgAcqMetaBase // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + virtual void Done(std::string Message,unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); virtual std::string Custom600Headers() const; virtual std::string DescURI() {return RealURI; }; @@ -575,7 +576,8 @@ class pkgAcqMetaClearSig : public pkgAcqMetaIndex public: virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); virtual std::string Custom600Headers() const; - virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + virtual void Done(std::string Message,unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); /** \brief Create a new pkgAcqMetaClearSig. */ -- cgit v1.2.3 From 1ce243188c2ba218f5dce8ec8b40556d58ed8ec2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 2 Oct 2014 18:28:55 +0200 Subject: cleanup around pkgAcqMetaSig and improved tests --- apt-pkg/acquire-item.cc | 91 +++++++++++++++++------------------- apt-pkg/acquire-item.h | 3 ++ test/integration/test-apt-update-ims | 34 ++++++++++++-- 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b61a1d833..0a128f7d5 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1576,7 +1576,7 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, ShortDesc(ShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(RealURI); // remove any partial downloaded sig-file in partial/. // it may confuse proxies and is too small to warrant a @@ -1625,68 +1625,65 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, { Item::Done(Message, Size, Hashes, Cfg); - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) + if(AuthPass == false) { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } + // queue for verify, note that we change DestFile here to point to + // the file we want to verify (needed to make gpgv work) - if (FileName != DestFile) - { - // We have to copy it into place - Local = true; - Desc.URI = "copy:" + FileName; - QueueURI(Desc); - return; - } + string FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return; + } - if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) - IMSHit = true; + if (FileName != DestFile) + { + // We have to copy it into place + Local = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; + } - // adjust paths if its a ims-hit - if(IMSHit) - { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - - TransactionManager->TransactionStageCopy(this, FinalFile, FinalFile); - } + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + IMSHit = true; + // adjust DestFile on i-m-s hit to the one we already have on disk + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + } + + // this is the file we verify from + MetaIndexFileSignature = DestFile; - // queue for verify - if(AuthPass == false) - { AuthPass = true; - Desc.URI = "gpgv:" + DestFile; + Desc.URI = "gpgv:" + MetaIndexFileSignature; DestFile = MetaIndexFile; QueueURI(Desc); + ActiveSubprocess = "gpgv"; return; } - - // queue to copy the file in place if it was not a ims hit, on ims - // hit the file is already at the right place - if(IMSHit == false) + else { - PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; - PartialFile += URItoFileName(RealURI); - - std::string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - - TransactionManager->TransactionStageCopy(this, PartialFile, FinalFile); - } + // verify was successful - // we parse the MetaIndexFile here because at this point we can - // trust the data - if(AuthPass == true) - { + // we parse the MetaIndexFile here (and not right after getting + // the pkgAcqMetaIndex) because at this point we can trust the data + // // load indexes and queue further downloads MetaIndexParser->Load(MetaIndexFile); QueueIndexes(true); - } - Complete = true; + // DestFile points to the the MetaIndeFile at this point, make it + // point back to the Release.gpg file + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, FinalFile); + + Complete = true; + } } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 97236f90a..083a73197 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -433,6 +433,9 @@ class pkgAcqMetaSig : public pkgAcqMetaBase /** \brief The file we need to verify */ std::string MetaIndexFile; + /** \brief The file we use to verify the MetaIndexFile with */ + std::string MetaIndexFileSignature; + /** \brief Long URI description used in the acquire system */ std::string URIDesc; diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 946dfc7af..38dcd73fd 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -20,11 +20,7 @@ runtest() { testfailure ls "rootdir/var/lib/apt/lists/partial/*" # check that I-M-S header is kept in redirections - testequal "Hit http://localhost:8080 unstable InRelease -Hit http://localhost:8080 unstable/main Sources -Hit http://localhost:8080 unstable/main amd64 Packages -Hit http://localhost:8080 unstable/main Translation-en -Reading package lists..." aptget update -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0 + testequal "$EXPECT" aptget update -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0 # ensure that we still do a hash check on ims hit msgtest 'Test I-M-S reverify' @@ -34,6 +30,34 @@ Reading package lists..." aptget update -o Debug::pkgAcquire::Worker=0 -o Debug testfailure ls "rootdir/var/lib/apt/lists/partial/*" } +EXPECT="Hit http://localhost:8080 unstable InRelease +Hit http://localhost:8080 unstable/main Sources +Hit http://localhost:8080 unstable/main amd64 Packages +Hit http://localhost:8080 unstable/main Translation-en +Reading package lists..." +# with InRelease +runtest + +# with gzip +echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex +runtest + +# FIXME: how can we get rid of this extra line +# "Get:1 http://localhost:8080 unstable Release.gpg" +# +# with Release/Release.gpg +EXPECT="Ign http://localhost:8080 unstable InRelease +Hit http://localhost:8080 unstable Release +Hit http://localhost:8080 unstable Release.gpg +Get:1 http://localhost:8080 unstable Release.gpg +Hit http://localhost:8080 unstable/main Sources +Hit http://localhost:8080 unstable/main amd64 Packages +Hit http://localhost:8080 unstable/main Translation-en +Reading package lists..." + +find aptarchive -name "InRelease" | xargs rm -f + +echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex runtest echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex -- cgit v1.2.3 From 1d970e6ce97385ed719a1ca169ec5cc7bfb82fea Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 3 Oct 2014 00:39:20 +0200 Subject: really do not download Release if InRelease does not verify --- apt-pkg/acquire-item.cc | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 0a128f7d5..3c66369cf 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -2175,26 +2175,6 @@ void pkgAcqMetaIndex::Failed(string Message, return; } - /* Always move the meta index, even if gpgv failed. This ensures - * that PackageFile objects are correctly filled in */ - if (FileExists(DestFile)) - { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - /* InRelease files become Release files, otherwise - * they would be considered as trusted later on */ - if (SigFile == DestFile) { - RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, - "Release"); - FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, - "Release"); - SigFile = FinalFile; - } - - // Done, queue for rename on transaction finished - TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); - } - _error->Warning(_("The data from '%s' is not signed. Packages " "from that repository can not be authenticated."), URIDesc.c_str()); @@ -2204,11 +2184,35 @@ void pkgAcqMetaIndex::Failed(string Message, // only allow going further if the users explicitely wants it if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { + /* Always move the meta index, even if gpgv failed. This ensures + * that PackageFile objects are correctly filled in */ + if (FileExists(DestFile)) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + /* InRelease files become Release files, otherwise + * they would be considered as trusted later on */ + if (SigFile == DestFile) { + RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, + "Release"); + FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, + "Release"); + SigFile = FinalFile; + } + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + } + QueueIndexes(false); } else { // warn if the repository is unsinged _error->Warning("Use --allow-insecure-repositories to force the update"); + TransactionManager->AbortTransaction(); + Status = StatError; + return; } + } /*}}}*/ -- cgit v1.2.3 From 0f56b51e125d24cf5af68459077ad1b682743bc2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 09:34:06 +0200 Subject: update test --- test/integration/test-apt-get-update-unauth-warning | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 510249747..75863615e 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -20,13 +20,17 @@ rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated files leads to warning testequal "Ign file: unstable InRelease -Ign file: unstable Release -Reading package lists... +Err file: unstable Release + W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated. -W: Use --allow-insecure-repositories to force the update" aptget update +W: Use --allow-insecure-repositories to force the update +W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release + +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update # no package foo testequal "Listing..." apt list foo +testequal "partial" ls rootdir/var/lib/apt/lists # allow override testequal "Ign file: unstable InRelease -- cgit v1.2.3 From eeac6897ebc6ecad6721ef9bd457fe51ef611755 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 09:42:12 +0200 Subject: add new "SetActiveSubprocess() --- apt-pkg/acquire-item.cc | 65 +++++++++++++++---------------------------------- apt-pkg/acquire-item.h | 6 +++++ 2 files changed, 25 insertions(+), 46 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 3c66369cf..9eb35ff33 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -212,6 +212,19 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const return false; } /*}}}*/ +void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess) +{ + ActiveSubprocess = subprocess; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + Mode = ActiveSubprocess.c_str(); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif +} + // Acquire::Item::ReportMirrorFailure /*{{{*/ // --------------------------------------------------------------------- void pkgAcquire::Item::ReportMirrorFailure(string FailCode) @@ -749,15 +762,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi Local = true; Desc.URI = "rred:" + FinalFile; QueueURI(Desc); - ActiveSubprocess = "rred"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - Mode = "rred"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic pop -#endif + SetActiveSubprocess("rred"); return; } @@ -882,15 +887,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Local = true; Desc.URI = "rred:" + FinalFile; QueueURI(Desc); - ActiveSubprocess = "rred"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - Mode = "rred"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic pop -#endif + SetActiveSubprocess("rred"); return; } // success in download/apply all diffs, clean up @@ -1241,15 +1238,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, DestFile += ".decomp"; Desc.URI = "copy:" + FileName; QueueURI(Desc); - ActiveSubprocess = "copy"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - Mode = "copy"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic pop -#endif + SetActiveSubprocess("copy"); return; } @@ -1313,15 +1302,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - ActiveSubprocess = decompProg; -#if __GNUC__ >= 4 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - Mode = ActiveSubprocess.c_str(); -#if __GNUC__ >= 4 - #pragma GCC diagnostic pop -#endif + SetActiveSubprocess(decompProg); } /*}}}*/ // pkgAcqIndex::StageDecompressDone - Final verification /*{{{*/ @@ -1662,7 +1643,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, Desc.URI = "gpgv:" + MetaIndexFileSignature; DestFile = MetaIndexFile; QueueURI(Desc); - ActiveSubprocess = "gpgv"; + SetActiveSubprocess("gpgv"); return; } else @@ -1848,15 +1829,7 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList AuthPass = true; Desc.URI = "gpgv:" + SigFile; QueueURI(Desc); - ActiveSubprocess = "gpgv"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - Mode = "gpgv"; -#if __GNUC__ >= 4 - #pragma GCC diagnostic pop -#endif + SetActiveSubprocess("gpgv"); return; } } diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 083a73197..bab2cc0d7 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -292,6 +292,12 @@ class pkgAcquire::Item : public WeakPointable */ void ReportMirrorFailure(std::string FailCode); + /** \brief Set the name of the current active subprocess + * + * See also #ActiveSubprocess + */ + void SetActiveSubprocess(const std::string &subprocess); + /** \brief Initialize an item. * * Adds the item to the list of items known to the acquire -- cgit v1.2.3 From 61a360be789ed81a939f09e89b939da7a103d81a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 09:43:48 +0200 Subject: fix incorrect docstrings for AcqMetaBase::TransactionStageRemoval/AcqMetaBase::TransactionStageCopy --- apt-pkg/acquire-item.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 9eb35ff33..d1cf53fd5 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1484,7 +1484,7 @@ void pkgAcqMetaBase::CommitTransaction() } } /*}}}*/ -// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/ // --------------------------------------------------------------------- /* */ void pkgAcqMetaBase::TransactionStageCopy(Item *I, @@ -1495,7 +1495,7 @@ void pkgAcqMetaBase::TransactionStageCopy(Item *I, I->DestFile = To; } /*}}}*/ -// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// AcqMetaBase::TransactionStageRemoval - Sage a file for removal /*{{{*/ // --------------------------------------------------------------------- /* */ void pkgAcqMetaBase::TransactionStageRemoval(Item *I, -- cgit v1.2.3 From f30976478e684fc19e48d71881805454ceb6ecae Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 11:45:42 +0200 Subject: Rework pkgAcqMeta{Index,Sig,ClearSig}::Done() for readability Move common code out but do not use subclassing for ::Done to make it easier to understand what each class is doing when its done --- apt-pkg/acquire-item.cc | 272 ++++++++++++----------------------- apt-pkg/acquire-item.h | 64 +++++---- test/integration/test-apt-update-ims | 26 +++- 3 files changed, 149 insertions(+), 213 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d1cf53fd5..6c04288a8 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1608,62 +1608,24 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, if(AuthPass == false) { - // queue for verify, note that we change DestFile here to point to - // the file we want to verify (needed to make gpgv work) - - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) - { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } - - if (FileName != DestFile) - { - // We have to copy it into place - Local = true; - Desc.URI = "copy:" + FileName; - QueueURI(Desc); - return; - } - - if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + if(CheckDownloadDone(Message, RealURI) == true) { - IMSHit = true; - // adjust DestFile on i-m-s hit to the one we already have on disk - DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI); + // destfile will be modified to point to MetaIndexFile for the + // gpgv method, so we need to save it here + MetaIndexFileSignature = DestFile; + QueueForSignatureVerify(MetaIndexFile, MetaIndexFileSignature); } - - // this is the file we verify from - MetaIndexFileSignature = DestFile; - - AuthPass = true; - Desc.URI = "gpgv:" + MetaIndexFileSignature; - DestFile = MetaIndexFile; - QueueURI(Desc); - SetActiveSubprocess("gpgv"); return; } else { - // verify was successful - - // we parse the MetaIndexFile here (and not right after getting - // the pkgAcqMetaIndex) because at this point we can trust the data - // - // load indexes and queue further downloads - MetaIndexParser->Load(MetaIndexFile); - QueueIndexes(true); - - // DestFile points to the the MetaIndeFile at this point, make it - // point back to the Release.gpg file - std::string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, FinalFile); + if(AuthDone(Message, RealURI) == true) + { + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); - Complete = true; + TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, FinalFile); + } } } /*}}}*/ @@ -1796,57 +1758,78 @@ string pkgAcqMetaIndex::Custom600Headers() const return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ +void pkgAcqMetaIndex::Done(string Message,unsigned long long Size, /*{{{*/ + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hashes,Cfg); - // MetaIndexes are done in two passes: one to download the - // metaindex with an appropriate method, and a second to verify it - // with the gpgv method - - if (AuthPass == true) + if(CheckDownloadDone(Message, RealURI)) { - AuthDone(Message); + // we have a Release file, now download the Signature, all further + // verify/queue for additional downloads will be done in the + // pkgAcqMetaSig::Done() code + std::string MetaIndexFile = DestFile; + new pkgAcqMetaSig(Owner, TransactionManager, + MetaIndexSigURI, MetaIndexSigURIDesc, + MetaIndexSigShortDesc, MetaIndexFile, IndexTargets, + MetaIndexParser); - // all cool, move Release file into place - Complete = true; + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); } - else - { - RetrievalDone(Message); - if (!Complete) - // Still more retrieving to do - return; +} + /*}}}*/ +bool pkgAcqMetaBase::AuthDone(string Message, const string &RealURI) /*{{{*/ +{ + // At this point, the gpgv method has succeeded, so there is a + // valid signature from a key in the trusted keyring. We + // perform additional verification of its contents, and use them + // to verify the indexes we are about to download - if (SigFile != "") - { - // There was a signature file, so pass it to gpgv for - // verification - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - std::cerr << "Metaindex acquired, queueing gpg verification (" - << SigFile << "," << DestFile << ")\n"; - AuthPass = true; - Desc.URI = "gpgv:" + SigFile; - QueueURI(Desc); - SetActiveSubprocess("gpgv"); - return; - } + if (!MetaIndexParser->Load(DestFile)) + { + Status = StatAuthError; + ErrorText = MetaIndexParser->ErrorText; + return false; } - if (Complete == true) + if (!VerifyVendor(Message, RealURI)) { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - if (SigFile == DestFile) - SigFile = FinalFile; - - // queue for copy in place - TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + return false; } + + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + std::cerr << "Signature verification succeeded: " + << DestFile << std::endl; + + // Download further indexes with verification + // + // it would be really nice if we could simply do + // if (IMSHit == false) QueueIndexes(true) + // and skip the download if the Release file has not changed + // - but right now the list cleaner will needs to be tricked + // to not delete all our packages/source indexes in this case + QueueIndexes(true); + + return true; +} + /*}}}*/ + /*{{{*/ +void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile, + const std::string &MetaIndexFileSignature) +{ + AuthPass = true; + Desc.URI = "gpgv:" + MetaIndexFileSignature; + DestFile = MetaIndexFile; + QueueURI(Desc); + SetActiveSubprocess("gpgv"); } /*}}}*/ -void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ + /*{{{*/ +bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message, + const std::string &RealURI) { // We have just finished downloading a Release file (it is not // verified yet) @@ -1856,7 +1839,7 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ { Status = StatError; ErrorText = "Method gave a blank filename"; - return; + return false; } if (FileName != DestFile) @@ -1864,7 +1847,7 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ Local = true; Desc.URI = "copy:" + FileName; QueueURI(Desc); - return; + return false; } // make sure to verify against the right file on I-M-S hit @@ -1873,101 +1856,13 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); - if (SigFile == DestFile) - { - SigFile = FinalFile; -#if 0 - // constructor of pkgAcqMetaClearSig moved it out of the way, - // now move it back in on IMS hit for the 'old' file - string const OldClearSig = DestFile + ".reverify"; - if (RealFileExists(OldClearSig) == true) - Rename(OldClearSig, FinalFile); -#endif - } DestFile = FinalFile; } - // queue a signature - if(SigFile != DestFile) - new pkgAcqMetaSig(Owner, TransactionManager, - MetaIndexSigURI, MetaIndexSigURIDesc, - MetaIndexSigShortDesc, DestFile, IndexTargets, - MetaIndexParser); - + // set Item to complete as the remaining work is all local (verify etc) Complete = true; -} - /*}}}*/ -void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ -{ - // At this point, the gpgv method has succeeded, so there is a - // valid signature from a key in the trusted keyring. We - // perform additional verification of its contents, and use them - // to verify the indexes we are about to download - - if (!MetaIndexParser->Load(DestFile)) - { - Status = StatAuthError; - ErrorText = MetaIndexParser->ErrorText; - return; - } - - if (!VerifyVendor(Message)) - { - return; - } - - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - std::cerr << "Signature verification succeeded: " - << DestFile << std::endl; -// we ensure this by other means -#if 0 - // do not trust any previously unverified content that we may have - string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); - if (DestFile != SigFile) - LastGoodSigFile.append(".gpg"); - LastGoodSigFile.append(".reverify"); - if(IMSHit == false && RealFileExists(LastGoodSigFile) == false) - { - for (vector ::const_iterator Target = IndexTargets->begin(); - Target != IndexTargets->end(); - ++Target) - { - // remove old indexes - std::string index = _config->FindDir("Dir::State::lists") + - URItoFileName((*Target)->URI); - unlink(index.c_str()); - // and also old gzipindexes - std::vector types = APT::Configuration::getCompressionTypes(); - for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - { - index += '.' + (*t); - unlink(index.c_str()); - } - } - } -#endif - - // Download further indexes with verification - // - // it would be really nice if we could simply do - // if (IMSHit == false) QueueIndexes(true) - // and skip the download if the Release file has not changed - // - but right now the list cleaner will needs to be tricked - // to not delete all our packages/source indexes in this case - QueueIndexes(true); - -#if 0 - // is it a clearsigned MetaIndex file? - if (DestFile == SigFile) - return; - - // Done, move signature file into position - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(RealURI) + ".gpg"; - Rename(SigFile,VerifiedSigFile); - chmod(VerifiedSigFile.c_str(),0644); -#endif + return true; } /*}}}*/ void pkgAcqMetaBase::QueueIndexes(bool verify) /*{{{*/ @@ -2056,7 +1951,7 @@ void pkgAcqMetaBase::QueueIndexes(bool verify) /*{{{*/ } } /*}}}*/ -bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/ +bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/ { string::size_type pos; @@ -2273,7 +2168,24 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, TransactionManager->AbortTransaction(); return; } - pkgAcqMetaIndex::Done(Message, Size, Hashes, Cnf); + + if(AuthPass == false) + { + if(CheckDownloadDone(Message, RealURI) == true) + QueueForSignatureVerify(DestFile, DestFile); + return; + } + else + { + if(AuthDone(Message, RealURI) == true) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + // queue for copy in place + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + } + } } /*}}}*/ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index bab2cc0d7..f12f57262 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -385,6 +385,41 @@ class pkgAcqMetaBase : public pkgAcquire::Item */ void QueueIndexes(bool verify); + + /** \brief Called when a file is finished being retrieved. + * + * If the file was not downloaded to DestFile, a copy process is + * set up to copy it to DestFile; otherwise, Complete is set to \b + * true and the file is moved to its final location. + * + * \param Message The message block received from the fetch + * subprocess. + */ + bool CheckDownloadDone(const std::string &Message, + const std::string &RealURI); + + /** \brief Queue the downloaded Signature for verification */ + void QueueForSignatureVerify(const std::string &MetaIndexFile, + const std::string &MetaIndexFileSignature); + + /** \brief Called when authentication succeeded. + * + * Sanity-checks the authenticated file, queues up the individual + * index files for download, and saves the signature in the lists + * directory next to the authenticated list file. + * + * \param Message The message block received from the fetch + * subprocess. + */ + bool AuthDone(std::string Message, const std::string &RealURI); + + /** \brief Check that the release file is a release file for the + * correct distribution. + * + * \return \b true if no fatal errors were encountered. + */ + bool VerifyVendor(std::string Message, const std::string &RealURI); + public: // transaction code void Add(Item *I); @@ -496,35 +531,6 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase */ std::string SigFile; - /** \brief Check that the release file is a release file for the - * correct distribution. - * - * \return \b true if no fatal errors were encountered. - */ - bool VerifyVendor(std::string Message); - - /** \brief Called when a file is finished being retrieved. - * - * If the file was not downloaded to DestFile, a copy process is - * set up to copy it to DestFile; otherwise, Complete is set to \b - * true and the file is moved to its final location. - * - * \param Message The message block received from the fetch - * subprocess. - */ - void RetrievalDone(std::string Message); - - /** \brief Called when authentication succeeded. - * - * Sanity-checks the authenticated file, queues up the individual - * index files for download, and saves the signature in the lists - * directory next to the authenticated list file. - * - * \param Message The message block received from the fetch - * subprocess. - */ - void AuthDone(std::string Message); - std::string URIDesc; std::string ShortDesc; diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 38dcd73fd..61b808b0f 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -30,6 +30,7 @@ runtest() { testfailure ls "rootdir/var/lib/apt/lists/partial/*" } +msgmsg "InRelease" EXPECT="Hit http://localhost:8080 unstable InRelease Hit http://localhost:8080 unstable/main Sources Hit http://localhost:8080 unstable/main amd64 Packages @@ -42,14 +43,11 @@ runtest echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex runtest -# FIXME: how can we get rid of this extra line -# "Get:1 http://localhost:8080 unstable Release.gpg" -# +msgmsg "Release/Release.gpg" # with Release/Release.gpg EXPECT="Ign http://localhost:8080 unstable InRelease Hit http://localhost:8080 unstable Release Hit http://localhost:8080 unstable Release.gpg -Get:1 http://localhost:8080 unstable Release.gpg Hit http://localhost:8080 unstable/main Sources Hit http://localhost:8080 unstable/main amd64 Packages Hit http://localhost:8080 unstable/main Translation-en @@ -62,3 +60,23 @@ runtest echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex runtest + + +# no Release.gpg or InRelease +msgmsg "Release only" +EXPECT="Ign http://localhost:8080 unstable InRelease +Hit http://localhost:8080 unstable Release +Ign http://localhost:8080 unstable Release.gpg +Hit http://localhost:8080 unstable/main Sources +Hit http://localhost:8080 unstable/main amd64 Packages +Hit http://localhost:8080 unstable/main Translation-en +Reading package lists..." + +find aptarchive -name "Release.gpg" | xargs rm -f + +echo 'Acquire::AllowInsecureRepositories "1";' > rootdir/etc/apt/apt.conf.d/insecure.conf +echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex +runtest + +echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex +runtest -- cgit v1.2.3 From 22b2ef9d29b8a467c0fca8637636e417e020e2f5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 13:18:45 +0200 Subject: add missing TransactionStageCopy() in pkgAcqDiffIndex::Done() --- apt-pkg/acquire-item.cc | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 6c04288a8..975965e46 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -555,21 +555,14 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList } + if(!ParseDiffIndex(DestFile)) + return Failed("", NULL); + + // queue for final move string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); - - // success in downloading the index - // rename the index FinalFile += string(".IndexDiff"); - if(Debug) - std::clog << "Renaming: " << DestFile << " -> " << FinalFile - << std::endl; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - DestFile = FinalFile; - - if(!ParseDiffIndex(DestFile)) - return Failed("", NULL); + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); Complete = true; Status = StatDone; -- cgit v1.2.3 From 673c9469abd656a92c7e8f1f91f919cad09f391e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 14:34:38 +0200 Subject: cleanup pkgAcq*::Failed() --- apt-pkg/acquire-item.cc | 113 ++++++++++----------- apt-pkg/acquire-item.h | 13 +-- .../integration/test-apt-get-update-unauth-warning | 2 +- 3 files changed, 55 insertions(+), 73 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 975965e46..401566cfc 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1502,8 +1502,8 @@ void pkgAcqMetaBase::TransactionStageRemoval(Item *I, // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgAcqMetaBase::GenerateAuthWarning(const std::string &RealURI, - const std::string &Message) +bool pkgAcqMetaBase::StopAuthentication(const std::string &RealURI, + const std::string &Message) { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@ -1626,13 +1626,9 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - // FIXME: duplicated code from pkgAcqMetaIndex - if (AuthPass == true) - { - bool Stop = GenerateAuthWarning(RealURI, Message); - if(Stop) + // check if we need to fail at this point + if (AuthPass == true && StopAuthentication(RealURI, Message)) return; - } // FIXME: meh, this is not really elegant string InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12, @@ -2021,50 +2017,28 @@ bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/ return true; } /*}}}*/ -// pkgAcqMetaIndex::Failed - no Release file present or no signature file present /*{{{*/ +// pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ // --------------------------------------------------------------------- /* */ void pkgAcqMetaIndex::Failed(string Message, pkgAcquire::MethodConfig * /*Cnf*/) { - string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + string FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - if (AuthPass == true) - { - bool Stop = GenerateAuthWarning(RealURI, Message); - if(Stop) - return; - } + _error->Warning(_("The repository '%s' does not have a Release file. " + "This is deprecated, please contact the owner of the " + "repository."), URIDesc.c_str()); - _error->Warning(_("The data from '%s' is not signed. Packages " - "from that repository can not be authenticated."), - URIDesc.c_str()); - - // No Release file was present, or verification failed, so fall + // No Release file was present so fall // back to queueing Packages files without verification // only allow going further if the users explicitely wants it if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { - /* Always move the meta index, even if gpgv failed. This ensures - * that PackageFile objects are correctly filled in */ + // Done, queue for rename on transaction finished if (FileExists(DestFile)) - { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - /* InRelease files become Release files, otherwise - * they would be considered as trusted later on */ - if (SigFile == DestFile) { - RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, - "Release"); - FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, - "Release"); - SigFile = FinalFile; - } - - // Done, queue for rename on transaction finished TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); - } + // queue without any kind of hashsum support QueueIndexes(false); } else { // warn if the repository is unsinged @@ -2073,7 +2047,6 @@ void pkgAcqMetaIndex::Failed(string Message, Status = StatError; return; } - } /*}}}*/ @@ -2097,39 +2070,17 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { - SigFile = DestFile; - // index targets + (worst case:) Release/Release.gpg ExpectedAdditionalItems = IndexTargets->size() + 2; -#if 0 - // keep the old InRelease around in case of transistent network errors - string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - if (RealFileExists(Final) == true) - { - string const LastGoodSig = DestFile + ".reverify"; - Rename(Final,LastGoodSig); - } -#endif } /*}}}*/ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ { -#if 0 - // if the file was never queued undo file-changes done in the constructor - if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false) - { - string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - string const LastGoodSig = DestFile + ".reverify"; - if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) - Rename(LastGoodSig, Final); - } -#endif } /*}}}*/ // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- -// FIXME: this can go away once the InRelease file is used widely string pkgAcqMetaClearSig::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); @@ -2204,7 +2155,45 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* Dequeue(); } else - pkgAcqMetaIndex::Failed(Message, Cnf); + { + if(StopAuthentication(RealURI, Message)) + return; + + _error->Warning(_("The data from '%s' is not signed. Packages " + "from that repository can not be authenticated."), + URIDesc.c_str()); + + // No Release file was present, or verification failed, so fall + // back to queueing Packages files without verification + // only allow going further if the users explicitely wants it + if(_config->FindB("Acquire::AllowInsecureRepositories") == true) + { + /* Always move the meta index, even if gpgv failed. This ensures + * that PackageFile objects are correctly filled in */ + if (FileExists(DestFile)) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + /* InRelease files become Release files, otherwise + * they would be considered as trusted later on */ + RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, + "Release"); + FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, + "Release"); + + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + } + QueueIndexes(false); + } else { + // warn if the repository is unsinged + _error->Warning("Use --allow-insecure-repositories to force the update"); + TransactionManager->AbortTransaction(); + Status = StatError; + return; + } + } } /*}}}*/ // AcqArchive::AcqArchive - Constructor /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index f12f57262..e0739dcd2 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -436,9 +436,9 @@ class pkgAcqMetaBase : public pkgAcquire::Item */ void TransactionStageRemoval(Item *I, const std::string &FinalFile); - // helper for the signature warning - bool GenerateAuthWarning(const std::string &RealURI, - const std::string &Message); + /** Check if the current item should fail at this point */ + bool StopAuthentication(const std::string &RealURI, + const std::string &Message); pkgAcqMetaBase(pkgAcquire *Owner, @@ -524,13 +524,6 @@ class pkgAcqMetaIndex : public pkgAcqMetaBase */ std::string RealURI; - /** \brief The file in which the signature for this index was stored. - * - * If empty, the signature and the md5sums of the individual - * indices will not be checked. - */ - std::string SigFile; - std::string URIDesc; std::string ShortDesc; diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 75863615e..37bcea623 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -22,7 +22,7 @@ rm -f $APTARCHIVE/dists/unstable/*Release* testequal "Ign file: unstable InRelease Err file: unstable Release -W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated. +W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository. W: Use --allow-insecure-repositories to force the update W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release -- cgit v1.2.3 From ba8a84216a84c5e1e02ad46f412a04728277cb36 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 14:39:16 +0200 Subject: rename AuthDone() -> CheckAuthDone() --- apt-pkg/acquire-item.cc | 8 +++----- apt-pkg/acquire-item.h | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 401566cfc..c90210af7 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1577,7 +1577,6 @@ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ /*}}}*/ // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- -/* The only header we use is the last-modified header. */ string pkgAcqMetaSig::Custom600Headers() const { string FinalFile = _config->FindDir("Dir::State::lists"); @@ -1612,7 +1611,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, } else { - if(AuthDone(Message, RealURI) == true) + if(CheckAuthDone(Message, RealURI) == true) { std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); @@ -1734,7 +1733,6 @@ void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) } // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- -/* The only header we use is the last-modified header. */ string pkgAcqMetaIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); @@ -1770,7 +1768,7 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size, /*{{{*/ } } /*}}}*/ -bool pkgAcqMetaBase::AuthDone(string Message, const string &RealURI) /*{{{*/ +bool pkgAcqMetaBase::CheckAuthDone(string Message, const string &RealURI) /*{{{*/ { // At this point, the gpgv method has succeeded, so there is a // valid signature from a key in the trusted keyring. We @@ -2121,7 +2119,7 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, } else { - if(AuthDone(Message, RealURI) == true) + if(CheckAuthDone(Message, RealURI) == true) { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index e0739dcd2..393f3a250 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -411,7 +411,7 @@ class pkgAcqMetaBase : public pkgAcquire::Item * \param Message The message block received from the fetch * subprocess. */ - bool AuthDone(std::string Message, const std::string &RealURI); + bool CheckAuthDone(std::string Message, const std::string &RealURI); /** \brief Check that the release file is a release file for the * correct distribution. -- cgit v1.2.3 From 42299a28ac40721f6cf29c9b786924c2cd4a210f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 14:43:05 +0200 Subject: fix test --- test/integration/test-apt-get-update-unauth-warning | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 37bcea623..27160b5f9 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -36,8 +36,7 @@ testequal "partial" ls rootdir/var/lib/apt/lists testequal "Ign file: unstable InRelease Ign file: unstable Release Reading package lists... -W: The data from 'file: unstable Release' is not signed. Packages from that repository can not be authenticated." aptget update --allow-insecure-repositories - +W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories # ensure we can not install the package testequal "WARNING: The following packages cannot be authenticated! foo -- cgit v1.2.3 From 2d0a7bb434ebef179ab4955dfb09262452213190 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 14:54:53 +0200 Subject: rename StopAuthentication -> CheckStopAuthentication and make it protected --- apt-pkg/acquire-item.cc | 11 +++++++---- apt-pkg/acquire-item.h | 9 ++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c90210af7..e23acbd2a 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1502,9 +1502,12 @@ void pkgAcqMetaBase::TransactionStageRemoval(Item *I, // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgAcqMetaBase::StopAuthentication(const std::string &RealURI, - const std::string &Message) +bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &RealURI, + const std::string &Message) { + // FIXME: this entire function can do now that we disallow going to + // a unauthenticated state and can cleanly rollback + string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); if(FileExists(Final)) @@ -1626,7 +1629,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); // check if we need to fail at this point - if (AuthPass == true && StopAuthentication(RealURI, Message)) + if (AuthPass == true && CheckStopAuthentication(RealURI, Message)) return; // FIXME: meh, this is not really elegant @@ -2154,7 +2157,7 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* } else { - if(StopAuthentication(RealURI, Message)) + if(CheckStopAuthentication(RealURI, Message)) return; _error->Warning(_("The data from '%s' is not signed. Packages " diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 393f3a250..02b8c13e8 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -413,6 +413,10 @@ class pkgAcqMetaBase : public pkgAcquire::Item */ bool CheckAuthDone(std::string Message, const std::string &RealURI); + /** Check if the current item should fail at this point */ + bool CheckStopAuthentication(const std::string &RealURI, + const std::string &Message); + /** \brief Check that the release file is a release file for the * correct distribution. * @@ -436,11 +440,6 @@ class pkgAcqMetaBase : public pkgAcquire::Item */ void TransactionStageRemoval(Item *I, const std::string &FinalFile); - /** Check if the current item should fail at this point */ - bool StopAuthentication(const std::string &RealURI, - const std::string &Message); - - pkgAcqMetaBase(pkgAcquire *Owner, const std::vector* IndexTargets, indexRecords* MetaIndexParser, -- cgit v1.2.3 From 0be13f1c8afdc4462bd0061130f943006915fbbc Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 6 Oct 2014 16:28:56 +0200 Subject: fix warnings --- apt-pkg/acquire-item.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index e23acbd2a..c5037e5e0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -2021,7 +2021,7 @@ bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/ // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqMetaIndex::Failed(string Message, +void pkgAcqMetaIndex::Failed(string /*Message*/, pkgAcquire::MethodConfig * /*Cnf*/) { string FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@ -2099,8 +2099,8 @@ string pkgAcqMetaClearSig::Custom600Headers() const /*}}}*/ // pkgAcqMetaClearSig::Done - We got a file /*{{{*/ // --------------------------------------------------------------------- -void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, - HashStringList const &Hashes, +void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long /*Size*/, + HashStringList const &/*Hashes*/, pkgAcquire::MethodConfig *Cnf) { // if we expect a ClearTextSignature (InRelase), ensure that -- cgit v1.2.3 From 8267fbd9c4d4a5add120282fe180c48e851958a5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 6 Oct 2014 11:34:07 +0200 Subject: fixup foldmarkers in acquire-item.cc Git-Dch: Ignore --- apt-pkg/acquire-item.cc | 142 +++++++++++++++++------------------------------- 1 file changed, 51 insertions(+), 91 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c5037e5e0..ad9198cba 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -212,7 +212,7 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const return false; } /*}}}*/ -void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess) +void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/ { ActiveSubprocess = subprocess; #if __GNUC__ >= 4 @@ -224,7 +224,7 @@ void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess) #pragma GCC diagnostic pop #endif } - + /*}}}*/ // Acquire::Item::ReportMirrorFailure /*{{{*/ // --------------------------------------------------------------------- void pkgAcquire::Item::ReportMirrorFailure(string FailCode) @@ -923,7 +923,6 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri } } /*}}}*/ - // AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/ bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) { @@ -938,12 +937,11 @@ bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) } return true; } - - + /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- -/* The package file is added to the queue and a second class is - instantiated to fetch the revision file */ +/* The package file is added to the queue and a second class is + instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashStringList const &ExpectedHash) @@ -960,13 +958,12 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, } /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ -// --------------------------------------------------------------------- pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager, IndexTarget const *Target, - HashStringList const &ExpectedHash, + HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, MetaIndexParser) { RealURI = Target->URI; @@ -981,7 +978,6 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, } /*}}}*/ // AcqIndex::AutoSelectCompression - Select compression /*{{{*/ -// --------------------------------------------------------------------- void pkgAcqIndex::AutoSelectCompression() { std::vector types = APT::Configuration::getCompressionTypes(); @@ -992,7 +988,7 @@ void pkgAcqIndex::AutoSelectCompression() t != types.end(); ++t) { std::string CompressedMetaKey = string(Target->MetaKey).append(".").append(*t); - if (*t == "uncompressed" || + if (*t == "uncompressed" || MetaIndexParser->Exists(CompressedMetaKey) == true) CompressionExtensions.append(*t).append(" "); } @@ -1005,9 +1001,9 @@ void pkgAcqIndex::AutoSelectCompression() if (CompressionExtensions.empty() == false) CompressionExtensions.erase(CompressionExtensions.end()-1); } + /*}}}*/ // AcqIndex::Init - defered Constructor /*{{{*/ -// --------------------------------------------------------------------- -void pkgAcqIndex::Init(string const &URI, string const &URIDesc, +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Stage = STAGE_DOWNLOAD; @@ -1036,7 +1032,7 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); if(Record) FileSize = Record->Size; - + InitByHashIfNeeded(MetaKey); } @@ -1048,8 +1044,6 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, } /*}}}*/ // AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) { // TODO: @@ -1085,7 +1079,7 @@ void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) string pkgAcqIndex::Custom600Headers() const { string Final = GetFinalFilename(); - + string msg = "\nIndex-File: true"; struct stat Buf; if (stat(Final.c_str(),&Buf) == 0) @@ -1094,10 +1088,8 @@ string pkgAcqIndex::Custom600Headers() const return msg; } /*}}}*/ -// pkgAcqIndex::Failed - getting the indexfile failed /*{{{*/ -// --------------------------------------------------------------------- -/* */ -void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +// pkgAcqIndex::Failed - getting the indexfile failed /*{{{*/ +void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) @@ -1119,9 +1111,7 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ TransactionManager->AbortTransaction(); } /*}}}*/ -// pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ std::string pkgAcqIndex::GetFinalFilename() const { std::string FinalFile = _config->FindDir("Dir::State::lists"); @@ -1130,10 +1120,8 @@ std::string pkgAcqIndex::GetFinalFilename() const FinalFile += '.' + CurrentCompressionExtension; return FinalFile; } - /*}}}*/ -// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ -// --------------------------------------------------------------------- -/* */ + /*}}}*/ +// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ void pkgAcqIndex::ReverifyAfterIMS() { // update destfile to *not* include the compression extension when doing @@ -1151,10 +1139,8 @@ void pkgAcqIndex::ReverifyAfterIMS() Desc.URI = "copy:" + FinalFile; QueueURI(Desc); } - /*}}}*/ - -// AcqIndex::ValidateFile - Validate the content of the downloaded file /*{{{*/ -// -------------------------------------------------------------------------- + /*}}}*/ +// AcqIndex::ValidateFile - Validate the content of the downloaded file /*{{{*/ bool pkgAcqIndex::ValidateFile(const std::string &FileName) { // FIXME: this can go away once we only ever download stuff that @@ -1180,7 +1166,7 @@ bool pkgAcqIndex::ValidateFile(const std::string &FileName) } return true; } - /*}}}*/ + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -1205,8 +1191,8 @@ void pkgAcqIndex::Done(string Message, break; } } - -// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ + /*}}}*/ +// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ void pkgAcqIndex::StageDownloadDone(string Message, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) @@ -1221,7 +1207,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, } Complete = true; - + // Handle the unzipd case string FileName = LookupTag(Message,"Alt-Filename"); if (FileName.empty() == false) @@ -1297,8 +1283,8 @@ void pkgAcqIndex::StageDownloadDone(string Message, SetActiveSubprocess(decompProg); } - /*}}}*/ -// pkgAcqIndex::StageDecompressDone - Final verification /*{{{*/ + /*}}}*/ +// pkgAcqIndex::StageDecompressDone - Final verification /*{{{*/ void pkgAcqIndex::StageDecompressDone(string Message, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) @@ -1318,30 +1304,28 @@ void pkgAcqIndex::StageDecompressDone(string Message, Failed(Message, Cfg); return; } - + // remove the compressed version of the file unlink(EraseFileName.c_str()); - + // Done, queue for rename on transaction finished TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); - + return; } - /*}}}*/ /*}}}*/ // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The Translation file is added to the queue */ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, - string URI,string URIDesc,string ShortDesc) + string URI,string URIDesc,string ShortDesc) : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList()) { } - /*}}}*/ -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, - pkgAcqMetaBase *TransactionManager, +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, + pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, - HashStringList const &ExpectedHashes, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) : pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser) { @@ -1352,7 +1336,6 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, } /*}}}*/ // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ -// --------------------------------------------------------------------- string pkgAcqIndexTrans::Custom600Headers() const { string Final = GetFinalFilename(); @@ -1364,8 +1347,6 @@ string pkgAcqIndexTrans::Custom600Headers() const } /*}}}*/ // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { size_t const nextExt = CompressionExtensions.find(' '); @@ -1378,9 +1359,9 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor - if (Cnf->LocalOnly == true || + if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) - { + { // Ignore this Status = StatDone; Complete = false; @@ -1391,17 +1372,13 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ -// AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ void pkgAcqMetaBase::Add(Item *I) { Transaction.push_back(I); } /*}}}*/ -// AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/ void pkgAcqMetaBase::AbortTransaction() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1426,9 +1403,7 @@ void pkgAcqMetaBase::AbortTransaction() } } /*}}}*/ -// AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/ bool pkgAcqMetaBase::TransactionHasError() { for (pkgAcquire::ItemIterator I = Transaction.begin(); @@ -1441,8 +1416,6 @@ bool pkgAcqMetaBase::TransactionHasError() } /*}}}*/ // AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqMetaBase::CommitTransaction() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1478,8 +1451,6 @@ void pkgAcqMetaBase::CommitTransaction() } /*}}}*/ // AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqMetaBase::TransactionStageCopy(Item *I, const std::string &From, const std::string &To) @@ -1489,8 +1460,6 @@ void pkgAcqMetaBase::TransactionStageCopy(Item *I, } /*}}}*/ // AcqMetaBase::TransactionStageRemoval - Sage a file for removal /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqMetaBase::TransactionStageRemoval(Item *I, const std::string &FinalFile) { @@ -1498,10 +1467,7 @@ void pkgAcqMetaBase::TransactionStageRemoval(Item *I, I->DestFile = FinalFile; } /*}}}*/ - /*{{{*/ // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ -// --------------------------------------------------------------------- -/* */ bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &RealURI, const std::string &Message) { @@ -1509,7 +1475,7 @@ bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &RealURI, // a unauthenticated state and can cleanly rollback string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - + if(FileExists(Final)) { Status = StatTransientNetworkError; @@ -1533,21 +1499,19 @@ bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &RealURI, Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); } - // gpgv method failed + // gpgv method failed ReportMirrorFailure("GPGFailure"); return false; } /*}}}*/ -// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager, string URI,string URIDesc,string ShortDesc, string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, + pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, HashStringList(), TransactionManager), RealURI(URI), MetaIndexFile(MetaIndexFile), URIDesc(URIDesc), ShortDesc(ShortDesc) @@ -1555,8 +1519,8 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - // remove any partial downloaded sig-file in partial/. - // it may confuse proxies and is too small to warrant a + // remove any partial downloaded sig-file in partial/. + // it may confuse proxies and is too small to warrant a // partial download anyway unlink(DestFile.c_str()); @@ -1592,7 +1556,7 @@ string pkgAcqMetaSig::Custom600Headers() const return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -// pkgAcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ +// pkgAcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, @@ -1612,13 +1576,12 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, } return; } - else + else { if(CheckAuthDone(Message, RealURI) == true) { std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); - TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, FinalFile); } } @@ -1718,7 +1681,7 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ Init(URIDesc, ShortDesc); } /*}}}*/ -// pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/ +// pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/ void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -1734,6 +1697,7 @@ void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) ExpectedAdditionalItems = IndexTargets->size(); QueueURI(Desc); } + /*}}}*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- string pkgAcqMetaIndex::Custom600Headers() const @@ -2018,9 +1982,7 @@ bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/ return true; } /*}}}*/ -// pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ void pkgAcqMetaIndex::Failed(string /*Message*/, pkgAcquire::MethodConfig * /*Cnf*/) { @@ -2047,11 +2009,10 @@ void pkgAcqMetaIndex::Failed(string /*Message*/, TransactionManager->AbortTransaction(); Status = StatError; return; - } + } } /*}}}*/ - -void pkgAcqMetaIndex::Finished() +void pkgAcqMetaIndex::Finished() /*{{{*/ { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "Finished: " << DestFile <TransactionHasError() == false) TransactionManager->CommitTransaction(); } - - + /*}}}*/ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &URI, string const &URIDesc, string const &ShortDesc, string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, -- cgit v1.2.3 From 04a54261afd1c99686109f102afc83346c01c930 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 6 Oct 2014 11:15:03 +0200 Subject: ensure partial dirs are 0700 and owned by _apt:root Reworks the API involved in creating and setting up the fetcher to be a bit more pleasent to look at and work with as e.g. an empty string for no lock isn't very nice. With the lock we can also stop creating all our partial directories "just in case". This way we can also be a bit more aggressive with the partial directory itself as with a lock, we know we will gone need it. --- apt-pkg/acquire.cc | 78 ++++++++++++++++++++++++++----------- apt-pkg/acquire.h | 19 ++++++--- apt-pkg/update.cc | 4 +- apt-private/private-install.cc | 6 +-- apt-private/private-update.cc | 4 +- cmdline/apt-get.cc | 18 ++------- cmdline/apt-helper.cc | 6 +-- test/integration/test-apt-get-clean | 1 + 8 files changed, 81 insertions(+), 55 deletions(-) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index ec565fcfa..9dee1b3cf 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -27,17 +27,20 @@ #include #include #include +#include + #include #include #include #include -#include - +#include +#include #include #include #include #include #include +#include #include /*}}}*/ @@ -57,8 +60,8 @@ pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NU if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; } -pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), - Configs(0), Log(Progress), ToFetch(0), +pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), + Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) { @@ -67,40 +70,69 @@ pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Wor QueueMode = QueueHost; if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; - Setup(Progress, ""); + SetLog(Progress); } /*}}}*/ -// Acquire::Setup - Delayed Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* Do everything needed to be a complete Acquire object and report the - success (or failure) back so the user knows that something is wrong… */ -bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock, - bool const createDirectories) +// Acquire::GetLock - lock directory and prepare for action /*{{{*/ +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) { - Log = Progress; + std::string const partial = parent + "partial"; + if (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false) + return false; - // check for existence and possibly create auxiliary directories - if (createDirectories == true) + if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + { + struct passwd *pw = getpwnam("_apt"); + struct group *gr = getgrnam("root"); + if (pw != NULL && gr != NULL && chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to _apt:root of directory %s failed", partial.c_str()); + } + if (chmod(partial.c_str(), 0700) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); + + return true; +} +bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) +{ + Log = Progress; + if (Lock.empty()) { string const listDir = _config->FindDir("Dir::State::lists"); - string const partialListDir = listDir + "partial/"; + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - string const partialArchivesDir = archivesDir + "partial/"; + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + return true; + } + return GetLock(Lock); +} +bool pkgAcquire::GetLock(std::string const &Lock) +{ + if (Lock.empty() == true) + return false; - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && - CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) - return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + // check for existence and possibly create auxiliary directories + string const listDir = _config->FindDir("Dir::State::lists"); + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && - CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) + if (Lock == listDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + } + if (Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); } - if (Lock.empty() == true || _config->FindB("Debug::NoLocking", false) == true) + if (_config->FindB("Debug::NoLocking", false) == true) return true; // Lock the directory this acquire object will work in - LockFD = GetLock(flCombine(Lock, "lock")); + LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 7bceb4323..f9eeb1641 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -351,17 +351,24 @@ class pkgAcquire * long as the pkgAcquire object does. * \param Lock defines a lock file that should be acquired to ensure * only one Acquire class is in action at the time or an empty string - * if no lock file should be used. - * \param createDirectories can be used to disable the creation of directories, - * e.g. if the fetcher is used with different directories later on + * if no lock file should be used. If set also all needed directories + * will be created. */ - bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = "", - bool const createDirectories = true); + APT_DEPRECATED bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = ""); void SetLog(pkgAcquireStatus *Progress) { Log = Progress; } + /** \brief acquire lock and perform directory setup + * + * \param Lock defines a lock file that should be acquired to ensure + * only one Acquire class is in action at the time or an empty string + * if no lock file should be used. If set also all needed directories + * will be created and setup. + */ + bool GetLock(std::string const &Lock); + /** \brief Construct a new pkgAcquire. */ - pkgAcquire(pkgAcquireStatus *Log) APT_DEPRECATED; + pkgAcquire(pkgAcquireStatus *Log); pkgAcquire(); /** \brief Destroy this pkgAcquire object. diff --git a/apt-pkg/update.cc b/apt-pkg/update.cc index 5d5b19626..2908a4820 100644 --- a/apt-pkg/update.cc +++ b/apt-pkg/update.cc @@ -27,8 +27,8 @@ bool ListUpdate(pkgAcquireStatus &Stat, pkgSourceList &List, int PulseInterval) { - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat, _config->FindDir("Dir::State::Lists")) == false) + pkgAcquire Fetcher(&Stat); + if (Fetcher.GetLock(_config->FindDir("Dir::State::Lists")) == false) return false; // Populate it with the source selection diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index 86ba52857..c06caeedd 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -119,14 +119,14 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety) return false; // Create the download object - pkgAcquire Fetcher; - AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); + AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); + pkgAcquire Fetcher(&Stat); if (_config->FindB("APT::Get::Print-URIs", false) == true) { // force a hashsum for compatibility reasons _config->CndSet("Acquire::ForceHash", "md5sum"); } - else if (Fetcher.Setup(&Stat, _config->FindDir("Dir::Cache::Archives")) == false) + else if (Fetcher.GetLock(_config->FindDir("Dir::Cache::Archives")) == false) return false; // Read the source list diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc index 1cf3012ed..df77ac33a 100644 --- a/apt-private/private-update.cc +++ b/apt-private/private-update.cc @@ -47,9 +47,7 @@ bool DoUpdate(CommandLine &CmdL) _config->CndSet("Acquire::ForceHash", "md5sum"); // get a fetcher - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat) == false) - return false; + pkgAcquire Fetcher(&Stat); // Populate it with the source selection and get all Indexes // (GetAll=true) diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 0cea05cb3..15696e19f 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -646,9 +646,7 @@ static bool DoDownload(CommandLine &CmdL) return false; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0)); - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat, "", false) == false) - return false; + pkgAcquire Fetcher(&Stat); pkgRecords Recs(Cache); pkgSourceList *SrcList = Cache.GetSourceList(); @@ -744,9 +742,8 @@ static bool DoSource(CommandLine &CmdL) return false; // Create the download object - AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); - pkgAcquire Fetcher; - Fetcher.SetLog(&Stat); + AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); + pkgAcquire Fetcher(&Stat); SPtrArray Dsc = new DscFile[CmdL.FileSize()]; @@ -1048,12 +1045,6 @@ static bool DoBuildDep(CommandLine &CmdL) if (_error->PendingError() == true) return false; - // Create the download object - AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat) == false) - return false; - bool StripMultiArch; string hostArch = _config->Find("APT::Get::Host-Architecture"); if (hostArch.empty() == false) @@ -1565,8 +1556,7 @@ static bool DoChangelog(CommandLine &CmdL) } AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - if (Fetcher.Setup(&Stat, "",false) == false) - return false; + Fetcher.SetLog(&Stat); bool const downOnly = _config->FindB("APT::Get::Download-Only", false); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index b89df61d6..c240008aa 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -48,11 +48,9 @@ static bool DoDownloadFile(CommandLine &CmdL) if (CmdL.FileSize() <= 2) return _error->Error(_("Must specify at least one pair url/filename")); - - pkgAcquire Fetcher; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - if (Fetcher.Setup(&Stat, "", false) == false) - return false; + pkgAcquire Fetcher(&Stat); + std::string download_uri = CmdL.FileList[1]; std::string targetfile = CmdL.FileList[2]; std::string hash; diff --git a/test/integration/test-apt-get-clean b/test/integration/test-apt-get-clean index 646ea31be..98f7c84d0 100755 --- a/test/integration/test-apt-get-clean +++ b/test/integration/test-apt-get-clean @@ -18,6 +18,7 @@ testsuccess aptget clean # generate some dirt and clean it up touch rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en +mkdir -p rootdir/var/cache/apt/archives touch rootdir/var/cache/apt/archives/foo_1_all.deb touch rootdir/var/cache/apt/archives/foo_2_all.deb touch rootdir/var/cache/apt/archives/foo_3_all.deb -- cgit v1.2.3 From 5684f71fa0f6c1b765aa53e22ca3b024c578b9c9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 6 Oct 2014 14:29:53 +0200 Subject: use _apt:root only for partial directories Using a different user for calling methods is intended to protect us from methods running amok (via remotely exploited bugs) by limiting what can be done by them. By using root:root for the final directories and just have the files in partial writeable by the methods we enhance this in sofar as a method can't modify already verified data in its parent directory anymore. As a side effect, this also clears most of the problems you could have if the final directories are shared without user-sharing or if these directories disappear as they are now again root owned and only the partial directories contain _apt owned files (usually none if apt isn't running) and the directory itself is autocreated with the right permissions. --- apt-pkg/acquire-item.cc | 133 +++++++++++++++++++------------- apt-pkg/acquire-item.h | 7 +- apt-pkg/acquire-worker.cc | 3 +- debian/apt.postinst | 15 ++-- test/integration/framework | 22 ++++-- test/integration/test-apt-get-download | 16 ++++ test/integration/test-apt-update-unauth | 20 ++--- 7 files changed, 137 insertions(+), 79 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index ad9198cba..6bfd14992 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -44,6 +44,9 @@ #include #include #include +#include +#include +#include #include /*}}}*/ @@ -62,6 +65,30 @@ static void printHashSumComparision(std::string const &URI, HashStringList const std::cerr << "\t- " << hs->toStr() << std::endl; } /*}}}*/ +static void changeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) +{ + // ensure the file is owned by root and has good permissions + struct passwd const * const pw = getpwnam(user); + struct group const * const gr = getgrnam(group); + if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + { + if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file); + } + if (chmod(file, mode) != 0) + _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file); +} +static std::string preparePartialFile(std::string const &file) +{ + std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += file; + return DestFile; +} +static std::string preparePartialFileFromURI(std::string const &uri) +{ + return preparePartialFile(URItoFileName(uri)); +} + // Acquire::Item::Item - Constructor /*{{{*/ #if __GNUC__ >= 4 @@ -178,6 +205,21 @@ bool pkgAcquire::Item::Rename(string From,string To) return true; } /*}}}*/ + +void pkgAcquire::Item::QueueURI(ItemDesc &Item) +{ + if (access(DestFile.c_str(), R_OK) == 0) + { + changeOwnerAndPermissionOfFile("preparePartialFile", DestFile.c_str(), "_apt", "root", 0600); + std::cerr << "QUEUE ITEM: " << DestFile << std::endl; + } + Owner->Enqueue(Item); +} +void pkgAcquire::Item::Dequeue() +{ + Owner->Dequeue(this); +} + bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/ { if(FileExists(DestFile)) @@ -293,8 +335,7 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, Desc.ShortDesc = Target->ShortDesc; Desc.URI = Target->URI + ".diff/Index"; - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(Desc.URI); + DestFile = preparePartialFileFromURI(Desc.URI); if(Debug) std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; @@ -454,8 +495,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ { // FIXME: make this use the method PackagesFileReadyInPartial = true; - std::string Partial = _config->FindDir("Dir::State::lists"); - Partial += "partial/" + URItoFileName(RealURI); + std::string const Partial = preparePartialFileFromURI(RealURI); FileFd From(CurrentPackagesFile, FileFd::ReadOnly); FileFd To(Partial, FileFd::WriteEmpty); @@ -585,9 +625,7 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { - - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(Target->URI); + DestFile = preparePartialFileFromURI(Target->URI); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -640,7 +678,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) } // queue for copy - PartialFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); + PartialFile = preparePartialFileFromURI(RealURI); DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); @@ -671,8 +709,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ { // calc sha1 of the just patched file - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += "partial/" + URItoFileName(RealURI); + std::string const FinalFile = preparePartialFileFromURI(RealURI); if(!FileExists(FinalFile)) { @@ -717,8 +754,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ // queue the right diff Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz"; Desc.Description = Description + " " + available_patches[0].file + string(".pdiff"); - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file); + DestFile = preparePartialFileFromURI(RealURI + ".diff/" + available_patches[0].file); if(Debug) std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; @@ -737,9 +773,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi Item::Done(Message, Size, Hashes, Cnf); // FIXME: verify this download too before feeding it to rred - - string FinalFile; - FinalFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); + std::string const FinalFile = preparePartialFileFromURI(RealURI); // success in downloading a diff, enter ApplyDiff state if(State == StateFetchDiff) @@ -800,10 +834,6 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { - - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(Target->URI); - Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); RealURI = Target->URI; @@ -813,8 +843,8 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, Desc.URI = RealURI + ".diff/" + patch.file + ".gz"; Desc.Description = Description + " " + patch.file + string(".pdiff"); - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI + ".diff/" + patch.file); + + DestFile = preparePartialFileFromURI(RealURI + ".diff/" + patch.file); if(Debug) std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; @@ -852,8 +882,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Item::Done(Message,Size,Hashes,Cnf); // FIXME: verify download before feeding it to rred - - string const FinalFile = _config->FindDir("Dir::State::lists") + "partial/" + URItoFileName(RealURI); + string const FinalFile = preparePartialFileFromURI(RealURI); if (State == StateFetchDiff) { @@ -909,8 +938,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri for (std::vector::const_iterator I = allPatches->begin(); I != allPatches->end(); ++I) { - std::string PartialFile = _config->FindDir("Dir::State::lists"); - PartialFile += "partial/" + URItoFileName(RealURI); + std::string const PartialFile = preparePartialFileFromURI(RealURI); std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz"; std::cerr << patch << std::endl; unlink(patch.c_str()); @@ -1008,8 +1036,7 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, { Stage = STAGE_DOWNLOAD; - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile = preparePartialFileFromURI(URI); CurrentCompressionExtension = CompressionExtensions.substr(0, CompressionExtensions.find(' ')); if (CurrentCompressionExtension == "uncompressed") @@ -1126,8 +1153,7 @@ void pkgAcqIndex::ReverifyAfterIMS() { // update destfile to *not* include the compression extension when doing // a reverify (as its uncompressed on disk already) - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); + DestFile = preparePartialFileFromURI(RealURI); // adjust DestFile if its compressed on disk if (_config->FindB("Acquire::GzipIndexes",false) == true) @@ -1253,8 +1279,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, // If we have compressed indexes enabled, queue for hash verification if (_config->FindB("Acquire::GzipIndexes",false)) { - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI) + '.' + CurrentCompressionExtension; + DestFile = preparePartialFileFromURI(RealURI + '.' + CurrentCompressionExtension); EraseFileName = ""; Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FileName; @@ -1395,9 +1420,7 @@ void pkgAcqMetaBase::AbortTransaction() (*I)->Status = pkgAcquire::Item::StatDone; // kill files in partial - string PartialFile = _config->FindDir("Dir::State::lists"); - PartialFile += "partial/"; - PartialFile += flNotDir((*I)->DestFile); + std::string const PartialFile = preparePartialFile(flNotDir((*I)->DestFile)); if(FileExists(PartialFile)) Rename(PartialFile, PartialFile + ".FAILED"); } @@ -1428,19 +1451,18 @@ void pkgAcqMetaBase::CommitTransaction() { if((*I)->PartialFile != "") { - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "mv " - << (*I)->PartialFile << " -> " - << (*I)->DestFile << " " - << (*I)->DescURI() - << std::endl; - Rename((*I)->PartialFile, (*I)->DestFile); - chmod((*I)->DestFile.c_str(),0644); + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "mv " << (*I)->PartialFile << " -> "<< (*I)->DestFile << " " + << (*I)->DescURI() << std::endl; + + Rename((*I)->PartialFile, (*I)->DestFile); + changeOwnerAndPermissionOfFile("CommitTransaction", (*I)->DestFile.c_str(), "root", "root", 0644); + } else { if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "rm " + std::clog << "rm " << (*I)->DestFile - << " " + << " " << (*I)->DescURI() << std::endl; unlink((*I)->DestFile.c_str()); @@ -1625,8 +1647,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // this ensures that any file in the lists/ dir is removed by the // transaction - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); + DestFile = preparePartialFileFromURI(RealURI); TransactionManager->TransactionStageRemoval(this, DestFile); // only allow going further if the users explicitely wants it @@ -1684,8 +1705,7 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ // pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/ void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) { - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); + DestFile = preparePartialFileFromURI(RealURI); // Create the item Desc.Description = URIDesc; @@ -1770,7 +1790,7 @@ bool pkgAcqMetaBase::CheckAuthDone(string Message, const string &RealURI) /*{{{* return true; } /*}}}*/ - /*{{{*/ +// pkgAcqMetaBase::QueueForSignatureVerify /*{{{*/ void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile, const std::string &MetaIndexFileSignature) { @@ -1781,7 +1801,7 @@ void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile, SetActiveSubprocess("gpgv"); } /*}}}*/ - /*{{{*/ +// pkgAcqMetaBase::CheckDownloadDone /*{{{*/ bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message, const std::string &RealURI) { @@ -2332,7 +2352,10 @@ bool pkgAcqArchive::QueueNext() if ((unsigned long long)Buf.st_size > Version->Size) unlink(DestFile.c_str()); else + { PartialSize = Buf.st_size; + changeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext", FinalFile.c_str(), "_apt", "root", 0600); + } } // Disables download of archives - useful if no real installation follows, @@ -2388,21 +2411,20 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList return; } - Complete = true; - // Reference filename if (FileName != DestFile) { StoreFilename = DestFile = FileName; Local = true; + Complete = true; return; } - + // Done, move it into position string FinalFile = _config->FindDir("Dir::Cache::Archives"); FinalFile += flNotDir(StoreFilename); Rename(DestFile,FinalFile); - + changeOwnerAndPermissionOfFile("pkgAcqArchive::Done", FinalFile.c_str(), "root", "root", 0644); StoreFilename = DestFile = FinalFile; Complete = true; } @@ -2498,7 +2520,10 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashe if ((Size > 0) && (unsigned long long)Buf.st_size > Size) unlink(DestFile.c_str()); else + { PartialSize = Buf.st_size; + changeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), "_apt", "root", 0600); + } } QueueURI(Desc); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 02b8c13e8..a3388ca3e 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -75,12 +75,11 @@ class pkgAcquire::Item : public WeakPointable * \param Item Metadata about this item (its URI and * description). */ - inline void QueueURI(ItemDesc &Item) - {Owner->Enqueue(Item);}; + void QueueURI(ItemDesc &Item); /** \brief Remove this item from its owner's queue. */ - inline void Dequeue() {Owner->Dequeue(this);}; - + void Dequeue(); + /** \brief Rename a file without modifying its timestamp. * * Many item methods call this as their final action. diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 54be8e99f..4a357bdab 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -371,7 +371,8 @@ bool pkgAcquire::Worker::RunMessages() { if (Itm == 0) { - _error->Error("Method gave invalid 400 URI Failure message"); + std::string const msg = LookupTag(Message,"Message"); + _error->Error("Method gave invalid 400 URI Failure message: %s", msg.c_str()); break; } diff --git a/debian/apt.postinst b/debian/apt.postinst index 01f78a1dd..b8f3edbe5 100755 --- a/debian/apt.postinst +++ b/debian/apt.postinst @@ -35,12 +35,15 @@ case "$1" in fi fi - # add unprivileged user for the apt methods - adduser --force-badname --system -home /var/empty \ - --no-create-home --quiet _apt || true - chown -R _apt:root \ - /var/lib/apt/lists \ - /var/cache/apt/archives + # add unprivileged user for the apt methods + adduser --force-badname --system -home /var/empty \ + --no-create-home --quiet _apt || true + + # deal with upgrades from experimental + if dpkg --compare-versions "$2" 'eq' '1.1~exp3'; then + # libapt will setup partial/ at runtime + chown -R root:root /var/lib/apt/lists /var/cache/apt/archives || true + fi # ensure tighter permissons on the logs, see LP: #975199 if dpkg --compare-versions "$2" lt-nl 0.9.7.7; then diff --git a/test/integration/framework b/test/integration/framework index e83606fae..688a1abf2 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -164,9 +164,10 @@ addtrap() { setupenvironment() { TMPWORKINGDIRECTORY=$(mktemp -d) - TESTDIRECTORY=$(readlink -f $(dirname $0)) + addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;" msgninfo "Preparing environment for ${CCMD}$(basename $0)${CINFO} in ${TMPWORKINGDIRECTORY}… " + TESTDIRECTORY=$(readlink -f $(dirname $0)) # allow overriding the default BUILDDIR location BUILDDIRECTORY=${APT_INTEGRATION_TESTS_BUILD_DIR:-"${TESTDIRECTORY}/../../build/bin"} LIBRARYPATH=${APT_INTEGRATION_TESTS_LIBRARY_PATH:-"${BUILDDIRECTORY}"} @@ -177,7 +178,6 @@ setupenvironment() { test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first" # ----- - addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;" cd $TMPWORKINGDIRECTORY mkdir rootdir aptarchive keys cd rootdir @@ -210,6 +210,7 @@ setupenvironment() { cp "${TESTDIRECTORY}/${SOURCESSFILE}" aptarchive/Sources fi cp $(find $TESTDIRECTORY -name '*.pub' -o -name '*.sec') keys/ + chmod 644 $(find keys -name '*.pub' -o -name '*.sec') ln -s ${TMPWORKINGDIRECTORY}/keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf @@ -837,9 +838,7 @@ setupaptarchive() { fi signreleasefiles if [ "$1" != '--no-update' ]; then - msgninfo "\tSync APT's cache with the archive… " - aptget update -qq - msgdone "info" + testsuccess aptget update -o Debug::pkgAcquire::Worker=true -o Debug::Acquire::gpgv=true fi } @@ -1175,6 +1174,19 @@ testfailure() { fi } +testaccessrights() { + msgtest "Test that file $1 has access rights set to" "$2" + if [ "$2" = "$(stat --format '%a' "$1")" ]; then + msgpass + else + echo >&2 + ls -l >&2 "$1" + echo -n >&2 "stat(1) reports access rights: " + stat --format '%a' "$1" + msgfail + fi +} + testwebserverlaststatuscode() { local DOWNLOG='rootdir/tmp/webserverstatus-testfile.log' local STATUS='rootdir/tmp/webserverstatus-statusfile.log' diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download index 58ed44f8f..0514542b3 100755 --- a/test/integration/test-apt-get-download +++ b/test/integration/test-apt-get-download @@ -11,8 +11,23 @@ buildsimplenativepackage 'apt' 'all' '1.0' 'stable' buildsimplenativepackage 'apt' 'all' '2.0' 'unstable' insertinstalledpackage 'vrms' 'all' '1.0' +umask 0027 + setupaptarchive +# apt-ftparchive knows how to chmod files +find aptarchive/dists -name '*Packages*' -type f | while read file; do + testaccessrights "$file" '644' +done +# created by the framework without special care +find aptarchive/dists -name '*Release*' -type f | while read file; do + testaccessrights "$file" '640' +done +# all copied files are properly chmodded +find rootdir/var/lib/apt/lists -type f | while read file; do + testaccessrights "$file" '644' +done + testdownload() { local APT="$2" if [ -n "$3" ]; then @@ -65,6 +80,7 @@ testsuccess aptget update # test with already stored deb testsuccess aptget install -d apt testsuccess test -s rootdir/var/cache/apt/archives/apt_2.0_all.deb +testaccessrights 'aptarchive/pool/apt_2.0_all.deb' '644' mv aptarchive/pool/apt_2.0_all.deb aptarchive/pool/apt_2.0_all.deb.gone testdownload apt_2.0_all.deb apt mv aptarchive/pool/apt_2.0_all.deb.gone aptarchive/pool/apt_2.0_all.deb diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index cf5195024..b7ccd6cf3 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -27,7 +27,7 @@ runtest() { find rootdir/var/lib/apt/lists/ -type f | xargs rm -f rm -f aptarchive/dists/unstable/*Release* - aptget update -qq --allow-insecure-repositories + testsuccess aptget update -qq --allow-insecure-repositories # FIXME: this really shouldn't be needed rm -f rootdir/var/lib/apt/lists/partial/* @@ -41,7 +41,6 @@ runtest() { aptarchive/dists/unstable/main/binary-i386/Packages.uncompressed # and ensure we re-check the downloaded data - msgtest "Check rollback on going from unauth -> auth" # change the local packages file PKGS=$(ls rootdir/var/lib/apt/lists/*Packages*) @@ -49,18 +48,22 @@ runtest() { ls rootdir/var/lib/apt/lists/ > lists.before # update and ensure all is reverted on the hashsum failure - aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0 > output.log 2>&1 || true + testfailure aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0 # ensure we have before what we have after + msgtest 'Check rollback on going from' 'unauth -> auth' ls rootdir/var/lib/apt/lists/ > lists.after - if diff -u lists.before lists.after; then + if cmp lists.before lists.after; then msgpass else - cat output.log - msgfail + echo >&2 '### Output of previous apt-get update ###' + cat >&2 rootdir/tmp/testfailure.output + echo >&2 '### Changes in the lists-directory: ###' + diff -u >&2 lists.before lists.after + msgfail fi - # move uncompressed back for release file + # move uncompressed back for release file mv aptarchive/dists/unstable/main/binary-i386/Packages.uncompressed \ aptarchive/dists/unstable/main/binary-i386/Packages } @@ -72,6 +75,5 @@ for COMPRESSEDINDEXES in 'false' 'true'; do else msgmsg 'Run tests with GzipIndexes disabled' fi - - runtest + runtest done -- cgit v1.2.3 From 4dbfe436c60880f2625e4d3a9d0127a83dd6276e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 7 Oct 2014 01:46:30 +0200 Subject: display errortext for all Err as well as Ign logs consistently using Item::Failed in all specializec classes helps setting up some information bits otherwise unset, so some errors had an empty reason as an error. Ign is upgraded to display the error message we ignored to further help in understanding what happens. --- apt-pkg/acquire-item.cc | 77 ++++++++-------- apt-private/acqprogress.cc | 2 + test/integration/framework | 13 ++- .../integration/test-apt-get-update-unauth-warning | 7 +- test/integration/test-apt-update-ims | 5 +- test/integration/test-apt-update-nofallback | 2 +- test/integration/test-apt-update-rollback | 101 ++++++++++----------- .../test-bug-595691-empty-and-broken-archive-files | 2 +- 8 files changed, 108 insertions(+), 101 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 6bfd14992..8fafca33b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -125,7 +125,6 @@ pkgAcquire::Item::~Item() fetch this object */ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - Status = StatIdle; if(ErrorText == "") ErrorText = LookupTag(Message,"Message"); UsedMirror = LookupTag(Message,"UsedMirror"); @@ -134,7 +133,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* This indicates that the file is not available right now but might be sometime later. If we do a retry cycle then this should be retried [CDROMs] */ - if (Cnf->LocalOnly == true && + if (Cnf != NULL && Cnf->LocalOnly == true && StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) { Status = StatIdle; @@ -143,8 +142,11 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } Status = StatError; + Complete = false; Dequeue(); - } + } + else + Status = StatIdle; // report mirror failure back to LP if we actually use a mirror string FailReason = LookupTag(Message, "FailReason"); @@ -208,11 +210,8 @@ bool pkgAcquire::Item::Rename(string From,string To) void pkgAcquire::Item::QueueURI(ItemDesc &Item) { - if (access(DestFile.c_str(), R_OK) == 0) - { + if (RealFileExists(DestFile)) changeOwnerAndPermissionOfFile("preparePartialFile", DestFile.c_str(), "_apt", "root", 0600); - std::cerr << "QUEUE ITEM: " << DestFile << std::endl; - } Owner->Enqueue(Item); } void pkgAcquire::Item::Dequeue() @@ -559,7 +558,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ return false; } /*}}}*/ -void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ +void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/ { if(Debug) std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl @@ -567,9 +566,8 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); - Complete = false; + Item::Failed(Message,Cnf); Status = StatDone; - Dequeue(); } /*}}}*/ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -596,7 +594,7 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList } if(!ParseDiffIndex(DestFile)) - return Failed("", NULL); + return Failed("Message: Couldn't parse pdiff index", Cnf); // queue for final move string FinalFile; @@ -713,7 +711,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ if(!FileExists(FinalFile)) { - Failed("No FinalFile " + FinalFile + " available", NULL); + Failed("Message: No FinalFile " + FinalFile + " available", NULL); return false; } @@ -747,7 +745,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ // error checking and falling back if no patch was found if(available_patches.empty() == true) { - Failed("No patches available", NULL); + Failed("Message: No patches available", NULL); return false; } @@ -852,13 +850,13 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, QueueURI(Desc); } /*}}}*/ -void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ +void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/ { if(Debug) std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl; - Complete = false; + + Item::Failed(Message,Cnf); Status = StatDone; - Dequeue(); // check if we are the first to fail, otherwise we are done here State = StateDoneDiff; @@ -1284,7 +1282,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FileName; QueueURI(Desc); - + SetActiveSubprocess("copy"); return; } @@ -1305,7 +1303,6 @@ void pkgAcqIndex::StageDownloadDone(string Message, DestFile += ".decomp"; Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - SetActiveSubprocess(decompProg); } /*}}}*/ @@ -1383,18 +1380,15 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) return; } + Item::Failed(Message,Cnf); + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { // Ignore this Status = StatDone; - Complete = false; - Dequeue(); - return; } - - Item::Failed(Message,Cnf); } /*}}}*/ // AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ @@ -1639,7 +1633,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } else { _error->Error("%s", downgrade_msg.c_str()); Rename(MetaIndexFile, MetaIndexFile+".FAILED"); - Status = pkgAcquire::Item::StatError; + Item::Failed("Message: " + downgrade_msg, Cnf); TransactionManager->AbortTransaction(); return; } @@ -1663,17 +1657,15 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ _error->Warning("Use --allow-insecure-repositories to force the update"); } + Item::Failed(Message,Cnf); + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor - if (Cnf->LocalOnly == true || + if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) - { + { // Ignore this Status = StatDone; - Complete = false; - Dequeue(); - return; } - Item::Failed(Message,Cnf); } /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ @@ -2003,9 +1995,12 @@ bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/ } /*}}}*/ // pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ -void pkgAcqMetaIndex::Failed(string /*Message*/, - pkgAcquire::MethodConfig * /*Cnf*/) +void pkgAcqMetaIndex::Failed(string Message, + pkgAcquire::MethodConfig * Cnf) { + pkgAcquire::Item::Failed(Message, Cnf); + Status = StatDone; + string FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); _error->Warning(_("The repository '%s' does not have a Release file. " @@ -2115,6 +2110,8 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long /*Size*/, /*}}}*/ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { + Item::Failed(Message, Cnf); + // we failed, we will not get additional items from this method ExpectedAdditionalItems = 0; @@ -2126,14 +2123,12 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile.append(URItoFileName(RealURI)); TransactionManager->TransactionStageRemoval(this, FinalFile); + Status = StatDone; new pkgAcqMetaIndex(Owner, TransactionManager, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); - if (Cnf->LocalOnly == true || - StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) - Dequeue(); } else { @@ -2149,9 +2144,11 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* // only allow going further if the users explicitely wants it if(_config->FindB("Acquire::AllowInsecureRepositories") == true) { + Status = StatDone; + /* Always move the meta index, even if gpgv failed. This ensures * that PackageFile objects are correctly filled in */ - if (FileExists(DestFile)) + if (FileExists(DestFile)) { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); @@ -2161,19 +2158,17 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* "Release"); FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, "Release"); - - + // Done, queue for rename on transaction finished TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); } QueueIndexes(false); } else { - // warn if the repository is unsinged + // warn if the repository is unsigned _error->Warning("Use --allow-insecure-repositories to force the update"); TransactionManager->AbortTransaction(); Status = StatError; - return; - } + } } } /*}}}*/ diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc index d6ce192ad..aa88d5334 100644 --- a/apt-private/acqprogress.cc +++ b/apt-private/acqprogress.cc @@ -117,6 +117,8 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm) if (Itm.Owner->Status == pkgAcquire::Item::StatDone) { cout << _("Ign ") << Itm.Description << endl; + if (Itm.Owner->ErrorText.empty() == false) + cout << " " << Itm.Owner->ErrorText << endl; } else { diff --git a/test/integration/framework b/test/integration/framework index 688a1abf2..29e5fafe6 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -715,7 +715,7 @@ buildaptarchivefromincoming() { aptftparchive -qq generate ftparchive.conf cd - > /dev/null msgdone "info" - generatereleasefiles + generatereleasefiles "$@" } buildaptarchivefromfiles() { @@ -830,14 +830,19 @@ setupflataptarchive() { } setupaptarchive() { - buildaptarchive + local NOUPDATE=0 + if [ "$1" = '--no-update' ]; then + NOUPDATE=1 + shift + fi + buildaptarchive "$@" if [ -e aptarchive/dists ]; then setupdistsaptarchive else setupflataptarchive fi - signreleasefiles - if [ "$1" != '--no-update' ]; then + signreleasefiles 'Joe Sixpack' "$@" + if [ "1" != "$NOUPDATE" ]; then testsuccess aptget update -o Debug::pkgAcquire::Worker=true -o Debug::Acquire::gpgv=true fi } diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 27160b5f9..8e212a3c4 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -20,11 +20,12 @@ rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated files leads to warning testequal "Ign file: unstable InRelease + File not found Err file: unstable Release - + File not found W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository. W: Use --allow-insecure-repositories to force the update -W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release +W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release File not found E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update @@ -34,7 +35,9 @@ testequal "partial" ls rootdir/var/lib/apt/lists # allow override testequal "Ign file: unstable InRelease + File not found Ign file: unstable Release + File not found Reading package lists... W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories # ensure we can not install the package diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 61b808b0f..8aa5a7262 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -44,8 +44,9 @@ echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex runtest msgmsg "Release/Release.gpg" -# with Release/Release.gpg +# with Release/Release.gpg EXPECT="Ign http://localhost:8080 unstable InRelease + 404 Not Found Hit http://localhost:8080 unstable Release Hit http://localhost:8080 unstable Release.gpg Hit http://localhost:8080 unstable/main Sources @@ -65,8 +66,10 @@ runtest # no Release.gpg or InRelease msgmsg "Release only" EXPECT="Ign http://localhost:8080 unstable InRelease + 404 Not Found Hit http://localhost:8080 unstable Release Ign http://localhost:8080 unstable Release.gpg + 404 Not Found Hit http://localhost:8080 unstable/main Sources Hit http://localhost:8080 unstable/main amd64 Packages Hit http://localhost:8080 unstable/main Translation-en diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback index c400dcc36..321472c2e 100755 --- a/test/integration/test-apt-update-nofallback +++ b/test/integration/test-apt-update-nofallback @@ -161,7 +161,7 @@ test_inrelease_to_invalid_inrelease() testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) -W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease +W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index ee8bc6926..5b9c200fe 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -19,44 +19,44 @@ create_fresh_archive() insertpackage 'unstable' 'old' 'all' '1.0' - setupaptarchive + setupaptarchive --no-update } add_new_package() { insertpackage "unstable" "new" "all" "1.0" insertsource "unstable" "new" "all" "1.0" - setupaptarchive --no-update - - avoid_ims_hit + setupaptarchive --no-update "$@" } break_repository_sources_index() { - printf "xxx" > $APTARCHIVE/dists/unstable/main/source/Sources - gzip -c $APTARCHIVE/dists/unstable/main/source/Sources > \ - $APTARCHIVE/dists/unstable/main/source/Sources.gz - avoid_ims_hit + printf 'xxx' > $APTARCHIVE/dists/unstable/main/source/Sources + compressfile "$APTARCHIVE/dists/unstable/main/source/Sources" "$@" } -test_inrelease_to_new_inrelease() { - msgmsg "Test InRelease to new InRelease works fine" +start_with_good_inrelease() { create_fresh_archive + testsuccess aptget update testequal "old/unstable 1.0 all" apt list -q +} - add_new_package +test_inrelease_to_new_inrelease() { + msgmsg 'Test InRelease to new InRelease works fine' + start_with_good_inrelease + add_new_package '+1hour' testsuccess aptget update -o Debug::Acquire::Transaction=1 - testequal "new/unstable 1.0 all old/unstable 1.0 all" apt list -q } test_inrelease_to_broken_hash_reverts_all() { - msgmsg "Test InRelease to broken InRelease reverts everything" - create_fresh_archive - add_new_package + msgmsg 'Test InRelease to broken InRelease reverts everything' + start_with_good_inrelease + + add_new_package '+1hour' # break the Sources file - break_repository_sources_index + break_repository_sources_index '+1hour' # test the error condition testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources Hash Sum mismatch @@ -66,14 +66,14 @@ E: Some index files failed to download. They have been ignored, or old ones used testequal "E: Unable to locate package new" aptget install new -s -qq } -test_inreleae_to_valid_release() { - msgmsg "Test InRelease to valid Release" - create_fresh_archive - add_new_package - # switch to a unsinged repo now +test_inrelease_to_valid_release() { + msgmsg 'Test InRelease to valid Release' + start_with_good_inrelease + + add_new_package '+1hour' + # switch to a unsigned repo now rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg - avoid_ims_hit # update fails testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq @@ -85,16 +85,17 @@ test_inreleae_to_valid_release() { testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release } -test_inreleae_to_release_reverts_all() { - msgmsg "Test InRelease to broken Release reverts everything" - create_fresh_archive +test_inrelease_to_release_reverts_all() { + msgmsg 'Test InRelease to broken Release reverts everything' + start_with_good_inrelease - # switch to a unsinged repo now - add_new_package + # switch to a unsigned repo now + add_new_package '+1hour' rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg + # break it - break_repository_sources_index + break_repository_sources_index '+1hour' # ensure error testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1 @@ -107,21 +108,19 @@ test_inreleae_to_release_reverts_all() { } test_unauthenticated_to_invalid_inrelease() { - msgmsg "Test UnAuthenticated to invalid InRelease reverts everything" + msgmsg 'Test UnAuthenticated to invalid InRelease reverts everything' create_fresh_archive - rm -rf rootdir/var/lib/apt/lists/* rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg - avoid_ims_hit - - testsuccess aptget update -qq --allow-insecure-repositories + + testsuccess aptget update --allow-insecure-repositories testequal "WARNING: The following packages cannot be authenticated! old E: There are problems and -y was used without --force-yes" aptget install -qq -y old - + # go to authenticated but not correct - add_new_package - break_repository_sources_index + add_new_package '+1hour' + break_repository_sources_index '+1hour' testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources Hash Sum mismatch @@ -134,14 +133,14 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y } test_inrelease_to_unauth_inrelease() { - msgmsg "Test InRelease to InRelease without sig" - create_fresh_archive - signreleasefiles 'Marvin Paranoid' - avoid_ims_hit - + msgmsg 'Test InRelease to InRelease without good sig' + start_with_good_inrelease + + signreleasefiles 'Marvin Paranoid' '+1hour' + testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2 -W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease +W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2 W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq @@ -150,13 +149,13 @@ W: Some index files failed to download. They have been ignored, or old ones used test_inrelease_to_broken_gzip() { msgmsg "Test InRelease to broken gzip" - create_fresh_archive - # append junk at the end of the gzip, this + start_with_good_inrelease + + # append junk at the end of the compressed file echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz - # remove uncompressed file, otherwise apt will just fallback fetching - # that + touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz + # remove uncompressed file to avoid fallback rm $APTARCHIVE/dists/unstable/main/source/Sources - avoid_ims_hit testfailure aptget update } @@ -174,7 +173,7 @@ ROOTDIR=${TMPWORKINGDIRECTORY}/rootdir APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" # test the following cases: -# - InRelease -> broken InRelease revert to previous state +# - InRelease -> broken InRelease revert to previous state # - empty lists dir and broken remote leaves nothing on the system # - InRelease -> hashsum mismatch for one file reverts all files to previous state # - Release/Release.gpg -> hashsum mismatch @@ -184,13 +183,13 @@ APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )" # - unauthenticated -> invalid InRelease # stuff to do: -# - ims-hit +# - ims-hit # - gzip-index tests test_inrelease_to_new_inrelease test_inrelease_to_broken_hash_reverts_all -test_inreleae_to_valid_release -test_inreleae_to_release_reverts_all +test_inrelease_to_valid_release +test_inrelease_to_release_reverts_all test_unauthenticated_to_invalid_inrelease test_inrelease_to_unauth_inrelease test_inrelease_to_broken_gzip diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index 683c174bd..fedf82c92 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -13,7 +13,7 @@ setupflataptarchive testaptgetupdate() { rm -rf rootdir/var/lib/apt aptget update 2>> testaptgetupdate.diff >> testaptgetupdate.diff || true - sed -i -e '/Ign / d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff + sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff GIVEN="$1" shift msgtest "Test for correctness of" "apt-get update with $*" -- cgit v1.2.3 From ea7682a0d91fee638eef06b63045b1499ddf50ac Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 14:01:35 +0200 Subject: UpperCase some functions for consistency changeOwnerAndPermissionOfFile->ChangeOwnerAndPermissionOfFile preparePartialFile->GetPartialFileName preparePartialFileFromURI->GetPartialFileNameFromURI Git-Dch: ignore --- apt-pkg/acquire-item.cc | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 8fafca33b..9c1609048 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -65,7 +65,7 @@ static void printHashSumComparision(std::string const &URI, HashStringList const std::cerr << "\t- " << hs->toStr() << std::endl; } /*}}}*/ -static void changeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) +static void ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) { // ensure the file is owned by root and has good permissions struct passwd const * const pw = getpwnam(user); @@ -78,15 +78,15 @@ static void changeOwnerAndPermissionOfFile(char const * const requester, char co if (chmod(file, mode) != 0) _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file); } -static std::string preparePartialFile(std::string const &file) +static std::string GetPartialFileName(std::string const &file) { std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += file; return DestFile; } -static std::string preparePartialFileFromURI(std::string const &uri) +static std::string GetPartialFileNameFromURI(std::string const &uri) { - return preparePartialFile(URItoFileName(uri)); + return GetPartialFileName(URItoFileName(uri)); } @@ -211,7 +211,7 @@ bool pkgAcquire::Item::Rename(string From,string To) void pkgAcquire::Item::QueueURI(ItemDesc &Item) { if (RealFileExists(DestFile)) - changeOwnerAndPermissionOfFile("preparePartialFile", DestFile.c_str(), "_apt", "root", 0600); + ChangeOwnerAndPermissionOfFile("GetPartialFileName", DestFile.c_str(), "_apt", "root", 0600); Owner->Enqueue(Item); } void pkgAcquire::Item::Dequeue() @@ -334,7 +334,7 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, Desc.ShortDesc = Target->ShortDesc; Desc.URI = Target->URI + ".diff/Index"; - DestFile = preparePartialFileFromURI(Desc.URI); + DestFile = GetPartialFileNameFromURI(Desc.URI); if(Debug) std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; @@ -494,7 +494,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ { // FIXME: make this use the method PackagesFileReadyInPartial = true; - std::string const Partial = preparePartialFileFromURI(RealURI); + std::string const Partial = GetPartialFileNameFromURI(RealURI); FileFd From(CurrentPackagesFile, FileFd::ReadOnly); FileFd To(Partial, FileFd::WriteEmpty); @@ -623,7 +623,7 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { - DestFile = preparePartialFileFromURI(Target->URI); + DestFile = GetPartialFileNameFromURI(Target->URI); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -676,7 +676,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) } // queue for copy - PartialFile = preparePartialFileFromURI(RealURI); + PartialFile = GetPartialFileNameFromURI(RealURI); DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); @@ -707,7 +707,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ { // calc sha1 of the just patched file - std::string const FinalFile = preparePartialFileFromURI(RealURI); + std::string const FinalFile = GetPartialFileNameFromURI(RealURI); if(!FileExists(FinalFile)) { @@ -752,7 +752,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ // queue the right diff Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz"; Desc.Description = Description + " " + available_patches[0].file + string(".pdiff"); - DestFile = preparePartialFileFromURI(RealURI + ".diff/" + available_patches[0].file); + DestFile = GetPartialFileNameFromURI(RealURI + ".diff/" + available_patches[0].file); if(Debug) std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; @@ -771,7 +771,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi Item::Done(Message, Size, Hashes, Cnf); // FIXME: verify this download too before feeding it to rred - std::string const FinalFile = preparePartialFileFromURI(RealURI); + std::string const FinalFile = GetPartialFileNameFromURI(RealURI); // success in downloading a diff, enter ApplyDiff state if(State == StateFetchDiff) @@ -842,7 +842,7 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, Desc.URI = RealURI + ".diff/" + patch.file + ".gz"; Desc.Description = Description + " " + patch.file + string(".pdiff"); - DestFile = preparePartialFileFromURI(RealURI + ".diff/" + patch.file); + DestFile = GetPartialFileNameFromURI(RealURI + ".diff/" + patch.file); if(Debug) std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; @@ -880,7 +880,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Item::Done(Message,Size,Hashes,Cnf); // FIXME: verify download before feeding it to rred - string const FinalFile = preparePartialFileFromURI(RealURI); + string const FinalFile = GetPartialFileNameFromURI(RealURI); if (State == StateFetchDiff) { @@ -936,7 +936,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri for (std::vector::const_iterator I = allPatches->begin(); I != allPatches->end(); ++I) { - std::string const PartialFile = preparePartialFileFromURI(RealURI); + std::string const PartialFile = GetPartialFileNameFromURI(RealURI); std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz"; std::cerr << patch << std::endl; unlink(patch.c_str()); @@ -1034,7 +1034,7 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, { Stage = STAGE_DOWNLOAD; - DestFile = preparePartialFileFromURI(URI); + DestFile = GetPartialFileNameFromURI(URI); CurrentCompressionExtension = CompressionExtensions.substr(0, CompressionExtensions.find(' ')); if (CurrentCompressionExtension == "uncompressed") @@ -1151,7 +1151,7 @@ void pkgAcqIndex::ReverifyAfterIMS() { // update destfile to *not* include the compression extension when doing // a reverify (as its uncompressed on disk already) - DestFile = preparePartialFileFromURI(RealURI); + DestFile = GetPartialFileNameFromURI(RealURI); // adjust DestFile if its compressed on disk if (_config->FindB("Acquire::GzipIndexes",false) == true) @@ -1277,7 +1277,7 @@ void pkgAcqIndex::StageDownloadDone(string Message, // If we have compressed indexes enabled, queue for hash verification if (_config->FindB("Acquire::GzipIndexes",false)) { - DestFile = preparePartialFileFromURI(RealURI + '.' + CurrentCompressionExtension); + DestFile = GetPartialFileNameFromURI(RealURI + '.' + CurrentCompressionExtension); EraseFileName = ""; Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FileName; @@ -1414,7 +1414,7 @@ void pkgAcqMetaBase::AbortTransaction() (*I)->Status = pkgAcquire::Item::StatDone; // kill files in partial - std::string const PartialFile = preparePartialFile(flNotDir((*I)->DestFile)); + std::string const PartialFile = GetPartialFileName(flNotDir((*I)->DestFile)); if(FileExists(PartialFile)) Rename(PartialFile, PartialFile + ".FAILED"); } @@ -1450,7 +1450,7 @@ void pkgAcqMetaBase::CommitTransaction() << (*I)->DescURI() << std::endl; Rename((*I)->PartialFile, (*I)->DestFile); - changeOwnerAndPermissionOfFile("CommitTransaction", (*I)->DestFile.c_str(), "root", "root", 0644); + ChangeOwnerAndPermissionOfFile("CommitTransaction", (*I)->DestFile.c_str(), "root", "root", 0644); } else { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1641,7 +1641,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // this ensures that any file in the lists/ dir is removed by the // transaction - DestFile = preparePartialFileFromURI(RealURI); + DestFile = GetPartialFileNameFromURI(RealURI); TransactionManager->TransactionStageRemoval(this, DestFile); // only allow going further if the users explicitely wants it @@ -1697,7 +1697,7 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ // pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/ void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) { - DestFile = preparePartialFileFromURI(RealURI); + DestFile = GetPartialFileNameFromURI(RealURI); // Create the item Desc.Description = URIDesc; @@ -2349,7 +2349,7 @@ bool pkgAcqArchive::QueueNext() else { PartialSize = Buf.st_size; - changeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext", FinalFile.c_str(), "_apt", "root", 0600); + ChangeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext", FinalFile.c_str(), "_apt", "root", 0600); } } @@ -2419,7 +2419,7 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList string FinalFile = _config->FindDir("Dir::Cache::Archives"); FinalFile += flNotDir(StoreFilename); Rename(DestFile,FinalFile); - changeOwnerAndPermissionOfFile("pkgAcqArchive::Done", FinalFile.c_str(), "root", "root", 0644); + ChangeOwnerAndPermissionOfFile("pkgAcqArchive::Done", FinalFile.c_str(), "root", "root", 0644); StoreFilename = DestFile = FinalFile; Complete = true; } @@ -2517,7 +2517,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashe else { PartialSize = Buf.st_size; - changeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), "_apt", "root", 0600); + ChangeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), "_apt", "root", 0600); } } -- cgit v1.2.3 From d6cf2345a35896448e19bfb294ffe66faab00f86 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 7 Oct 2014 20:51:07 +0200 Subject: don't show ErrorText for Ign by default Some distributions (or repositories) do not have as much "Ign-discipline" as I would like to, so that could be pretty distracting for our users if enabled by default. It is handy for testcases though. Git-Dch: Ignore --- apt-private/acqprogress.cc | 3 ++- test/integration/framework | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc index aa88d5334..14a53eacb 100644 --- a/apt-private/acqprogress.cc +++ b/apt-private/acqprogress.cc @@ -117,7 +117,8 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm) if (Itm.Owner->Status == pkgAcquire::Item::StatDone) { cout << _("Ign ") << Itm.Description << endl; - if (Itm.Owner->ErrorText.empty() == false) + if (Itm.Owner->ErrorText.empty() == false && + _config->FindB("Acquire::Progress::Ignore::ShowErrorText", false) == true) cout << " " << Itm.Owner->ErrorText << endl; } else diff --git a/test/integration/framework b/test/integration/framework index 0aa648fb6..ad3c33c28 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -215,6 +215,7 @@ setupenvironment() { cp $(find $TESTDIRECTORY -name '*.pub' -o -name '*.sec') keys/ chmod 644 $(find keys -name '*.pub' -o -name '*.sec') ln -s ${TMPWORKINGDIRECTORY}/keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf echo "Debug::NoLocking \"true\";" >> aptconfig.conf @@ -232,8 +233,11 @@ setupenvironment() { echo "DPKG::options:: \"--log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log\";" >> aptconfig.conf echo 'quiet::NoUpdate "true";' >> aptconfig.conf echo 'quiet::NoStatistic "true";' >> aptconfig.conf + # too distracting for users, but helpful to detect changes + echo 'Acquire::Progress::Ignore::ShowErrorText "true";' >> aptconfig.conf + echo "Acquire::https::CaInfo \"${TESTDIR}/apt.pem\";" > rootdir/etc/apt/apt.conf.d/99https - echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary + echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary configcompression '.' 'gz' #'bz2' 'lzma' 'xz' # cleanup the environment a bit -- cgit v1.2.3 From 0045df3fc7c3c1dba084682805b50203472d443f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 7 Oct 2014 23:52:12 +0200 Subject: do not show IP in output of testcases On travis-ci connect.cc detects a rotation, triggering it store the IP which is later appended to the error message, which is all nice and great if we deal with a real server, but in the testcases it just triggers failures as strings do not match. Git-Dch: Ignore --- apt-pkg/acquire-method.cc | 5 ++++- test/integration/framework | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 82f2fb3ce..cbcbea247 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -102,7 +102,10 @@ void pkgAcqMethod::Fail(string Err,bool Transient) if (Queue != 0) { std::cout << "400 URI Failure\nURI: " << Queue->Uri << "\n" - << "Message: " << Err << " " << IP << "\n"; + << "Message: " << Err; + if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true) + std::cout << " " << IP; + std::cout << "\n"; Dequeue(); } else diff --git a/test/integration/framework b/test/integration/framework index ad3c33c28..75cec204c 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -235,6 +235,9 @@ setupenvironment() { echo 'quiet::NoStatistic "true";' >> aptconfig.conf # too distracting for users, but helpful to detect changes echo 'Acquire::Progress::Ignore::ShowErrorText "true";' >> aptconfig.conf + # in testcases, it can appear as if localhost has a rotation setup, + # hide this as we can't really deal with it properly + echo 'Acquire::Failure::ShowIP "false";' >> aptconfig.conf echo "Acquire::https::CaInfo \"${TESTDIR}/apt.pem\";" > rootdir/etc/apt/apt.conf.d/99https echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary -- cgit v1.2.3 From f9a3c4bde867e70e8c89b6ed5924ab9fab517096 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 8 Oct 2014 00:37:32 +0200 Subject: fix http-pipeline-messup testcase The test generates failures if the created deb files have the same size, so we try a little harder to avoid having the same size for them. Git-Dch: Ignore --- test/integration/test-http-pipeline-messup | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/integration/test-http-pipeline-messup b/test/integration/test-http-pipeline-messup index 9c59e1825..405574e8a 100755 --- a/test/integration/test-http-pipeline-messup +++ b/test/integration/test-http-pipeline-messup @@ -7,10 +7,11 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture "i386" -buildsimplenativepackage 'pkga' 'all' '1.0' 'stable' -buildsimplenativepackage 'pkgb' 'all' '1.0' 'stable' -buildsimplenativepackage 'pkgc' 'all' '1.0' 'stable' -buildsimplenativepackage 'pkgd' 'all' '1.0' 'stable' +# try a little harder to create a size mismatch +buildsimplenativepackage 'pkga' 'all' '1.0' 'stable' "Depends: foo" '' '' '' '' 'none' +buildsimplenativepackage 'pkgb' 'all' '1.0' 'stable' "Depends: foo" '' '' '' '' 'none' +buildsimplenativepackage 'pkgc' 'all' '1.0' 'stable' "Depends: f$(for i in $(seq 0 1000); do printf 'o'; done)" '' '' '' '' 'none' +buildsimplenativepackage 'pkgd' 'all' '1.0' 'stable' "Depends: f$(for i in $(seq 0 1000); do printf 'o'; done)" '' '' '' '' 'none' setupaptarchive --no-update @@ -21,17 +22,20 @@ changetowebserver \ -o 'aptwebserver::overwrite::.*pkgb.*::filename=/pool/pkgc_1.0_all.deb' \ -o 'aptwebserver::overwrite::.*pkgd.*::filename=/pool/pkga_1.0_all.deb' -testsuccess aptget update -o Debug::Acquire::http=1 -o Debug::pkgAcquire::Worker=1 +echo 'Debug::Acquire::http "true"; +Debug::pkgAcquire::Worker "true";' > rootdir/etc/apt/apt.conf.d/99debug + +testsuccess aptget update # messup is bigger than pipeline: checks if fixup isn't trying to hard -testfailure aptget download pkga pkgb pkgc pkgd "$@" -o Acquire::http::Pipeline-Depth=2 +testfailure aptget download pkga pkgb pkgc pkgd -o Acquire::http::Pipeline-Depth=2 testfailure test -f pkga_1.0_all.deb # ensure that pipeling is enabled for rest of this test echo 'Acquire::http::Pipeline-Depth 10;' > rootdir/etc/apt/apt.conf.d/99enable-pipeline # the output is a bit strange: it looks like it has downloaded pkga 4 times -testsuccess aptget download pkga pkgb pkgc pkgd -o Debug::Acquire::http=1 -o Debug::pkgAcquire::Worker=1 +testsuccess aptget download pkga pkgb pkgc pkgd for pkg in 'pkga' 'pkgb' 'pkgc' 'pkgd'; do testsuccess test -f ${pkg}_1.0_all.deb testsuccess cmp incoming/${pkg}_1.0_all.deb ${pkg}_1.0_all.deb @@ -40,4 +44,4 @@ done # while hashes will pass (as none are available), sizes will not match, so failure # checks that no hashes means that pipeline depth is ignored as we can't fixup -testfailure aptget download pkga pkgb pkgc pkgd "$@" --allow-unauthenticated -o Acquire::ForceHash=ROT26 +testfailure aptget download pkga pkgb pkgc pkgd --allow-unauthenticated -o Acquire::ForceHash=ROT26 -- cgit v1.2.3