diff options
Diffstat (limited to 'apt-pkg')
| -rw-r--r-- | apt-pkg/acquire-item.cc | 394 | ||||
| -rw-r--r-- | apt-pkg/acquire-item.h | 12 | ||||
| -rw-r--r-- | apt-pkg/acquire-worker.cc | 87 | ||||
| -rw-r--r-- | apt-pkg/contrib/strutl.cc | 92 | ||||
| -rw-r--r-- | apt-pkg/deb/debmetaindex.cc | 18 | ||||
| -rw-r--r-- | apt-pkg/indexfile.cc | 4 | ||||
| -rw-r--r-- | apt-pkg/indexfile.h | 1 |
7 files changed, 375 insertions, 233 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b3eb75d16..dc45a6acd 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -683,7 +683,20 @@ class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem /*{{{*/ class pkgAcquire::Item::Private { public: + struct AlternateURI + { + std::string const URI; + std::unordered_map<std::string, std::string> changefields; + AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {} + }; + std::list<AlternateURI> AlternativeURIs; std::vector<std::string> PastRedirections; + std::unordered_map<std::string, std::string> CustomFields; + unsigned int Retries; + + Private() : Retries(_config->FindI("Acquire::Retries", 0)) + { + } }; APT_IGNORE_DEPRECATED_PUSH pkgAcquire::Item::Item(pkgAcquire * const owner) : @@ -704,7 +717,48 @@ pkgAcquire::Item::~Item() /*}}}*/ std::string pkgAcquire::Item::Custom600Headers() const /*{{{*/ { - return std::string(); + std::ostringstream header; + for (auto const &f : d->CustomFields) + if (f.second.empty() == false) + header << '\n' + << f.first << ": " << f.second; + return header.str(); +} + /*}}}*/ +std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFields() /*{{{*/ +{ + return d->CustomFields; +} + /*}}}*/ +bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/ +{ + if (d->AlternativeURIs.empty()) + return false; + auto const AltUri = d->AlternativeURIs.front(); + d->AlternativeURIs.pop_front(); + NewURI = AltUri.URI; + auto &CustomFields = ModifyCustomFields(); + for (auto const &f : AltUri.changefields) + { + if (f.second.empty()) + CustomFields.erase(f.first); + else + CustomFields[f.first] = f.second; + } + return true; +} + /*}}}*/ +void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/ +{ + if (at_the_back) + d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields)); + else + d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields)); +} + /*}}}*/ +unsigned int &pkgAcquire::Item::ModifyRetries() /*{{{*/ +{ + return d->Retries; } /*}}}*/ std::string pkgAcquire::Item::ShortDesc() const /*{{{*/ @@ -778,6 +832,13 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con Dequeue(); } + FailMessage(Message); + + if (QueueCounter > 1) + Status = StatIdle; +} +void pkgAcquire::Item::FailMessage(string const &Message) +{ string const FailReason = LookupTag(Message, "FailReason"); enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER; if ( FailReason == "MaximumSizeExceeded") @@ -851,9 +912,6 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con ReportMirrorFailureToCentral(*this, FailReason, ErrorText); else ReportMirrorFailureToCentral(*this, ErrorText, ErrorText); - - if (QueueCounter > 1) - Status = StatIdle; } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -899,7 +957,7 @@ void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Has } } Status = StatDone; - ErrorText = string(); + ErrorText.clear(); Owner->Dequeue(this); } /*}}}*/ @@ -1037,6 +1095,16 @@ pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/ { if (TransactionManager != this) TransactionManager->Add(this); + ModifyCustomFields() = { + {"Target-Site", Target.Option(IndexTarget::SITE)}, + {"Target-Repo-URI", Target.Option(IndexTarget::REPO_URI)}, + {"Target-Base-URI", Target.Option(IndexTarget::BASE_URI)}, + {"Target-Component", Target.Option(IndexTarget::COMPONENT)}, + {"Target-Release", Target.Option(IndexTarget::RELEASE)}, + {"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)}, + {"Target-Language", Target.Option(IndexTarget::LANGUAGE)}, + {"Target-Type", "index"}, + }; } /*}}}*/ pkgAcqTransactionItem::~pkgAcqTransactionItem() /*{{{*/ @@ -1214,7 +1282,8 @@ bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const s // --------------------------------------------------------------------- string pkgAcqMetaBase::Custom600Headers() const { - std::string Header = "\nIndex-File: true"; + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + Header.append("\nIndex-File: true"); std::string MaximumSize; strprintf(MaximumSize, "\nMaximum-Size: %i", _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000)); @@ -3205,20 +3274,17 @@ void pkgAcqIndex::StageDecompressDone() /*}}}*/ pkgAcqIndex::~pkgAcqIndex() {} - // AcqArchive::AcqArchive - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This just sets up the initial fetch environment and queues the first possibilitiy */ -pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources, - pkgRecords * const Recs,pkgCache::VerIterator const &Version, - string &StoreFilename) : - Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs), - StoreFilename(StoreFilename), Vf(Version.FileList()), - Trusted(false) +APT_IGNORE_DEPRECATED_PUSH +pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources, + pkgRecords *const Recs, pkgCache::VerIterator const &Version, + string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs), + StoreFilename(StoreFilename), Vf(), + Trusted(false) { - Retries = _config->FindI("Acquire::Retries",0); - if (Version.Arch() == 0) { _error->Error(_("I wasn't able to locate a file for the %s package. " @@ -3227,32 +3293,6 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour Version.ParentPkg().FullName().c_str()); return; } - - /* We need to find a filename to determine the extension. We make the - assumption here that all the available sources for this version share - the same extension.. */ - // Skip not source sources, they do not have file fields. - for (; Vf.end() == false; ++Vf) - { - if (Vf.File().Flagged(pkgCache::Flag::NotSource)) - continue; - break; - } - - // Does not really matter here.. we are going to fail out below - if (Vf.end() != true) - { - // If this fails to get a file name we will bomb out below. - pkgRecords::Parser &Parse = Recs->Lookup(Vf); - if (_error->PendingError() == true) - return; - - // Generate the final file name as: package_version_arch.foo - StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' + - QuoteString(Version.VerStr(),"_:") + '_' + - QuoteString(Version.Arch(),"_:.") + - "." + flExtension(Parse.FileName()); - } // check if we have one trusted source for the package. if so, switch // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode @@ -3285,126 +3325,155 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour if (allowUnauth == true && seenUntrusted == true) Trusted = false; - // Select a source - if (QueueNext() == false && _error->PendingError() == false) - _error->Error(_("Can't find a source to download version '%s' of '%s'"), - Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); -} - /*}}}*/ -// AcqArchive::QueueNext - Queue the next file source /*{{{*/ -// --------------------------------------------------------------------- -/* This queues the next available file version for download. It checks if - the archive is already available in the cache and stashs the MD5 for - checking later. */ -bool pkgAcqArchive::QueueNext() -{ - for (; Vf.end() == false; ++Vf) + StoreFilename.clear(); + std::set<string> targetComponents, targetCodenames, targetSuites; + for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf) { - pkgCache::PkgFileIterator const PkgF = Vf.File(); - // Ignore not source sources + auto const PkgF = Vf.File(); + if (unlikely(PkgF.end())) + continue; if (PkgF.Flagged(pkgCache::Flag::NotSource)) continue; - - // Try to cross match against the source list pkgIndexFile *Index; if (Sources->FindIndex(PkgF, Index) == false) - continue; - LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource); - - // only try to get a trusted package from another source if that source - // is also trusted - if(Trusted && !Index->IsTrusted()) + continue; + if (Trusted && Index->IsTrusted() == false) continue; - // Grab the text package record pkgRecords::Parser &Parse = Recs->Lookup(Vf); - if (_error->PendingError() == true) - return false; - - string PkgFile = Parse.FileName(); - ExpectedHashes = Parse.Hashes(); - - if (PkgFile.empty() == true) - return _error->Error(_("The package index files are corrupted. No Filename: " - "field for package %s."), - Version.ParentPkg().Name()); + // collect the hashes from the indexes + auto hsl = Parse.Hashes(); + if (ExpectedHashes.empty()) + ExpectedHashes = hsl; + else + { + // bad things will likely happen, but the user might be "lucky" still + // if the sources provide the same hashtypes (so that they aren't mixed) + for (auto const &hs : hsl) + if (ExpectedHashes.push_back(hs) == false) + { + _error->Warning("Sources disagree on hashes for supposely identical version '%s' of '%s'.", + Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); + break; + } + } + // only allow local volatile sources to have no hashes + if (PkgF.Flagged(pkgCache::Flag::LocalSource)) + LocalSource = true; + else if (hsl.empty()) + continue; - Desc.URI = Index->ArchiveURI(PkgFile); - Desc.Description = Index->ArchiveInfo(Version); - Desc.Owner = this; - Desc.ShortDesc = Version.ParentPkg().FullName(true); + std::string poolfilename = Parse.FileName(); + if (poolfilename.empty()) + continue; - // See if we already have the file. (Legacy filenames) - FileSize = Version->Size; - string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile); - struct stat Buf; - if (stat(FinalFile.c_str(),&Buf) == 0) + std::remove_reference<decltype(ModifyCustomFields())>::type fields; { - // Make sure the size matches - if ((unsigned long long)Buf.st_size == Version->Size) + auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index); + if (debIndex != nullptr) { - Complete = true; - Local = true; - Status = StatDone; - StoreFilename = DestFile = FinalFile; - return true; + auto const IT = debIndex->GetIndexTarget(); + fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI)); + fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE)); + fields.emplace("Target-Site", IT.Option(IndexTarget::SITE)); } - - /* Hmm, we have a file and its size does not match, this means it is - an old style mismatched arch */ - RemoveFile("pkgAcqArchive::QueueNext", FinalFile); } - - // Check it again using the new style output filenames - FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename); - if (stat(FinalFile.c_str(),&Buf) == 0) + fields.emplace("Target-Base-URI", Index->ArchiveURI("")); + if (PkgF->Component != 0) + fields.emplace("Target-Component", PkgF.Component()); + auto const RelF = PkgF.ReleaseFile(); + if (RelF.end() == false) { - // Make sure the size matches - if ((unsigned long long)Buf.st_size == Version->Size) - { - Complete = true; - Local = true; - Status = StatDone; - StoreFilename = DestFile = FinalFile; - return true; - } - - /* Hmm, we have a file and its size does not match, this shouldn't - happen.. */ - RemoveFile("pkgAcqArchive::QueueNext", FinalFile); + if (RelF->Codename != 0) + fields.emplace("Target-Codename", RelF.Codename()); + if (RelF->Archive != 0) + fields.emplace("Target-Suite", RelF.Archive()); } + fields.emplace("Target-Architecture", Version.Arch()); + fields.emplace("Target-Type", flExtension(poolfilename)); - DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename); - - // Check the destination file - if (stat(DestFile.c_str(),&Buf) == 0) + if (StoreFilename.empty()) { - // Hmm, the partial file is too big, erase it - if ((unsigned long long)Buf.st_size > Version->Size) - RemoveFile("pkgAcqArchive::QueueNext", DestFile); - else - PartialSize = Buf.st_size; + /* We pick a filename based on the information we have for the version, + but we don't know what extension such a file should have, so we look + at the filenames used online and assume that they are the same for + all repositories containing this file */ + StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' + + QuoteString(Version.VerStr(), "_:") + '_' + + QuoteString(Version.Arch(), "_:.") + + "." + flExtension(poolfilename); + + Desc.URI = Index->ArchiveURI(poolfilename); + Desc.Description = Index->ArchiveInfo(Version); + Desc.Owner = this; + Desc.ShortDesc = Version.ParentPkg().FullName(true); + auto &customfields = ModifyCustomFields(); + for (auto const &f : fields) + customfields[f.first] = f.second; + FileSize = Version->Size; } + else + PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true); + } + if (StoreFilename.empty()) + { + _error->Error(_("Can't find a source to download version '%s' of '%s'"), + Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); + return; + } - // Disables download of archives - useful if no real installation follows, - // e.g. if we are just interested in proposed installation order - if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true) + // Check if we already downloaded the file + struct stat Buf; + auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename); + if (stat(FinalFile.c_str(), &Buf) == 0) + { + // Make sure the size matches + if ((unsigned long long)Buf.st_size == Version->Size) { Complete = true; Local = true; Status = StatDone; StoreFilename = DestFile = FinalFile; - return true; + return; } - // Create the item - Local = false; - ++Vf; - QueueURI(Desc); - return true; + /* Hmm, we have a file and its size does not match, this shouldn't + happen.. */ + RemoveFile("pkgAcqArchive::QueueNext", FinalFile); } + + // Check the destination file + DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename); + if (stat(DestFile.c_str(), &Buf) == 0) + { + // Hmm, the partial file is too big, erase it + if ((unsigned long long)Buf.st_size > Version->Size) + RemoveFile("pkgAcqArchive::QueueNext", DestFile); + else + PartialSize = Buf.st_size; + } + + // Disables download of archives - useful if no real installation follows, + // e.g. if we are just interested in proposed installation order + if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true) + { + Complete = true; + Local = true; + Status = StatDone; + StoreFilename = DestFile = FinalFile; + return; + } + + // Create the item + Local = false; + QueueURI(Desc); +} +APT_IGNORE_DEPRECATED_POP + /*}}}*/ +bool pkgAcqArchive::QueueNext() /*{{{*/ +{ return false; -} +} /*}}}*/ // AcqArchive::Done - Finished fetching /*{{{*/ // --------------------------------------------------------------------- @@ -3437,36 +3506,6 @@ void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes, void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) { Item::Failed(Message,Cnf); - - /* We don't really want to retry on failed media swaps, this prevents - that. An interesting observation is that permanent failures are not - recorded. */ - if (Cnf->Removable == true && - StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) - { - // Vf = Version.FileList(); - while (Vf.end() == false) ++Vf; - StoreFilename = string(); - return; - } - - Status = StatIdle; - if (QueueNext() == false) - { - // This is the retry counter - if (Retries != 0 && - Cnf->LocalOnly == false && - StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) - { - Retries--; - Vf = Version.FileList(); - if (QueueNext() == true) - return; - } - - StoreFilename = string(); - Status = StatError; - } } /*}}}*/ APT_PURE bool pkgAcqArchive::IsTrusted() const /*{{{*/ @@ -3754,14 +3793,12 @@ pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/ /*}}}*/ // AcqFile::pkgAcqFile - Constructor /*{{{*/ -pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes, - unsigned long long const Size,string const &Dsc,string const &ShortDesc, +APT_IGNORE_DEPRECATED_PUSH +pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringList const &Hashes, + unsigned long long const Size, string const &Dsc, string const &ShortDesc, const string &DestDir, const string &DestFilename, - bool const IsIndexFile) : - Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes) + bool const IsIndexFile) : Item(Owner), d(NULL), Retries(0), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes) { - Retries = _config->FindI("Acquire::Retries",0); - if(!DestFilename.empty()) DestFile = DestFilename; else if(!DestDir.empty()) @@ -3791,6 +3828,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringLis QueueURI(Desc); } +APT_IGNORE_DEPRECATED_POP /*}}}*/ // AcqFile::Done - Item downloaded OK /*{{{*/ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, @@ -3840,24 +3878,10 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, } } /*}}}*/ -// AcqFile::Failed - Failure handler /*{{{*/ -// --------------------------------------------------------------------- -/* Here we try other sources */ -void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf) +void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ { - Item::Failed(Message,Cnf); - - // This is the retry counter - if (Retries != 0 && - Cnf->LocalOnly == false && - StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) - { - --Retries; - QueueURI(Desc); - Status = StatIdle; - return; - } - + // FIXME: Remove this pointless overload on next ABI break + Item::Failed(Message, Cnf); } /*}}}*/ string pkgAcqFile::Custom600Headers() const /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index a5c7d848a..cf227d1b5 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -27,6 +27,7 @@ #include <map> #include <string> +#include <unordered_map> #include <vector> #ifndef APT_8_CLEANER_HEADERS @@ -174,6 +175,7 @@ class pkgAcquire::Item : public WeakPointable /*{{{*/ * \sa pkgAcqMethod */ virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); + APT_HIDDEN void FailMessage(std::string const &Message); /** \brief Invoked by the acquire worker to check if the successfully * fetched object is also the objected we wanted to have. @@ -238,6 +240,13 @@ class pkgAcquire::Item : public WeakPointable /*{{{*/ * no trailing newline. */ virtual std::string Custom600Headers() const; + // Retries should really be a member of the Item, but can't be for ABI reasons + APT_HIDDEN unsigned int &ModifyRetries(); + // this is more a hack than a proper external interface, hence hidden + APT_HIDDEN std::unordered_map<std::string, std::string> &ModifyCustomFields(); + // this isn't the super nicest interface either… + APT_HIDDEN bool PopAlternativeURI(std::string &NewURI); + APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back); /** \brief A "descriptive" URI-like string. * @@ -987,6 +996,7 @@ class pkgAcqArchive : public pkgAcquire::Item std::string &StoreFilename; /** \brief The next file for this version to try to download. */ + APT_DEPRECATED_MSG("Unused member") pkgCache::VerFileIterator Vf; /** \brief How many (more) times to try to find a new source from @@ -994,6 +1004,7 @@ class pkgAcqArchive : public pkgAcquire::Item * * Set from Acquire::Retries. */ + APT_DEPRECATED_MSG("Unused member. See pkgAcqItem::Retries.") unsigned int Retries; /** \brief \b true if this version file is being downloaded from a @@ -1171,6 +1182,7 @@ class pkgAcqFile : public pkgAcquire::Item /** \brief How many times to retry the download, set from * Acquire::Retries. */ + APT_DEPRECATED_MSG("Unused member. See pkgAcqItem::Retries.") unsigned int Retries; /** \brief Should this file be considered a index file */ diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 39158aed7..995750dea 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -211,6 +211,39 @@ static bool isDoomedItem(pkgAcquire::Item const * const Itm) return false; return TransItm->TransactionManager->State != pkgAcqTransactionItem::TransactionStarted; } +static HashStringList GetHashesFromMessage(std::string const &Prefix, std::string const &Message) +{ + HashStringList hsl; + for (char const *const *type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const tagname = Prefix + *type + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + hsl.push_back(HashString(*type, hashsum)); + } + return hsl; +} +static void APT_NONNULL(3) ChangeSiteIsMirrorChange(std::string const &NewURI, pkgAcquire::ItemDesc &desc, pkgAcquire::Item *const Owner) +{ + if (URI::SiteOnly(NewURI) == URI::SiteOnly(desc.URI)) + return; + + auto const firstSpace = desc.Description.find(" "); + if (firstSpace != std::string::npos) + { + std::string const OldSite = desc.Description.substr(0, firstSpace); + if (likely(APT::String::Startswith(desc.URI, OldSite))) + { + std::string const OldExtra = desc.URI.substr(OldSite.length() + 1); + if (likely(APT::String::Endswith(NewURI, OldExtra))) + { + std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length()); + Owner->UsedMirror = URI::ArchiveOnly(NewSite); + desc.Description.replace(0, firstSpace, Owner->UsedMirror); + } + } + } +} bool pkgAcquire::Worker::RunMessages() { while (MessageQueue.empty() == false) @@ -377,13 +410,7 @@ bool pkgAcquire::Worker::RunMessages() std::string const givenfilename = LookupTag(Message, "Filename"); std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename; // see if we got hashes to verify - for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) - { - std::string const tagname = std::string(*type) + "-Hash"; - std::string const hashsum = LookupTag(Message, tagname.c_str()); - if (hashsum.empty() == false) - ReceivedHashes.push_back(HashString(*type, hashsum)); - } + ReceivedHashes = GetHashesFromMessage("", Message); // not all methods always sent Hashes our way if (ReceivedHashes.usable() == false) { @@ -506,6 +533,9 @@ bool pkgAcquire::Worker::RunMessages() Itm = nullptr; bool errTransient = false, errAuthErr = false; + if (StringToBool(LookupTag(Message, "Transient-Failure"), false) == true) + errTransient = true; + else { std::string const failReason = LookupTag(Message, "FailReason"); { @@ -522,15 +552,40 @@ bool pkgAcquire::Worker::RunMessages() for (auto const Owner: ItmOwners) { - if (errAuthErr && Owner->GetExpectedHashes().empty() == false) - Owner->Status = pkgAcquire::Item::StatAuthError; - else if (errTransient) - Owner->Status = pkgAcquire::Item::StatTransientNetworkError; - auto SavedDesc = Owner->GetItemDesc(); - if (isDoomedItem(Owner) == false) - Owner->Failed(Message,Config); - if (Log != nullptr) - Log->Fail(SavedDesc); + std::string NewURI; + if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0) + { + --Owner->ModifyRetries(); + Owner->FailMessage(Message); + auto SavedDesc = Owner->GetItemDesc(); + if (Log != nullptr) + Log->Fail(SavedDesc); + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(SavedDesc); + } + else if (Owner->PopAlternativeURI(NewURI)) + { + Owner->FailMessage(Message); + auto &desc = Owner->GetItemDesc(); + if (Log != nullptr) + Log->Fail(desc); + ChangeSiteIsMirrorChange(NewURI, desc, Owner); + desc.URI = NewURI; + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(desc); + } + else + { + if (errAuthErr && Owner->GetExpectedHashes().empty() == false) + Owner->Status = pkgAcquire::Item::StatAuthError; + else if (errTransient) + Owner->Status = pkgAcquire::Item::StatTransientNetworkError; + auto SavedDesc = Owner->GetItemDesc(); + if (isDoomedItem(Owner) == false) + Owner->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + } } ItemDone(); diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index f2fc0b6a6..638f09e9d 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -694,34 +694,80 @@ int stringcasecmp(string::const_iterator A,string::const_iterator AEnd, /*}}}*/ // LookupTag - Lookup the value of a tag in a taged string /*{{{*/ // --------------------------------------------------------------------- -/* The format is like those used in package files and the method +/* The format is like those used in package files and the method communication system */ -string LookupTag(const string &Message,const char *Tag,const char *Default) +std::string LookupTag(const std::string &Message, const char *TagC, const char *Default) { - // Look for a matching tag. - int Length = strlen(Tag); - for (string::const_iterator I = Message.begin(); I + Length < Message.end(); ++I) + std::string tag = std::string("\n") + TagC + ":"; + if (Default == nullptr) + Default = ""; + if (Message.length() < tag.length()) + return Default; + std::transform(tag.begin(), tag.end(), tag.begin(), tolower_ascii); + auto valuestart = Message.cbegin(); + // maybe the message starts directly with tag + if (Message[tag.length() - 2] == ':') { - // Found the tag - if (I[Length] == ':' && stringcasecmp(I,I+Length,Tag) == 0) + std::string lowstart = std::string("\n") + Message.substr(0, tag.length() - 1); + std::transform(lowstart.begin(), lowstart.end(), lowstart.begin(), tolower_ascii); + if (lowstart == tag) + valuestart = std::next(valuestart, tag.length() - 1); + } + // the tag is somewhere in the message + if (valuestart == Message.cbegin()) + { + auto const tagbegin = std::search(Message.cbegin(), Message.cend(), tag.cbegin(), tag.cend(), + [](char const a, char const b) { return tolower_ascii(a) == b; }); + if (tagbegin == Message.cend()) + return Default; + valuestart = std::next(tagbegin, tag.length()); + } + auto const is_whitespace = [](char const c) { return isspace_ascii(c) != 0 && c != '\n'; }; + auto const is_newline = [](char const c) { return c == '\n'; }; + std::string result; + valuestart = std::find_if_not(valuestart, Message.cend(), is_whitespace); + // is the first line of the value empty? + if (valuestart != Message.cend() && *valuestart == '\n') + { + valuestart = std::next(valuestart); + if (valuestart != Message.cend() && *valuestart == ' ') + valuestart = std::next(valuestart); + } + // extract the value over multiple lines removing trailing whitespace + while (valuestart < Message.cend()) + { + auto const linebreak = std::find_if(valuestart, Message.cend(), is_newline); + auto valueend = std::prev(linebreak); + // skip spaces at the end of the line + while (valueend > valuestart && is_whitespace(*valueend)) + valueend = std::prev(valueend); + // append found line to result { - // Find the end of line and strip the leading/trailing spaces - string::const_iterator J; - I += Length + 1; - for (; isspace_ascii(*I) != 0 && I < Message.end(); ++I); - for (J = I; *J != '\n' && J < Message.end(); ++J); - for (; J > I && isspace_ascii(J[-1]) != 0; --J); - - return string(I,J); + std::string tmp(valuestart, std::next(valueend)); + if (tmp != ".") + { + if (result.empty()) + result.assign(std::move(tmp)); + else + result.append(tmp); + } } - - for (; *I != '\n' && I < Message.end(); ++I); - } - - // Failed to find a match - if (Default == 0) - return string(); - return Default; + // see if the value is multiline + if (linebreak == Message.cend()) + break; + valuestart = std::next(linebreak); + if (valuestart == Message.cend() || *valuestart != ' ') + break; + result.append("\n"); + // skip the space leading a multiline (Keep all other whitespaces in the value) + valuestart = std::next(valuestart); + } + auto const valueend = result.find_last_not_of("\n"); + if (valueend == std::string::npos) + result.clear(); + else + result.erase(valueend + 1); + return result; } /*}}}*/ // StringToBool - Converts a string into a boolean /*{{{*/ diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index ad27e2dcd..2688052a4 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -118,8 +118,6 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, { bool const flatArchive = (Dist[Dist.length() - 1] == '/'); std::string const baseURI = constructMetaIndexURI(URI, Dist, ""); - std::string const Release = (Dist == "/") ? "" : Dist; - std::string const Site = ::URI::ArchiveOnly(URI); std::string DefCompressionTypes; { @@ -208,8 +206,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, constexpr static auto BreakPoint = "$(NATIVE_ARCHITECTURE)"; // available in templates std::map<std::string, std::string> Options; - Options.insert(std::make_pair("SITE", Site)); - Options.insert(std::make_pair("RELEASE", Release)); + Options.insert(ReleaseOptions.begin(), ReleaseOptions.end()); if (tplMetaKey.find("$(COMPONENT)") != std::string::npos) Options.emplace("COMPONENT", E->Name); if (tplMetaKey.find("$(LANGUAGE)") != std::string::npos) @@ -290,7 +287,6 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, } // not available in templates, but in the indextarget - Options.insert(ReleaseOptions.begin(), ReleaseOptions.end()); Options.insert(std::make_pair("IDENTIFIER", Identifier)); Options.insert(std::make_pair("TARGET_OF", Type)); Options.insert(std::make_pair("CREATED_BY", *T)); @@ -1070,7 +1066,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ } for (auto&& key: KeysA) { - if (key == "BASE_URI" || key == "REPO_URI") + if (key == "BASE_URI" || key == "REPO_URI" || key == "SITE" || key == "RELEASE") continue; auto const a = OptionsA.find(key); auto const b = OptionsB.find(key); @@ -1083,9 +1079,11 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ static debReleaseIndex * GetDebReleaseIndexBy(std::vector<metaIndex *> &List, std::string const &URI, std::string const &Dist, std::map<std::string, std::string> const &Options) { - std::map<std::string,std::string> ReleaseOptions = {{ - { "BASE_URI", constructMetaIndexURI(URI, Dist, "") }, - { "REPO_URI", URI }, + std::map<std::string, std::string> ReleaseOptions{{ + {"BASE_URI", constructMetaIndexURI(URI, Dist, "")}, + {"REPO_URI", URI}, + {"SITE", ::URI::ArchiveOnly(URI)}, + {"RELEASE", (Dist == "/") ? "" : Dist}, }}; if (GetBoolOption(Options, "allow-insecure", _config->FindB("Acquire::AllowInsecureRepositories"))) ReleaseOptions.emplace("ALLOW_INSECURE", "true"); @@ -1134,6 +1132,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ bool const &IsSrc, std::map<std::string, std::string> const &Options) const { auto const Deb = GetDebReleaseIndexBy(List, URI, Dist, Options); + if (Deb == nullptr) + return false; bool const UsePDiffs = GetBoolOption(Options, "pdiffs", _config->FindB("Acquire::PDiffs", true)); diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc index 3ac5b3679..0e205a562 100644 --- a/apt-pkg/indexfile.cc +++ b/apt-pkg/indexfile.cc @@ -273,6 +273,10 @@ std::string pkgDebianIndexTargetFile::GetProgressDescription() const { return Target.Description; } +IndexTarget pkgDebianIndexTargetFile::GetIndexTarget() const +{ + return Target; +} pkgDebianIndexRealFile::pkgDebianIndexRealFile(std::string const &pFile, bool const Trusted) :/*{{{*/ pkgDebianIndexFile(Trusted), d(NULL) diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index 10b15fde4..7ebbd66f1 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -199,6 +199,7 @@ public: virtual std::string Describe(bool const Short = false) const APT_OVERRIDE; virtual bool Exists() const APT_OVERRIDE; virtual unsigned long Size() const APT_OVERRIDE; + IndexTarget GetIndexTarget() const APT_HIDDEN; pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted); virtual ~pkgDebianIndexTargetFile(); |