Merge branch 'feature/altretries'

Generalizing the behaviour of retrying a download on the same server (if
enabled via options) as well as retrying a download via a different
alternative server from the acquire item responsible for deb files to
the handling of items in general so that all are effected.

7 files changed, 375 insertions, 233 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index b3eb75d16..dc45a6acd 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -683,7 +683,20 @@ class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem		/*{{{*/
 class pkgAcquire::Item::Private
 {
 public:
+   struct AlternateURI
+   {
+      std::string const URI;
+      std::unordered_map<std::string, std::string> changefields;
+      AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {}
+   };
+   std::list<AlternateURI> AlternativeURIs;
    std::vector<std::string> PastRedirections;
+   std::unordered_map<std::string, std::string> CustomFields;
+   unsigned int Retries;
+
+   Private() : Retries(_config->FindI("Acquire::Retries", 0))
+   {
+   }
 };
 APT_IGNORE_DEPRECATED_PUSH
 pkgAcquire::Item::Item(pkgAcquire * const owner) :
@@ -704,7 +717,48 @@ pkgAcquire::Item::~Item()
 									/*}}}*/
 std::string pkgAcquire::Item::Custom600Headers() const			/*{{{*/
 {
-   return std::string();
+   std::ostringstream header;
+   for (auto const &f : d->CustomFields)
+      if (f.second.empty() == false)
+	 header << '\n'
+		<< f.first << ": " << f.second;
+   return header.str();
+}
+									/*}}}*/
+std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFields() /*{{{*/
+{
+   return d->CustomFields;
+}
+									/*}}}*/
+bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/
+{
+   if (d->AlternativeURIs.empty())
+      return false;
+   auto const AltUri = d->AlternativeURIs.front();
+   d->AlternativeURIs.pop_front();
+   NewURI = AltUri.URI;
+   auto &CustomFields = ModifyCustomFields();
+   for (auto const &f : AltUri.changefields)
+   {
+      if (f.second.empty())
+	 CustomFields.erase(f.first);
+      else
+	 CustomFields[f.first] = f.second;
+   }
+   return true;
+}
+									/*}}}*/
+void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/
+{
+   if (at_the_back)
+      d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields));
+   else
+      d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields));
+}
+									/*}}}*/
+unsigned int &pkgAcquire::Item::ModifyRetries() /*{{{*/
+{
+   return d->Retries;
 }
 									/*}}}*/
 std::string pkgAcquire::Item::ShortDesc() const				/*{{{*/
@@ -778,6 +832,13 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
       Dequeue();
    }
 
+   FailMessage(Message);
+
+   if (QueueCounter > 1)
+      Status = StatIdle;
+}
+void pkgAcquire::Item::FailMessage(string const &Message)
+{
    string const FailReason = LookupTag(Message, "FailReason");
    enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER;
    if ( FailReason == "MaximumSizeExceeded")
@@ -851,9 +912,6 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con
       ReportMirrorFailureToCentral(*this, FailReason, ErrorText);
    else
       ReportMirrorFailureToCentral(*this, ErrorText, ErrorText);
-
-   if (QueueCounter > 1)
-      Status = StatIdle;
 }
 									/*}}}*/
 // Acquire::Item::Start - Item has begun to download			/*{{{*/
@@ -899,7 +957,7 @@ void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Has
       }
    }
    Status = StatDone;
-   ErrorText = string();
+   ErrorText.clear();
    Owner->Dequeue(this);
 }
 									/*}}}*/
@@ -1037,6 +1095,16 @@ pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner,	/*{{{*/
 {
    if (TransactionManager != this)
       TransactionManager->Add(this);
+   ModifyCustomFields() = {
+      {"Target-Site", Target.Option(IndexTarget::SITE)},
+      {"Target-Repo-URI", Target.Option(IndexTarget::REPO_URI)},
+      {"Target-Base-URI", Target.Option(IndexTarget::BASE_URI)},
+      {"Target-Component", Target.Option(IndexTarget::COMPONENT)},
+      {"Target-Release", Target.Option(IndexTarget::RELEASE)},
+      {"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)},
+      {"Target-Language", Target.Option(IndexTarget::LANGUAGE)},
+      {"Target-Type", "index"},
+   };
 }
 									/*}}}*/
 pkgAcqTransactionItem::~pkgAcqTransactionItem()				/*{{{*/
@@ -1214,7 +1282,8 @@ bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const s
 // ---------------------------------------------------------------------
 string pkgAcqMetaBase::Custom600Headers() const
 {
-   std::string Header = "\nIndex-File: true";
+   std::string Header = pkgAcqTransactionItem::Custom600Headers();
+   Header.append("\nIndex-File: true");
    std::string MaximumSize;
    strprintf(MaximumSize, "\nMaximum-Size: %i",
              _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
@@ -3205,20 +3274,17 @@ void pkgAcqIndex::StageDecompressDone()
 									/*}}}*/
 pkgAcqIndex::~pkgAcqIndex() {}
 
-
 // AcqArchive::AcqArchive - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* This just sets up the initial fetch environment and queues the first
    possibilitiy */
-pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
-			     pkgRecords * const Recs,pkgCache::VerIterator const &Version,
-			     string &StoreFilename) :
-               Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
-               StoreFilename(StoreFilename), Vf(Version.FileList()),
-	       Trusted(false)
+APT_IGNORE_DEPRECATED_PUSH
+pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources,
+			     pkgRecords *const Recs, pkgCache::VerIterator const &Version,
+			     string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
+						      StoreFilename(StoreFilename), Vf(),
+						      Trusted(false)
 {
-   Retries = _config->FindI("Acquire::Retries",0);
-
    if (Version.Arch() == 0)
    {
       _error->Error(_("I wasn't able to locate a file for the %s package. "
@@ -3227,32 +3293,6 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
 		    Version.ParentPkg().FullName().c_str());
       return;
    }
-   
-   /* We need to find a filename to determine the extension. We make the
-      assumption here that all the available sources for this version share
-      the same extension.. */
-   // Skip not source sources, they do not have file fields.
-   for (; Vf.end() == false; ++Vf)
-   {
-      if (Vf.File().Flagged(pkgCache::Flag::NotSource))
-	 continue;
-      break;
-   }
-   
-   // Does not really matter here.. we are going to fail out below
-   if (Vf.end() != true)
-   {     
-      // If this fails to get a file name we will bomb out below.
-      pkgRecords::Parser &Parse = Recs->Lookup(Vf);
-      if (_error->PendingError() == true)
-	 return;
-            
-      // Generate the final file name as: package_version_arch.foo
-      StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
-	              QuoteString(Version.VerStr(),"_:") + '_' +
-     	              QuoteString(Version.Arch(),"_:.") + 
-	              "." + flExtension(Parse.FileName());
-   }
 
    // check if we have one trusted source for the package. if so, switch
    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
@@ -3285,126 +3325,155 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
    if (allowUnauth == true && seenUntrusted == true)
       Trusted = false;
 
-   // Select a source
-   if (QueueNext() == false && _error->PendingError() == false)
-      _error->Error(_("Can't find a source to download version '%s' of '%s'"),
-		    Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
-}
-									/*}}}*/
-// AcqArchive::QueueNext - Queue the next file source			/*{{{*/
-// ---------------------------------------------------------------------
-/* This queues the next available file version for download. It checks if
-   the archive is already available in the cache and stashs the MD5 for
-   checking later. */
-bool pkgAcqArchive::QueueNext()
-{
-   for (; Vf.end() == false; ++Vf)
+   StoreFilename.clear();
+   std::set<string> targetComponents, targetCodenames, targetSuites;
+   for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf)
    {
-      pkgCache::PkgFileIterator const PkgF = Vf.File();
-      // Ignore not source sources
+      auto const PkgF = Vf.File();
+      if (unlikely(PkgF.end()))
+	 continue;
       if (PkgF.Flagged(pkgCache::Flag::NotSource))
 	 continue;
-
-      // Try to cross match against the source list
       pkgIndexFile *Index;
       if (Sources->FindIndex(PkgF, Index) == false)
-	    continue;
-      LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
-
-      // only try to get a trusted package from another source if that source
-      // is also trusted
-      if(Trusted && !Index->IsTrusted()) 
+	 continue;
+      if (Trusted && Index->IsTrusted() == false)
 	 continue;
 
-      // Grab the text package record
       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
-      if (_error->PendingError() == true)
-	 return false;
-
-      string PkgFile = Parse.FileName();
-      ExpectedHashes = Parse.Hashes();
-
-      if (PkgFile.empty() == true)
-	 return _error->Error(_("The package index files are corrupted. No Filename: "
-			      "field for package %s."),
-			      Version.ParentPkg().Name());
+      // collect the hashes from the indexes
+      auto hsl = Parse.Hashes();
+      if (ExpectedHashes.empty())
+	 ExpectedHashes = hsl;
+      else
+      {
+	 // bad things will likely happen, but the user might be "lucky" still
+	 // if the sources provide the same hashtypes (so that they aren't mixed)
+	 for (auto const &hs : hsl)
+	    if (ExpectedHashes.push_back(hs) == false)
+	    {
+	       _error->Warning("Sources disagree on hashes for supposely identical version '%s' of '%s'.",
+			       Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
+	       break;
+	    }
+      }
+      // only allow local volatile sources to have no hashes
+      if (PkgF.Flagged(pkgCache::Flag::LocalSource))
+	 LocalSource = true;
+      else if (hsl.empty())
+	 continue;
 
-      Desc.URI = Index->ArchiveURI(PkgFile);
-      Desc.Description = Index->ArchiveInfo(Version);
-      Desc.Owner = this;
-      Desc.ShortDesc = Version.ParentPkg().FullName(true);
+      std::string poolfilename = Parse.FileName();
+      if (poolfilename.empty())
+	 continue;
 
-      // See if we already have the file. (Legacy filenames)
-      FileSize = Version->Size;
-      string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
-      struct stat Buf;
-      if (stat(FinalFile.c_str(),&Buf) == 0)
+      std::remove_reference<decltype(ModifyCustomFields())>::type fields;
       {
-	 // Make sure the size matches
-	 if ((unsigned long long)Buf.st_size == Version->Size)
+	 auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index);
+	 if (debIndex != nullptr)
 	 {
-	    Complete = true;
-	    Local = true;
-	    Status = StatDone;
-	    StoreFilename = DestFile = FinalFile;
-	    return true;
+	    auto const IT = debIndex->GetIndexTarget();
+	    fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI));
+	    fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE));
+	    fields.emplace("Target-Site", IT.Option(IndexTarget::SITE));
 	 }
-	 
-	 /* Hmm, we have a file and its size does not match, this means it is
-	    an old style mismatched arch */
-	 RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
       }
-
-      // Check it again using the new style output filenames
-      FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
-      if (stat(FinalFile.c_str(),&Buf) == 0)
+      fields.emplace("Target-Base-URI", Index->ArchiveURI(""));
+      if (PkgF->Component != 0)
+	 fields.emplace("Target-Component", PkgF.Component());
+      auto const RelF = PkgF.ReleaseFile();
+      if (RelF.end() == false)
       {
-	 // Make sure the size matches
-	 if ((unsigned long long)Buf.st_size == Version->Size)
-	 {
-	    Complete = true;
-	    Local = true;
-	    Status = StatDone;
-	    StoreFilename = DestFile = FinalFile;
-	    return true;
-	 }
-	 
-	 /* Hmm, we have a file and its size does not match, this shouldn't
-	    happen.. */
-	 RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
+	 if (RelF->Codename != 0)
+	    fields.emplace("Target-Codename", RelF.Codename());
+	 if (RelF->Archive != 0)
+	    fields.emplace("Target-Suite", RelF.Archive());
       }
+      fields.emplace("Target-Architecture", Version.Arch());
+      fields.emplace("Target-Type", flExtension(poolfilename));
 
-      DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
-      
-      // Check the destination file
-      if (stat(DestFile.c_str(),&Buf) == 0)
+      if (StoreFilename.empty())
       {
-	 // Hmm, the partial file is too big, erase it
-	 if ((unsigned long long)Buf.st_size > Version->Size)
-	    RemoveFile("pkgAcqArchive::QueueNext", DestFile);
-	 else
-	    PartialSize = Buf.st_size;
+	 /* We pick a filename based on the information we have for the version,
+	    but we don't know what extension such a file should have, so we look
+	    at the filenames used online and assume that they are the same for
+	    all repositories containing this file */
+	 StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' +
+			 QuoteString(Version.VerStr(), "_:") + '_' +
+			 QuoteString(Version.Arch(), "_:.") +
+			 "." + flExtension(poolfilename);
+
+	 Desc.URI = Index->ArchiveURI(poolfilename);
+	 Desc.Description = Index->ArchiveInfo(Version);
+	 Desc.Owner = this;
+	 Desc.ShortDesc = Version.ParentPkg().FullName(true);
+	 auto &customfields = ModifyCustomFields();
+	 for (auto const &f : fields)
+	    customfields[f.first] = f.second;
+	 FileSize = Version->Size;
       }
+      else
+	 PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true);
+   }
+   if (StoreFilename.empty())
+   {
+      _error->Error(_("Can't find a source to download version '%s' of '%s'"),
+		    Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
+      return;
+   }
 
-      // Disables download of archives - useful if no real installation follows,
-      // e.g. if we are just interested in proposed installation order
-      if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
+   // Check if we already downloaded the file
+   struct stat Buf;
+   auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
+   if (stat(FinalFile.c_str(), &Buf) == 0)
+   {
+      // Make sure the size matches
+      if ((unsigned long long)Buf.st_size == Version->Size)
       {
 	 Complete = true;
 	 Local = true;
 	 Status = StatDone;
 	 StoreFilename = DestFile = FinalFile;
-	 return true;
+	 return;
       }
 
-      // Create the item
-      Local = false;
-      ++Vf;
-      QueueURI(Desc);
-      return true;
+      /* Hmm, we have a file and its size does not match, this shouldn't
+	 happen.. */
+      RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
    }
+
+   // Check the destination file
+   DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
+   if (stat(DestFile.c_str(), &Buf) == 0)
+   {
+      // Hmm, the partial file is too big, erase it
+      if ((unsigned long long)Buf.st_size > Version->Size)
+	 RemoveFile("pkgAcqArchive::QueueNext", DestFile);
+      else
+	 PartialSize = Buf.st_size;
+   }
+
+   // Disables download of archives - useful if no real installation follows,
+   // e.g. if we are just interested in proposed installation order
+   if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
+   {
+      Complete = true;
+      Local = true;
+      Status = StatDone;
+      StoreFilename = DestFile = FinalFile;
+      return;
+   }
+
+   // Create the item
+   Local = false;
+   QueueURI(Desc);
+}
+APT_IGNORE_DEPRECATED_POP
+									/*}}}*/
+bool pkgAcqArchive::QueueNext() /*{{{*/
+{
    return false;
-}   
+}
 									/*}}}*/
 // AcqArchive::Done - Finished fetching					/*{{{*/
 // ---------------------------------------------------------------------
@@ -3437,36 +3506,6 @@ void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
 void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 {
    Item::Failed(Message,Cnf);
-
-   /* We don't really want to retry on failed media swaps, this prevents
-      that. An interesting observation is that permanent failures are not
-      recorded. */
-   if (Cnf->Removable == true &&
-       StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
-   {
-      // Vf = Version.FileList();
-      while (Vf.end() == false) ++Vf;
-      StoreFilename = string();
-      return;
-   }
-
-   Status = StatIdle;
-   if (QueueNext() == false)
-   {
-      // This is the retry counter
-      if (Retries != 0 &&
-	  Cnf->LocalOnly == false &&
-	  StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
-      {
-	 Retries--;
-	 Vf = Version.FileList();
-	 if (QueueNext() == true)
-	    return;
-      }
-
-      StoreFilename = string();
-      Status = StatError;
-   }
 }
 									/*}}}*/
 APT_PURE bool pkgAcqArchive::IsTrusted() const				/*{{{*/
@@ -3754,14 +3793,12 @@ pkgAcqChangelog::~pkgAcqChangelog()					/*{{{*/
 									/*}}}*/
 
 // AcqFile::pkgAcqFile - Constructor					/*{{{*/
-pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
-		       unsigned long long const Size,string const &Dsc,string const &ShortDesc,
+APT_IGNORE_DEPRECATED_PUSH
+pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringList const &Hashes,
+		       unsigned long long const Size, string const &Dsc, string const &ShortDesc,
 		       const string &DestDir, const string &DestFilename,
-                       bool const IsIndexFile) :
-                       Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
+		       bool const IsIndexFile) : Item(Owner), d(NULL), Retries(0), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
 {
-   Retries = _config->FindI("Acquire::Retries",0);
-
    if(!DestFilename.empty())
       DestFile = DestFilename;
    else if(!DestDir.empty())
@@ -3791,6 +3828,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringLis
 
    QueueURI(Desc);
 }
+APT_IGNORE_DEPRECATED_POP
 									/*}}}*/
 // AcqFile::Done - Item downloaded OK					/*{{{*/
 void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
@@ -3840,24 +3878,10 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
    }
 }
 									/*}}}*/
-// AcqFile::Failed - Failure handler					/*{{{*/
-// ---------------------------------------------------------------------
-/* Here we try other sources */
-void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
+void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
 {
-   Item::Failed(Message,Cnf);
-
-   // This is the retry counter
-   if (Retries != 0 &&
-       Cnf->LocalOnly == false &&
-       StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
-   {
-      --Retries;
-      QueueURI(Desc);
-      Status = StatIdle;
-      return;
-   }
-
+   // FIXME: Remove this pointless overload on next ABI break
+   Item::Failed(Message, Cnf);
 }
 									/*}}}*/
 string pkgAcqFile::Custom600Headers() const				/*{{{*/
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index a5c7d848a..cf227d1b5 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -27,6 +27,7 @@
 
 #include <map>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #ifndef APT_8_CLEANER_HEADERS
@@ -174,6 +175,7 @@ class pkgAcquire::Item : public WeakPointable				/*{{{*/
     *  \sa pkgAcqMethod
     */
    virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   APT_HIDDEN void FailMessage(std::string const &Message);
 
    /** \brief Invoked by the acquire worker to check if the successfully
     * fetched object is also the objected we wanted to have.
@@ -238,6 +240,13 @@ class pkgAcquire::Item : public WeakPointable				/*{{{*/
     *  no trailing newline.
     */
    virtual std::string Custom600Headers() const;
+   // Retries should really be a member of the Item, but can't be for ABI reasons
+   APT_HIDDEN unsigned int &ModifyRetries();
+   // this is more a hack than a proper external interface, hence hidden
+   APT_HIDDEN std::unordered_map<std::string, std::string> &ModifyCustomFields();
+   // this isn't the super nicest interface either…
+   APT_HIDDEN bool PopAlternativeURI(std::string &NewURI);
+   APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back);
 
    /** \brief A "descriptive" URI-like string.
     *
@@ -987,6 +996,7 @@ class pkgAcqArchive : public pkgAcquire::Item
    std::string &StoreFilename;
 
    /** \brief The next file for this version to try to download. */
+   APT_DEPRECATED_MSG("Unused member")
    pkgCache::VerFileIterator Vf;
 
    /** \brief How many (more) times to try to find a new source from
@@ -994,6 +1004,7 @@ class pkgAcqArchive : public pkgAcquire::Item
     *
     *  Set from Acquire::Retries.
     */
+   APT_DEPRECATED_MSG("Unused member. See pkgAcqItem::Retries.")
    unsigned int Retries;
 
    /** \brief \b true if this version file is being downloaded from a
@@ -1171,6 +1182,7 @@ class pkgAcqFile : public pkgAcquire::Item
    /** \brief How many times to retry the download, set from
     *  Acquire::Retries.
     */
+   APT_DEPRECATED_MSG("Unused member. See pkgAcqItem::Retries.")
    unsigned int Retries;
 
    /** \brief Should this file be considered a index file */
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 39158aed7..995750dea 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -211,6 +211,39 @@ static bool isDoomedItem(pkgAcquire::Item const * const Itm)
       return false;
    return TransItm->TransactionManager->State != pkgAcqTransactionItem::TransactionStarted;
 }
+static HashStringList GetHashesFromMessage(std::string const &Prefix, std::string const &Message)
+{
+   HashStringList hsl;
+   for (char const *const *type = HashString::SupportedHashes(); *type != NULL; ++type)
+   {
+      std::string const tagname = Prefix + *type + "-Hash";
+      std::string const hashsum = LookupTag(Message, tagname.c_str());
+      if (hashsum.empty() == false)
+	 hsl.push_back(HashString(*type, hashsum));
+   }
+   return hsl;
+}
+static void APT_NONNULL(3) ChangeSiteIsMirrorChange(std::string const &NewURI, pkgAcquire::ItemDesc &desc, pkgAcquire::Item *const Owner)
+{
+   if (URI::SiteOnly(NewURI) == URI::SiteOnly(desc.URI))
+      return;
+
+   auto const firstSpace = desc.Description.find(" ");
+   if (firstSpace != std::string::npos)
+   {
+      std::string const OldSite = desc.Description.substr(0, firstSpace);
+      if (likely(APT::String::Startswith(desc.URI, OldSite)))
+      {
+	 std::string const OldExtra = desc.URI.substr(OldSite.length() + 1);
+	 if (likely(APT::String::Endswith(NewURI, OldExtra)))
+	 {
+	    std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length());
+	    Owner->UsedMirror = URI::ArchiveOnly(NewSite);
+	    desc.Description.replace(0, firstSpace, Owner->UsedMirror);
+	 }
+      }
+   }
+}
 bool pkgAcquire::Worker::RunMessages()
 {
    while (MessageQueue.empty() == false)
@@ -377,13 +410,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       std::string const givenfilename = LookupTag(Message, "Filename");
 	       std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename;
 	       // see if we got hashes to verify
-	       for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
-	       {
-		  std::string const tagname = std::string(*type) + "-Hash";
-		  std::string const hashsum = LookupTag(Message, tagname.c_str());
-		  if (hashsum.empty() == false)
-		     ReceivedHashes.push_back(HashString(*type, hashsum));
-	       }
+	       ReceivedHashes = GetHashesFromMessage("", Message);
 	       // not all methods always sent Hashes our way
 	       if (ReceivedHashes.usable() == false)
 	       {
@@ -506,6 +533,9 @@ bool pkgAcquire::Worker::RunMessages()
 	    Itm = nullptr;
 
 	    bool errTransient = false, errAuthErr = false;
+	    if (StringToBool(LookupTag(Message, "Transient-Failure"), false) == true)
+	       errTransient = true;
+	    else
 	    {
 	       std::string const failReason = LookupTag(Message, "FailReason");
 	       {
@@ -522,15 +552,40 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    for (auto const Owner: ItmOwners)
 	    {
-	       if (errAuthErr && Owner->GetExpectedHashes().empty() == false)
-		  Owner->Status = pkgAcquire::Item::StatAuthError;
-	       else if (errTransient)
-		  Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
-	       auto SavedDesc = Owner->GetItemDesc();
-	       if (isDoomedItem(Owner) == false)
-		  Owner->Failed(Message,Config);
-	       if (Log != nullptr)
-		  Log->Fail(SavedDesc);
+	       std::string NewURI;
+	       if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0)
+	       {
+		  --Owner->ModifyRetries();
+		  Owner->FailMessage(Message);
+		  auto SavedDesc = Owner->GetItemDesc();
+		  if (Log != nullptr)
+		     Log->Fail(SavedDesc);
+		  if (isDoomedItem(Owner) == false)
+		     OwnerQ->Owner->Enqueue(SavedDesc);
+	       }
+	       else if (Owner->PopAlternativeURI(NewURI))
+	       {
+		  Owner->FailMessage(Message);
+		  auto &desc = Owner->GetItemDesc();
+		  if (Log != nullptr)
+		     Log->Fail(desc);
+		  ChangeSiteIsMirrorChange(NewURI, desc, Owner);
+		  desc.URI = NewURI;
+		  if (isDoomedItem(Owner) == false)
+		     OwnerQ->Owner->Enqueue(desc);
+	       }
+	       else
+	       {
+		  if (errAuthErr && Owner->GetExpectedHashes().empty() == false)
+		     Owner->Status = pkgAcquire::Item::StatAuthError;
+		  else if (errTransient)
+		     Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
+		  auto SavedDesc = Owner->GetItemDesc();
+		  if (isDoomedItem(Owner) == false)
+		     Owner->Failed(Message, Config);
+		  if (Log != nullptr)
+		     Log->Fail(SavedDesc);
+	       }
 	    }
 	    ItemDone();
 
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index f2fc0b6a6..638f09e9d 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -694,34 +694,80 @@ int stringcasecmp(string::const_iterator A,string::const_iterator AEnd,
 									/*}}}*/
 // LookupTag - Lookup the value of a tag in a taged string		/*{{{*/
 // ---------------------------------------------------------------------
-/* The format is like those used in package files and the method 
+/* The format is like those used in package files and the method
    communication system */
-string LookupTag(const string &Message,const char *Tag,const char *Default)
+std::string LookupTag(const std::string &Message, const char *TagC, const char *Default)
 {
-   // Look for a matching tag.
-   int Length = strlen(Tag);
-   for (string::const_iterator I = Message.begin(); I + Length < Message.end(); ++I)
+   std::string tag = std::string("\n") + TagC + ":";
+   if (Default == nullptr)
+      Default = "";
+   if (Message.length() < tag.length())
+      return Default;
+   std::transform(tag.begin(), tag.end(), tag.begin(), tolower_ascii);
+   auto valuestart = Message.cbegin();
+   // maybe the message starts directly with tag
+   if (Message[tag.length() - 2] == ':')
    {
-      // Found the tag
-      if (I[Length] == ':' && stringcasecmp(I,I+Length,Tag) == 0)
+      std::string lowstart = std::string("\n") + Message.substr(0, tag.length() - 1);
+      std::transform(lowstart.begin(), lowstart.end(), lowstart.begin(), tolower_ascii);
+      if (lowstart == tag)
+	 valuestart = std::next(valuestart, tag.length() - 1);
+   }
+   // the tag is somewhere in the message
+   if (valuestart == Message.cbegin())
+   {
+      auto const tagbegin = std::search(Message.cbegin(), Message.cend(), tag.cbegin(), tag.cend(),
+					[](char const a, char const b) { return tolower_ascii(a) == b; });
+      if (tagbegin == Message.cend())
+	 return Default;
+      valuestart = std::next(tagbegin, tag.length());
+   }
+   auto const is_whitespace = [](char const c) { return isspace_ascii(c) != 0 && c != '\n'; };
+   auto const is_newline = [](char const c) { return c == '\n'; };
+   std::string result;
+   valuestart = std::find_if_not(valuestart, Message.cend(), is_whitespace);
+   // is the first line of the value empty?
+   if (valuestart != Message.cend() && *valuestart == '\n')
+   {
+      valuestart = std::next(valuestart);
+      if (valuestart != Message.cend() && *valuestart == ' ')
+	 valuestart = std::next(valuestart);
+   }
+   // extract the value over multiple lines removing trailing whitespace
+   while (valuestart < Message.cend())
+   {
+      auto const linebreak = std::find_if(valuestart, Message.cend(), is_newline);
+      auto valueend = std::prev(linebreak);
+      // skip spaces at the end of the line
+      while (valueend > valuestart && is_whitespace(*valueend))
+	 valueend = std::prev(valueend);
+      // append found line to result
       {
-	 // Find the end of line and strip the leading/trailing spaces
-	 string::const_iterator J;
-	 I += Length + 1;
-	 for (; isspace_ascii(*I) != 0 && I < Message.end(); ++I);
-	 for (J = I; *J != '\n' && J < Message.end(); ++J);
-	 for (; J > I && isspace_ascii(J[-1]) != 0; --J);
-	 
-	 return string(I,J);
+	 std::string tmp(valuestart, std::next(valueend));
+	 if (tmp != ".")
+	 {
+	    if (result.empty())
+	       result.assign(std::move(tmp));
+	    else
+	       result.append(tmp);
+	 }
       }
-      
-      for (; *I != '\n' && I < Message.end(); ++I);
-   }   
-   
-   // Failed to find a match
-   if (Default == 0)
-      return string();
-   return Default;
+      // see if the value is multiline
+      if (linebreak == Message.cend())
+	 break;
+      valuestart = std::next(linebreak);
+      if (valuestart == Message.cend() || *valuestart != ' ')
+	 break;
+      result.append("\n");
+      // skip the space leading a multiline (Keep all other whitespaces in the value)
+      valuestart = std::next(valuestart);
+   }
+   auto const valueend = result.find_last_not_of("\n");
+   if (valueend == std::string::npos)
+      result.clear();
+   else
+      result.erase(valueend + 1);
+   return result;
 }
 									/*}}}*/
 // StringToBool - Converts a string into a boolean			/*{{{*/
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index ad27e2dcd..2688052a4 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -118,8 +118,6 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 {
    bool const flatArchive = (Dist[Dist.length() - 1] == '/');
    std::string const baseURI = constructMetaIndexURI(URI, Dist, "");
-   std::string const Release = (Dist == "/") ? "" : Dist;
-   std::string const Site = ::URI::ArchiveOnly(URI);
 
    std::string DefCompressionTypes;
    {
@@ -208,8 +206,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 		  constexpr static auto BreakPoint = "$(NATIVE_ARCHITECTURE)";
 		  // available in templates
 		  std::map<std::string, std::string> Options;
-		  Options.insert(std::make_pair("SITE", Site));
-		  Options.insert(std::make_pair("RELEASE", Release));
+		  Options.insert(ReleaseOptions.begin(), ReleaseOptions.end());
 		  if (tplMetaKey.find("$(COMPONENT)") != std::string::npos)
 		     Options.emplace("COMPONENT", E->Name);
 		  if (tplMetaKey.find("$(LANGUAGE)") != std::string::npos)
@@ -290,7 +287,6 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 		  }
 
 		  // not available in templates, but in the indextarget
-		  Options.insert(ReleaseOptions.begin(), ReleaseOptions.end());
 		  Options.insert(std::make_pair("IDENTIFIER", Identifier));
 		  Options.insert(std::make_pair("TARGET_OF", Type));
 		  Options.insert(std::make_pair("CREATED_BY", *T));
@@ -1070,7 +1066,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
       }
       for (auto&& key: KeysA)
       {
-	 if (key == "BASE_URI" || key == "REPO_URI")
+	 if (key == "BASE_URI" || key == "REPO_URI" || key == "SITE" || key == "RELEASE")
 	    continue;
 	 auto const a = OptionsA.find(key);
 	 auto const b = OptionsB.find(key);
@@ -1083,9 +1079,11 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
    static debReleaseIndex * GetDebReleaseIndexBy(std::vector<metaIndex *> &List, std::string const &URI,
 			   std::string const &Dist, std::map<std::string, std::string> const &Options)
    {
-      std::map<std::string,std::string> ReleaseOptions = {{
-	 { "BASE_URI", constructMetaIndexURI(URI, Dist, "") },
-	 { "REPO_URI", URI },
+      std::map<std::string, std::string> ReleaseOptions{{
+	 {"BASE_URI", constructMetaIndexURI(URI, Dist, "")},
+	 {"REPO_URI", URI},
+	 {"SITE", ::URI::ArchiveOnly(URI)},
+	 {"RELEASE", (Dist == "/") ? "" : Dist},
       }};
       if (GetBoolOption(Options, "allow-insecure", _config->FindB("Acquire::AllowInsecureRepositories")))
 	 ReleaseOptions.emplace("ALLOW_INSECURE", "true");
@@ -1134,6 +1132,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 			   bool const &IsSrc, std::map<std::string, std::string> const &Options) const
    {
       auto const Deb = GetDebReleaseIndexBy(List, URI, Dist, Options);
+      if (Deb == nullptr)
+	 return false;
 
       bool const UsePDiffs = GetBoolOption(Options, "pdiffs", _config->FindB("Acquire::PDiffs", true));
 
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 3ac5b3679..0e205a562 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -273,6 +273,10 @@ std::string pkgDebianIndexTargetFile::GetProgressDescription() const
 {
    return Target.Description;
 }
+IndexTarget pkgDebianIndexTargetFile::GetIndexTarget() const
+{
+   return Target;
+}
 
 pkgDebianIndexRealFile::pkgDebianIndexRealFile(std::string const &pFile, bool const Trusted) :/*{{{*/
    pkgDebianIndexFile(Trusted), d(NULL)
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 10b15fde4..7ebbd66f1 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -199,6 +199,7 @@ public:
    virtual std::string Describe(bool const Short = false) const APT_OVERRIDE;
    virtual bool Exists() const APT_OVERRIDE;
    virtual unsigned long Size() const APT_OVERRIDE;
+   IndexTarget GetIndexTarget() const APT_HIDDEN;
 
    pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted);
    virtual ~pkgDebianIndexTargetFile();