Merge branch 'feature/amtshilfe' into 'master'

reimplement mirror method

See merge request apt-team/apt!1

10 files changed, 443 insertions, 187 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 9b163081b..611876dd2 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -16,6 +16,7 @@
 #include <config.h>
 
 #include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire-worker.h>
 #include <apt-pkg/acquire.h>
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/configuration.h>
@@ -34,6 +35,7 @@
 #include <algorithm>
 #include <ctime>
 #include <iostream>
+#include <memory>
 #include <numeric>
 #include <random>
 #include <sstream>
@@ -1070,6 +1072,12 @@ bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI)	/*{{{*/
 									/*}}}*/
 int pkgAcquire::Item::Priority()					/*{{{*/
 {
+   // Stage 0: Files requested by methods
+   // - they will usually not end up here, but if they do we make sure
+   //   to get them as soon as possible as they are probably blocking
+   //   the processing of files by the requesting method
+   if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr)
+      return 5000;
    // Stage 1: Meta indices and diff indices
    // - those need to be fetched first to have progress reporting working
    //   for the rest
@@ -3892,3 +3900,114 @@ string pkgAcqFile::Custom600Headers() const				/*{{{*/
 }
 									/*}}}*/
 pkgAcqFile::~pkgAcqFile() {}
+
+void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
+{
+   pkgAcqFile::Failed(Message, Cnf);
+   if (Status == StatIdle)
+      return;
+   if (RealFileExists(DestFile))
+      Rename(DestFile, DestFile + ".FAILED");
+   Worker->ReplyAux(Desc);
+}
+									/*}}}*/
+void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/
+			 pkgAcquire::MethodConfig const *const Cnf)
+{
+   pkgAcqFile::Done(Message, CalcHashes, Cnf);
+   if (Status == StatDone)
+      Worker->ReplyAux(Desc);
+   else if (Status == StatAuthError || Status == StatError)
+      Worker->ReplyAux(Desc);
+}
+									/*}}}*/
+std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/
+{
+   if (MaximumSize == 0)
+      return pkgAcqFile::Custom600Headers();
+   std::string maxsize;
+   strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize);
+   return pkgAcqFile::Custom600Headers().append(maxsize);
+}
+									/*}}}*/
+void pkgAcqAuxFile::Finished() /*{{{*/
+{
+   auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
+   if (APT::String::Startswith(DestFile, dirname))
+   {
+      // the file is never returned by method requesting it, so fix up the permission now
+      if (FileExists(DestFile))
+      {
+	 ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644);
+	 if (Status == StatDone)
+	    return;
+      }
+   }
+   else
+   {
+      dirname = flNotFile(DestFile);
+      RemoveFile("pkgAcqAuxFile::Finished", DestFile);
+      RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED");
+      rmdir(dirname.c_str());
+   }
+   DestFile.clear();
+}
+									/*}}}*/
+// GetAuxFileNameFromURI						/*{{{*/
+static std::string GetAuxFileNameFromURIInLists(std::string const &uri)
+{
+   // check if we have write permission for our usual location.
+   auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
+   char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
+   std::string const tmpfile_tpl = flCombine(dirname, filetag);
+   std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
+   int const fd = mkstemp(tmpfile.get());
+   if (fd == -1 && errno == EACCES)
+      return "";
+   RemoveFile("GetAuxFileNameFromURI", tmpfile.get());
+   close(fd);
+   return flCombine(dirname, URItoFileName(uri));
+}
+static std::string GetAuxFileNameFromURI(std::string const &uri)
+{
+   auto const lists = GetAuxFileNameFromURIInLists(uri);
+   if (lists.empty() == false)
+      return lists;
+
+   std::string tmpdir_tpl;
+   strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str());
+   std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free };
+   if (mkdtemp(tmpdir.get()) == nullptr)
+   {
+      _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get());
+      return flCombine("/nonexistent/auxfiles/", URItoFileName(uri));
+   }
+   chmod(tmpdir.get(), 0755);
+   auto const filename = flCombine(tmpdir.get(), URItoFileName(uri));
+   _error->PushToStack();
+   FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly);
+   if (in.IsOpen())
+   {
+      FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive);
+      CopyFile(in, out);
+   }
+   _error->RevertToStack();
+   return filename;
+}
+									/*}}}*/
+pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker,
+			     std::string const &ShortDesc, std::string const &Desc, std::string const &URI,
+			     HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false),
+												   Owner(Owner), Worker(Worker), MaximumSize(MaximumSize)
+{
+   /* very bad failures can happen while constructing which causes
+      us to hang as the aux request is never answered (e.g. method not available)
+      Ideally we catch failures earlier, but a safe guard can't hurt. */
+   if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching)
+      return;
+   Failed(std::string("400 URI Failure\n") +
+	     "URI: " + URI + "\n" +
+	     "Filename: " + DestFile,
+	  nullptr);
+}
+pkgAcqAuxFile::~pkgAcqAuxFile() {}
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index cf227d1b5..46d79df92 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -1238,6 +1238,25 @@ class pkgAcqFile : public pkgAcquire::Item
    virtual ~pkgAcqFile();
 };
 									/*}}}*/
+class APT_HIDDEN pkgAcqAuxFile : public pkgAcqFile /*{{{*/
+{
+   pkgAcquire::Item *const Owner;
+   pkgAcquire::Worker *const Worker;
+   unsigned long long MaximumSize;
+
+   public:
+   virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE;
+   virtual void Done(std::string const &Message, HashStringList const &CalcHashes,
+		     pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE;
+   virtual std::string Custom600Headers() const APT_OVERRIDE;
+   virtual void Finished() APT_OVERRIDE;
+
+   pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker,
+		 std::string const &ShortDesc, std::string const &Desc, std::string const &URI,
+		 HashStringList const &Hashes, unsigned long long const MaximumSize);
+   virtual ~pkgAcqAuxFile();
+};
+									/*}}}*/
 /** @} */
 
 #endif
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index 309b5dcf9..8934d87a0 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -27,7 +27,10 @@
 #include <apt-pkg/sha2.h>
 #include <apt-pkg/strutl.h>
 
+#include <algorithm>
 #include <iostream>
+#include <iterator>
+#include <sstream>
 #include <string>
 #include <vector>
 #include <stdarg.h>
@@ -39,33 +42,44 @@
 
 using namespace std;
 
+// poor mans unordered_map::try_emplace for C++11 as it is a C++17 feature /*{{{*/
+template <typename Arg>
+static void try_emplace(std::unordered_map<std::string, std::string> &fields, std::string &&name, Arg &&value)
+{
+   if (fields.find(name) == fields.end())
+      fields.emplace(std::move(name), std::forward<Arg>(value));
+}
+									/*}}}*/
+
 // AcqMethod::pkgAcqMethod - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* This constructs the initialization text */
 pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags)
 {
-   std::cout << "100 Capabilities\n"
-	     << "Version: " << Ver << "\n";
-
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "Version", Ver);
    if ((Flags & SingleInstance) == SingleInstance)
-      std::cout << "Single-Instance: true\n";
+      try_emplace(fields, "Single-Instance", "true");
 
    if ((Flags & Pipeline) == Pipeline)
-      std::cout << "Pipeline: true\n";
+      try_emplace(fields, "Pipeline", "true");
 
    if ((Flags & SendConfig) == SendConfig)
-      std::cout << "Send-Config: true\n";
+      try_emplace(fields, "Send-Config", "true");
 
    if ((Flags & LocalOnly) == LocalOnly)
-      std::cout <<"Local-Only: true\n";
+      try_emplace(fields, "Local-Only", "true");
 
    if ((Flags & NeedsCleanup) == NeedsCleanup)
-      std::cout << "Needs-Cleanup: true\n";
+      try_emplace(fields, "Needs-Cleanup", "true");
 
    if ((Flags & Removable) == Removable)
-      std::cout << "Removable: true\n";
+      try_emplace(fields, "Removable", "true");
 
-   std::cout << "\n" << std::flush;
+   if ((Flags & AuxRequests) == AuxRequests)
+      try_emplace(fields, "AuxRequests", "true");
+
+   SendMessage("100 Capabilities", std::move(fields));
 
    SetNonBlock(STDIN_FILENO,true);
 
@@ -73,6 +87,26 @@ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags)
    QueueBack = 0;
 }
 									/*}}}*/
+void pkgAcqMethod::SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields) /*{{{*/
+{
+   std::cout << header << '\n';
+   for (auto const &f : fields)
+   {
+      if (f.second.empty())
+	 continue;
+      std::cout << f.first << ": ";
+      auto const lines = VectorizeString(f.second, '\n');
+      if (likely(lines.empty() == false))
+      {
+	 std::copy(lines.begin(), std::prev(lines.end()), std::ostream_iterator<std::string>(std::cout, "\n "));
+	 std::cout << *lines.rbegin();
+      }
+      std::cout << '\n';
+   }
+   std::cout << '\n'
+	     << std::flush;
+}
+									/*}}}*/
 // AcqMethod::Fail - A fetch has failed					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -97,40 +131,35 @@ void pkgAcqMethod::Fail(bool Transient)
 }
 									/*}}}*/
 // AcqMethod::Fail - A fetch has failed					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void pkgAcqMethod::Fail(string Err,bool Transient)
 {
    // Strip out junk from the error messages
-   for (string::iterator I = Err.begin(); I != Err.end(); ++I)
-   {
-      if (*I == '\r') 
-	 *I = ' ';
-      if (*I == '\n') 
-	 *I = ' ';
-   }
-
-   if (Queue != 0)
-   {
-      std::cout << "400 URI Failure\nURI: " << Queue->Uri << "\n"
-		<< "Message: " << Err;
-      if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true)
-	 std::cout << " " << IP;
-      std::cout << "\n";
-      Dequeue();
-   }
+   std::transform(Err.begin(), Err.end(), Err.begin(), [](char const c) {
+      if (c == '\r' || c == '\n')
+	 return ' ';
+      return c;
+   });
+   if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true)
+      Err.append(" ").append(IP);
+
+   std::unordered_map<std::string, std::string> fields;
+   if (Queue != nullptr)
+      try_emplace(fields, "URI", Queue->Uri);
    else
-      std::cout << "400 URI Failure\nURI: <UNKNOWN>\nMessage: " << Err << "\n";
+      try_emplace(fields, "URI", "<UNKNOWN>");
+   try_emplace(fields, "Message", Err);
 
    if(FailReason.empty() == false)
-      std::cout << "FailReason: " << FailReason << "\n";
+      try_emplace(fields, "FailReason", FailReason);
    if (UsedMirror.empty() == false)
-      std::cout << "UsedMirror: " << UsedMirror << "\n";
-   // Set the transient flag
+      try_emplace(fields, "UsedMirror", UsedMirror);
    if (Transient == true)
-      std::cout << "Transient-Failure: true\n";
+      try_emplace(fields, "Transient-Failure", "true");
 
-   std::cout << "\n" << std::flush;
+   SendMessage("400 URI Failure", std::move(fields));
+
+   if (Queue != nullptr)
+      Dequeue();
 }
 									/*}}}*/
 // AcqMethod::DropPrivsOrDie - Drop privileges or die		/*{{{*/
@@ -146,96 +175,79 @@ void pkgAcqMethod::DropPrivsOrDie()
 
 									/*}}}*/
 // AcqMethod::URIStart - Indicate a download is starting		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void pkgAcqMethod::URIStart(FetchResult &Res)
 {
    if (Queue == 0)
       abort();
 
-   std::cout << "200 URI Start\n"
-	     << "URI: " << Queue->Uri << "\n";
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "URI", Queue->Uri);
    if (Res.Size != 0)
-      std::cout << "Size: " << std::to_string(Res.Size) << "\n";
-
+      try_emplace(fields, "Size", std::to_string(Res.Size));
    if (Res.LastModified != 0)
-      std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified, true) << "\n";
-
+      try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true));
    if (Res.ResumePoint != 0)
-      std::cout << "Resume-Point: " << std::to_string(Res.ResumePoint) << "\n";
-
+      try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint));
    if (UsedMirror.empty() == false)
-      std::cout << "UsedMirror: " << UsedMirror << "\n";
+      try_emplace(fields, "UsedMirror", UsedMirror);
 
-   std::cout << "\n" << std::flush;
+   SendMessage("200 URI Start", std::move(fields));
 }
 									/*}}}*/
 // AcqMethod::URIDone - A URI is finished				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-static void printHashStringList(char const *const Prefix, HashStringList const *const list)
+static void printHashStringList(std::unordered_map<std::string, std::string> &fields, std::string const &Prefix, HashStringList const &list)
 {
-      for (HashStringList::const_iterator hash = list->begin(); hash != list->end(); ++hash)
-      {
-	 // very old compatibility name for MD5Sum
-	 if (hash->HashType() == "MD5Sum")
-	    std::cout << Prefix << "MD5-Hash: " << hash->HashValue() << "\n";
-	 std::cout << hash->HashType() << "-Hash: " << hash->HashValue() << "\n";
-      }
+   for (auto const &hash : list)
+   {
+      // very old compatibility name for MD5Sum
+      if (hash.HashType() == "MD5Sum")
+	 try_emplace(fields, Prefix + "MD5-Hash", hash.HashValue());
+      try_emplace(fields, Prefix + hash.HashType() + "-Hash", hash.HashValue());
+   }
 }
 void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
 {
    if (Queue == 0)
       abort();
 
-   std::cout << "201 URI Done\n"
-	     << "URI: " << Queue->Uri << "\n";
-
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "URI", Queue->Uri);
    if (Res.Filename.empty() == false)
-      std::cout << "Filename: " << Res.Filename << "\n";
-
+      try_emplace(fields, "Filename", Res.Filename);
    if (Res.Size != 0)
-      std::cout << "Size: " << std::to_string(Res.Size) << "\n";
-
+      try_emplace(fields, "Size", std::to_string(Res.Size));
    if (Res.LastModified != 0)
-      std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified, true) << "\n";
-
-   printHashStringList("", &Res.Hashes);
+      try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true));
+   printHashStringList(fields, "", Res.Hashes);
 
    if (UsedMirror.empty() == false)
-      std::cout << "UsedMirror: " << UsedMirror << "\n";
+      try_emplace(fields, "UsedMirror", UsedMirror);
    if (Res.GPGVOutput.empty() == false)
    {
-      std::cout << "GPGVOutput:\n";
-      for (vector<string>::const_iterator I = Res.GPGVOutput.begin();
-	   I != Res.GPGVOutput.end(); ++I)
-	 std::cout << " " << *I << "\n";
+      std::ostringstream os;
+      std::copy(Res.GPGVOutput.begin(), Res.GPGVOutput.end() - 1, std::ostream_iterator<std::string>(os, "\n"));
+      os << *Res.GPGVOutput.rbegin();
+      try_emplace(fields, "GPGVOutput", os.str());
    }
-
    if (Res.ResumePoint != 0)
-      std::cout << "Resume-Point: " << std::to_string(Res.ResumePoint) << "\n";
-
+      try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint));
    if (Res.IMSHit == true)
-      std::cout << "IMS-Hit: true\n";
+      try_emplace(fields, "IMS-Hit", "true");
 
-   if (Alt != 0)
+   if (Alt != nullptr)
    {
       if (Alt->Filename.empty() == false)
-	 std::cout << "Alt-Filename: " << Alt->Filename << "\n";
-
+	 try_emplace(fields, "Alt-Filename", Alt->Filename);
       if (Alt->Size != 0)
-	 std::cout << "Alt-Size: " << std::to_string(Alt->Size) << "\n";
-
+	 try_emplace(fields, "Alt-Size", std::to_string(Alt->Size));
       if (Alt->LastModified != 0)
-	 std::cout << "Alt-Last-Modified: " << TimeRFC1123(Alt->LastModified, true) << "\n";
-
-      printHashStringList("Alt-", &Alt->Hashes);
-
+	 try_emplace(fields, "Alt-Last-Modified", TimeRFC1123(Alt->LastModified, true));
       if (Alt->IMSHit == true)
-	 std::cout << "Alt-IMS-Hit: true\n";
+	 try_emplace(fields, "Alt-IMS-Hit", "true");
+      printHashStringList(fields, "Alt-", Alt->Hashes);
    }
 
-   std::cout << "\n" << std::flush;
+   SendMessage("201 URI Done", std::move(fields));
    Dequeue();
 }
 									/*}}}*/
@@ -459,9 +471,10 @@ void pkgAcqMethod::Status(const char *Format,...)
  * the worker will enqueue again later on to the right queue */
 void pkgAcqMethod::Redirect(const string &NewURI)
 {
-   std::cout << "103 Redirect\nURI: " << Queue->Uri << "\n"
-	     << "New-URI: " << NewURI << "\n"
-	     << "\n" << std::flush;
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "URI", Queue->Uri);
+   try_emplace(fields, "New-URI", NewURI);
+   SendMessage("103 Redirect", std::move(fields));
    Dequeue();
 }
                                                                         /*}}}*/
diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h
index 2de9cf5c2..664b95c18 100644
--- a/apt-pkg/acquire-method.h
+++ b/apt-pkg/acquire-method.h
@@ -26,6 +26,7 @@
 #include <time.h>
 
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #ifndef APT_8_CLEANER_HEADERS
@@ -99,6 +100,7 @@ class pkgAcqMethod
    virtual void Fail(std::string Why, bool Transient = false);
    virtual void URIStart(FetchResult &Res);
    virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0);
+   void SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields);
 
    bool MediaFail(std::string Required,std::string Drive);
    virtual void Exit() {};
@@ -106,10 +108,16 @@ class pkgAcqMethod
    void PrintStatus(char const * const header, const char* Format, va_list &args) const;
 
    public:
-   enum CnfFlags {SingleInstance = (1<<0),
-                  Pipeline = (1<<1), SendConfig = (1<<2),
-                  LocalOnly = (1<<3), NeedsCleanup = (1<<4), 
-                  Removable = (1<<5)};
+   enum CnfFlags
+   {
+      SingleInstance = (1 << 0),
+      Pipeline = (1 << 1),
+      SendConfig = (1 << 2),
+      LocalOnly = (1 << 3),
+      NeedsCleanup = (1 << 4),
+      Removable = (1 << 5),
+      AuxRequests = (1 << 6)
+   };
 
    void Log(const char *Format,...);
    void Status(const char *Format,...);
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 995750dea..6cbf8b7aa 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -192,7 +192,8 @@ bool pkgAcquire::Worker::ReadMessages()
 // ---------------------------------------------------------------------
 /* This takes the messages from the message queue and runs them through
    the parsers in order. */
-enum class APT_HIDDEN MessageType {
+enum class APT_HIDDEN MessageType
+{
    CAPABILITIES = 100,
    LOG = 101,
    STATUS = 102,
@@ -200,6 +201,7 @@ enum class APT_HIDDEN MessageType {
    WARNING = 104,
    URI_START = 200,
    URI_DONE = 201,
+   AUX_REQUEST = 351,
    URI_FAILURE = 400,
    GENERAL_FAILURE = 401,
    MEDIA_CHANGE = 403
@@ -306,6 +308,7 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    std::string const NewURI = LookupTag(Message,"New-URI",URI.c_str());
             Itm->URI = NewURI;
+	    auto const AltUris = VectorizeString(LookupTag(Message, "Alternate-URIs"), '\n');
 
 	    ItemDone();
 
@@ -333,28 +336,14 @@ bool pkgAcquire::Worker::RunMessages()
 	       if (Log != nullptr)
 		  Log->Done(desc);
 
-	       // if we change site, treat it as a mirror change
-	       if (URI::SiteOnly(NewURI) != URI::SiteOnly(desc.URI))
-	       {
-		  auto const firstSpace = desc.Description.find(" ");
-		  if (firstSpace != std::string::npos)
-		  {
-		     std::string const OldSite = desc.Description.substr(0, firstSpace);
-		     if (likely(APT::String::Startswith(desc.URI, OldSite)))
-		     {
-			std::string const OldExtra = desc.URI.substr(OldSite.length() + 1);
-			if (likely(APT::String::Endswith(NewURI, OldExtra)))
-			{
-			   std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length());
-			   Owner->UsedMirror = URI::ArchiveOnly(NewSite);
-			   desc.Description.replace(0, firstSpace, Owner->UsedMirror);
-			}
-		     }
-		  }
-	       }
+	       ChangeSiteIsMirrorChange(NewURI, desc, Owner);
 	       desc.URI = NewURI;
 	       if (isDoomedItem(Owner) == false)
+	       {
+		  for (auto alt = AltUris.crbegin(); alt != AltUris.crend(); ++alt)
+		     Owner->PushAlternativeURI(std::string(*alt), {}, false);
 		  OwnerQ->Owner->Enqueue(desc);
+	       }
 	    }
             break;
          }
@@ -512,6 +501,42 @@ bool pkgAcquire::Worker::RunMessages()
 	    break;
 	 }
 
+	 case MessageType::AUX_REQUEST:
+	 {
+	    if (Itm == nullptr)
+	    {
+	       _error->Error("Method gave invalid Aux Request message");
+	       break;
+	    }
+	    else if (Config->GetAuxRequests() == false)
+	    {
+	       std::vector<Item *> const ItmOwners = Itm->Owners;
+	       Message.append("\nMessage: Method tried to make an Aux Request while not being allowed to do them");
+	       OwnerQ->ItemDone(Itm);
+	       Itm = nullptr;
+	       HandleFailure(ItmOwners, Config, Log, Message, false, false);
+	       ItemDone();
+
+	       std::string Msg = "600 URI Acquire\n";
+	       Msg.reserve(200);
+	       Msg += "URI: " + LookupTag(Message, "Aux-URI", "");
+	       Msg += "\nFilename: /nonexistent/auxrequest.blocked";
+	       Msg += "\n\n";
+	       if (Debug == true)
+		  clog << " -> " << Access << ':' << QuoteString(Msg, "\n") << endl;
+	       OutQueue += Msg;
+	       OutReady = true;
+	       break;
+	    }
+
+	    auto maxsizestr = LookupTag(Message, "MaximumSize", "");
+	    unsigned long long const MaxSize = maxsizestr.empty() ? 0 : strtoull(maxsizestr.c_str(), nullptr, 10);
+	    new pkgAcqAuxFile(Itm->Owner, this, LookupTag(Message, "Aux-ShortDesc", ""),
+			      LookupTag(Message, "Aux-Description", ""), LookupTag(Message, "Aux-URI", ""),
+			      GetHashesFromMessage("Aux-", Message), MaxSize);
+	    break;
+	 }
+
 	 case MessageType::URI_FAILURE:
 	 {
 	    if (Itm == nullptr)
@@ -549,44 +574,7 @@ bool pkgAcquire::Worker::RunMessages()
 		  errAuthErr = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons);
 	       }
 	    }
-
-	    for (auto const Owner: ItmOwners)
-	    {
-	       std::string NewURI;
-	       if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0)
-	       {
-		  --Owner->ModifyRetries();
-		  Owner->FailMessage(Message);
-		  auto SavedDesc = Owner->GetItemDesc();
-		  if (Log != nullptr)
-		     Log->Fail(SavedDesc);
-		  if (isDoomedItem(Owner) == false)
-		     OwnerQ->Owner->Enqueue(SavedDesc);
-	       }
-	       else if (Owner->PopAlternativeURI(NewURI))
-	       {
-		  Owner->FailMessage(Message);
-		  auto &desc = Owner->GetItemDesc();
-		  if (Log != nullptr)
-		     Log->Fail(desc);
-		  ChangeSiteIsMirrorChange(NewURI, desc, Owner);
-		  desc.URI = NewURI;
-		  if (isDoomedItem(Owner) == false)
-		     OwnerQ->Owner->Enqueue(desc);
-	       }
-	       else
-	       {
-		  if (errAuthErr && Owner->GetExpectedHashes().empty() == false)
-		     Owner->Status = pkgAcquire::Item::StatAuthError;
-		  else if (errTransient)
-		     Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
-		  auto SavedDesc = Owner->GetItemDesc();
-		  if (isDoomedItem(Owner) == false)
-		     Owner->Failed(Message, Config);
-		  if (Log != nullptr)
-		     Log->Fail(SavedDesc);
-	       }
-	    }
+	    HandleFailure(ItmOwners, Config, Log, Message, errTransient, errAuthErr);
 	    ItemDone();
 
 	    break;
@@ -604,6 +592,49 @@ bool pkgAcquire::Worker::RunMessages()
    return true;
 }
 									/*}}}*/
+void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners, /*{{{*/
+				       pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log,
+				       std::string const &Message, bool const errTransient, bool const errAuthErr)
+{
+   for (auto const Owner : ItmOwners)
+   {
+      std::string NewURI;
+      if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0)
+      {
+	 --Owner->ModifyRetries();
+	 Owner->FailMessage(Message);
+	 auto SavedDesc = Owner->GetItemDesc();
+	 if (Log != nullptr)
+	    Log->Fail(SavedDesc);
+	 if (isDoomedItem(Owner) == false)
+	    OwnerQ->Owner->Enqueue(SavedDesc);
+      }
+      else if (Owner->PopAlternativeURI(NewURI))
+      {
+	 Owner->FailMessage(Message);
+	 auto &desc = Owner->GetItemDesc();
+	 if (Log != nullptr)
+	    Log->Fail(desc);
+	 ChangeSiteIsMirrorChange(NewURI, desc, Owner);
+	 desc.URI = NewURI;
+	 if (isDoomedItem(Owner) == false)
+	    OwnerQ->Owner->Enqueue(desc);
+      }
+      else
+      {
+	 if (errAuthErr && Owner->GetExpectedHashes().empty() == false)
+	    Owner->Status = pkgAcquire::Item::StatAuthError;
+	 else if (errTransient)
+	    Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
+	 auto SavedDesc = Owner->GetItemDesc();
+	 if (isDoomedItem(Owner) == false)
+	    Owner->Failed(Message, Config);
+	 if (Log != nullptr)
+	    Log->Fail(SavedDesc);
+      }
+   }
+}
+									/*}}}*/
 // Worker::Capabilities - 100 Capabilities handler			/*{{{*/
 // ---------------------------------------------------------------------
 /* This parses the capabilities message and dumps it into the configuration
@@ -620,18 +651,13 @@ bool pkgAcquire::Worker::Capabilities(string Message)
    Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false);
    Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false);
    Config->Removable = StringToBool(LookupTag(Message,"Removable"),false);
+   Config->SetAuxRequests(StringToBool(LookupTag(Message, "AuxRequests"), false));
 
    // Some debug text
    if (Debug == true)
    {
       clog << "Configured access method " << Config->Access << endl;
-      clog << "Version:" << Config->Version <<
-	      " SingleInstance:" << Config->SingleInstance <<
-	      " Pipeline:" << Config->Pipeline <<
-	      " SendConfig:" << Config->SendConfig <<
-	      " LocalOnly: " << Config->LocalOnly <<
-	      " NeedsCleanup: " << Config->NeedsCleanup <<
-	      " Removable: " << Config->Removable << endl;
+      clog << "Version:" << Config->Version << " SingleInstance:" << Config->SingleInstance << " Pipeline:" << Config->Pipeline << " SendConfig:" << Config->SendConfig << " LocalOnly: " << Config->LocalOnly << " NeedsCleanup: " << Config->NeedsCleanup << " Removable: " << Config->Removable << " AuxRequests: " << Config->GetAuxRequests() << endl;
    }
 
    return true;
@@ -772,6 +798,46 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
    return true;
 }
 									/*}}}*/
+// Worker::ReplyAux - reply to an aux request from this worker		/*{{{*/
+bool pkgAcquire::Worker::ReplyAux(pkgAcquire::ItemDesc const &Item)
+{
+   if (OutFd == -1)
+      return false;
+
+   if (isDoomedItem(Item.Owner))
+      return true;
+
+   string Message = "600 URI Acquire\n";
+   Message.reserve(200);
+   Message += "URI: " + Item.URI;
+   if (RealFileExists(Item.Owner->DestFile))
+   {
+      if (Item.Owner->Status == pkgAcquire::Item::StatDone)
+      {
+	 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
+	 ChangeOwnerAndPermissionOfFile("Worker::ReplyAux", Item.Owner->DestFile.c_str(),
+					SandboxUser.c_str(), ROOT_GROUP, 0600);
+	 Message += "\nFilename: " + Item.Owner->DestFile;
+      }
+      else
+      {
+	 // we end up here in case we would need root-rights to delete a file,
+	 // but we run the command as non-root… (yes, it is unlikely)
+	 Message += "\nFilename: " + flCombine("/nonexistent", Item.Owner->DestFile);
+      }
+   }
+   else
+      Message += "\nFilename: " + Item.Owner->DestFile;
+   Message += "\n\n";
+
+   if (Debug == true)
+      clog << " -> " << Access << ':' << QuoteString(Message, "\n") << endl;
+   OutQueue += Message;
+   OutReady = true;
+
+   return true;
+}
+									/*}}}*/
 // Worker::OutFdRead - Out bound FD is ready				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h
index 8fc686880..f04db2b3c 100644
--- a/apt-pkg/acquire-worker.h
+++ b/apt-pkg/acquire-worker.h
@@ -276,6 +276,7 @@ class pkgAcquire::Worker : public WeakPointable
     *  queue.
     */
    bool QueueItem(pkgAcquire::Queue::QItem *Item);
+   APT_HIDDEN bool ReplyAux(pkgAcquire::ItemDesc const &Item);
 
    /** \brief Start up the worker and fill in #Config.
     *
@@ -328,6 +329,9 @@ class pkgAcquire::Worker : public WeakPointable
 
 private:
    APT_HIDDEN void PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm);
+   APT_HIDDEN void HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners,
+				 pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log,
+				 std::string const &Message, bool const errTransient, bool const errAuthErr);
 };
 
 /** @} */
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index aabcb0aba..b25a4434a 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -78,13 +78,13 @@ void pkgAcquire::Initialize()
 }
 									/*}}}*/
 // Acquire::GetLock - lock directory and prepare for action		/*{{{*/
-static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
+static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode)
 {
-   std::string const partial = parent + "partial";
-   mode_t const mode = umask(S_IWGRP | S_IWOTH);
+   std::string const partial = parent + postfix;
+   mode_t const old_umask = umask(S_IWGRP | S_IWOTH);
    bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
 	 CreateAPTDirectoryIfNeeded(parent, partial) == false);
-   umask(mode);
+   umask(old_umask);
    if (creation_fail == true)
       return false;
 
@@ -100,7 +100,7 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const
             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
       }
    }
-   if (chmod(partial.c_str(), 0700) != 0)
+   if (chmod(partial.c_str(), mode) != 0)
       _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
 
    _error->PushToStack();
@@ -117,10 +117,12 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
    if (Lock.empty())
    {
       string const listDir = _config->FindDir("Dir::State::lists");
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
+	 return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str());
       string const archivesDir = _config->FindDir("Dir::Cache::Archives");
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
       return true;
    }
@@ -137,14 +139,19 @@ bool pkgAcquire::GetLock(std::string const &Lock)
 
    if (Lock == listDir)
    {
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
    }
    if (Lock == archivesDir)
    {
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
    }
+   if (Lock == listDir || Lock == archivesDir)
+   {
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
+	 return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str());
+   }
 
    if (_config->FindB("Debug::NoLocking", false) == true)
       return true;
@@ -288,7 +295,6 @@ static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item,
 	 "\nFilename: " + Item->DestFile +
 	 "\nFailReason: WeakHashSums";
 
-      auto SavedDesc = Item->GetItemDesc();
       Item->Status = pkgAcquire::Item::StatAuthError;
       Item->Failed(Message, Config);
       if (Log != nullptr)
@@ -303,7 +309,10 @@ void pkgAcquire::Enqueue(ItemDesc &Item)
    const MethodConfig *Config;
    string Name = QueueName(Item.URI,Config);
    if (Name.empty() == true)
+   {
+      Item.Owner->Status = pkgAcquire::Item::StatError;
       return;
+   }
 
    /* the check for running avoids that we produce errors
       in logging before we actually have started, which would
@@ -769,11 +778,12 @@ bool pkgAcquire::Clean(string Dir)
    for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D))
    {
       // Skip some entries
-      if (strcmp(E->d_name,"lock") == 0 ||
-	  strcmp(E->d_name,"partial") == 0 ||
-	  strcmp(E->d_name,"lost+found") == 0 ||
-	  strcmp(E->d_name,".") == 0 ||
-	  strcmp(E->d_name,"..") == 0)
+      if (strcmp(E->d_name, "lock") == 0 ||
+	  strcmp(E->d_name, "partial") == 0 ||
+	  strcmp(E->d_name, "auxfiles") == 0 ||
+	  strcmp(E->d_name, "lost+found") == 0 ||
+	  strcmp(E->d_name, ".") == 0 ||
+	  strcmp(E->d_name, "..") == 0)
 	 continue;
 
       // Look in the get list and if not found nuke
@@ -845,12 +855,25 @@ pkgAcquire::UriIterator pkgAcquire::UriEnd()
 }
 									/*}}}*/
 // Acquire::MethodConfig::MethodConfig - Constructor			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
-   Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
-   Removable(false)
+class pkgAcquire::MethodConfig::Private
+{
+   public:
+   bool AuxRequests = false;
+};
+pkgAcquire::MethodConfig::MethodConfig() : d(new Private()), Next(0), SingleInstance(false),
+					   Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
+					   Removable(false)
+{
+}
+									/*}}}*/
+bool pkgAcquire::MethodConfig::GetAuxRequests() const /*{{{*/
+{
+   return d->AuxRequests;
+}
+									/*}}}*/
+void pkgAcquire::MethodConfig::SetAuxRequests(bool const value) /*{{{*/
 {
+   d->AuxRequests = value;
 }
 									/*}}}*/
 // Queue::Queue - Constructor						/*{{{*/
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index e58aeef65..1cf4da5bf 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -636,9 +636,10 @@ class pkgAcquire::UriIterator
 /** \brief Information about the properties of a single acquire method.	{{{*/
 struct pkgAcquire::MethodConfig
 {
+   class Private;
    /** \brief dpointer placeholder (for later in case we need it) */
-   void * const d;
-   
+   Private *const d;
+
    /** \brief The next link on the acquire method list.
     *
     *  \todo Why not an STL container?
@@ -688,6 +689,9 @@ struct pkgAcquire::MethodConfig
     */
    MethodConfig();
 
+   APT_HIDDEN bool GetAuxRequests() const;
+   APT_HIDDEN void SetAuxRequests(bool const value);
+
    virtual ~MethodConfig();
 };
 									/*}}}*/
diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc
index 04a7c4910..1abc638ee 100644
--- a/apt-pkg/clean.cc
+++ b/apt-pkg/clean.cc
@@ -62,11 +62,12 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache)
    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
    {
       // Skip some files..
-      if (strcmp(Dir->d_name,"lock") == 0 ||
-	  strcmp(Dir->d_name,"partial") == 0 ||
-	  strcmp(Dir->d_name,"lost+found") == 0 ||
-	  strcmp(Dir->d_name,".") == 0 ||
-	  strcmp(Dir->d_name,"..") == 0)
+      if (strcmp(Dir->d_name, "lock") == 0 ||
+	  strcmp(Dir->d_name, "partial") == 0 ||
+	  strcmp(Dir->d_name, "auxfiles") == 0 ||
+	  strcmp(Dir->d_name, "lost+found") == 0 ||
+	  strcmp(Dir->d_name, ".") == 0 ||
+	  strcmp(Dir->d_name, "..") == 0)
 	 continue;
 
       struct stat St;
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index 207f0d902..5c34113e7 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -138,7 +138,6 @@ bool pkgInitConfig(Configuration &Cnf)
    Cnf.CndSet("Dir::State", STATE_DIR + 1);
    Cnf.CndSet("Dir::State::lists","lists/");
    Cnf.CndSet("Dir::State::cdroms","cdroms.list");
-   Cnf.CndSet("Dir::State::mirrors","mirrors/");
 
    // Cache
    Cnf.CndSet("Dir::Cache", CACHE_DIR + 1);