Merge tag '1.3_rc2' into ubuntu

apt Debian release 1.3~rc2

11 files changed, 241 insertions, 155 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index f13d2f6ae..7abb7b206 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -410,7 +410,7 @@ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
       return false;
    }
    // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too
-   if (TransactionManager->BaseURI.empty() == false &&
+   if (TransactionManager->BaseURI.empty() == false && UsedMirror.empty() &&
 	 URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI))
    {
       // this ensures we rewrite only once and only the first step
@@ -451,7 +451,11 @@ std::string pkgAcquire::Item::GetFinalFilename() const
 }
 std::string pkgAcqDiffIndex::GetFinalFilename() const
 {
-   return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+   std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+   // we don't want recompress, so lets keep whatever we got
+   if (CurrentCompressionExtension == "uncompressed")
+      return FinalFile;
+   return FinalFile + "." + CurrentCompressionExtension;
 }
 std::string pkgAcqIndex::GetFinalFilename() const
 {
@@ -488,7 +492,10 @@ std::string pkgAcqIndex::GetMetaKey() const
 }
 std::string pkgAcqDiffIndex::GetMetaKey() const
 {
-   return GetDiffIndexFileName(Target.MetaKey);
+   auto const metakey = GetDiffIndexFileName(Target.MetaKey);
+   if (CurrentCompressionExtension == "uncompressed")
+      return metakey;
+   return metakey + "." + CurrentCompressionExtension;
 }
 									/*}}}*/
 //pkgAcqTransactionItem::TransactionState and specialisations for child classes	/*{{{*/
@@ -991,6 +998,9 @@ std::string pkgAcquire::Item::HashSum() const				/*{{{*/
 									/*}}}*/
 bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI)	/*{{{*/
 {
+   // store can fail due to permission errors and the item will "loop" then
+   if (APT::String::Startswith(NewURI, "store:"))
+      return false;
    if (d->PastRedirections.empty())
    {
       d->PastRedirections.push_back(NewURI);
@@ -1223,9 +1233,15 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st
    if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
    {
       if (APT::String::Endswith(I->Desc.URI, "InRelease"))
+      {
 	 TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("InRelease"));
+	 TransactionManager->UsedMirror = I->UsedMirror;
+      }
       else if (APT::String::Endswith(I->Desc.URI, "Release"))
+      {
 	 TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release"));
+	 TransactionManager->UsedMirror = I->UsedMirror;
+      }
    }
 
    std::string const FileName = LookupTag(Message,"Filename");
@@ -1962,44 +1978,33 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
                                  pkgAcqMetaClearSig * const TransactionManager,
                                  IndexTarget const &Target)
-   : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
+   : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
 {
    // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
    ExpectedAdditionalItems = 40;
-
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
-   Desc.Owner = this;
-   Desc.Description = GetDiffIndexFileName(Target.Description);
-   Desc.ShortDesc = Target.ShortDesc;
-   Desc.URI = GetDiffIndexURI(Target);
-
-   DestFile = GetPartialFileNameFromURI(Desc.URI);
+   CompressionExtensions.clear();
+   {
+      std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+      if (types.empty() == false)
+      {
+	 std::ostringstream os;
+	 std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
+	       if (type == "uncompressed")
+	          return true;
+	       return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
+	 });
+	 os << *types.rbegin();
+	 CompressionExtensions = os.str();
+      }
+   }
+   if (Target.Option(IndexTarget::COMPRESSIONTYPES).find("by-hash") != std::string::npos)
+      CompressionExtensions = "by-hash " + CompressionExtensions;
+   Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc);
 
    if(Debug)
       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
-
-   QueueURI(Desc);
-}
-									/*}}}*/
-// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
-// ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-string pkgAcqDiffIndex::Custom600Headers() const
-{
-   if (TransactionManager->LastMetaIndexParser != NULL)
-      return "\nIndex-File: true";
-
-   string const Final = GetFinalFilename();
-
-   if(Debug)
-      std::clog << "Custom600Header-IMS: " << Final << std::endl;
-
-   struct stat Buf;
-   if (stat(Final.c_str(),&Buf) != 0)
-      return "\nIndex-File: true";
-   
-   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
 }
 									/*}}}*/
 void pkgAcqDiffIndex::QueueOnIMSHit() const				/*{{{*/
@@ -2030,7 +2035,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 	 << std::endl;
 
-   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+   FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
    pkgTagFile TF(&Fd);
    if (Fd.IsOpen() == false || Fd.Failed())
       return false;
@@ -2402,7 +2407,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 									/*}}}*/
 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 {
-   pkgAcqBaseIndex::Failed(Message,Cnf);
+   if (CommonFailed(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Message, Cnf))
+      return;
+
    Status = StatDone;
    ExpectedAdditionalItems = 0;
 
@@ -2820,10 +2827,12 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
                          pkgAcqMetaClearSig * const TransactionManager,
-                         IndexTarget const &Target)
+                         IndexTarget const &Target, bool const Derived)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
    CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
 {
+   if (Derived)
+      return;
    Init(Target.URI, Target.Description, Target.ShortDesc);
 
    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
@@ -2856,10 +2865,6 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
    DestFile = GetPartialFileNameFromURI(URI);
    NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
 
-   // store file size of the download to ensure the fetcher gives
-   // accurate progress reporting
-   FileSize = GetExpectedHashes().FileSize();
-
    if (CurrentCompressionExtension == "uncompressed")
    {
       Desc.URI = URI;
@@ -2874,6 +2879,8 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
 	 Desc.URI = URI + '.' + CurrentCompressionExtension;
 	 DestFile = DestFile + '.' + CurrentCompressionExtension;
       }
+      else
+	 Desc.URI = URI;
 
       HashStringList const Hashes = GetExpectedHashes();
       HashString const * const TargetHash = Hashes.find(NULL);
@@ -2896,6 +2903,9 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
       DestFile = DestFile + '.' + CurrentCompressionExtension;
    }
 
+   // store file size of the download to ensure the fetcher gives
+   // accurate progress reporting
+   FileSize = GetExpectedHashes().FileSize();
 
    Desc.Description = URIDesc;
    Desc.Owner = this;
@@ -2928,20 +2938,40 @@ string pkgAcqIndex::Custom600Headers() const
 }
 									/*}}}*/
 // AcqIndex::Failed - getting the indexfile failed			/*{{{*/
-void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
+bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
+      std::string const &Message, pkgAcquire::MethodConfig const * const Cnf)
 {
    pkgAcqBaseIndex::Failed(Message,Cnf);
 
+   if (UsedMirror.empty() == false && UsedMirror != "DIRECT" &&
+	 LookupTag(Message, "FailReason") == "HttpError404")
+   {
+      UsedMirror = "DIRECT";
+      if (Desc.URI.find("/by-hash/") != std::string::npos)
+	 CompressionExtensions = "by-hash " + CompressionExtensions;
+      else
+	 CompressionExtensions = CurrentCompressionExtension + ' ' + CompressionExtensions;
+      Init(TargetURI, TargetDesc, Desc.ShortDesc);
+      Status = StatIdle;
+      return true;
+   }
+
    // authorisation matches will not be fixed by other compression types
    if (Status != StatAuthError)
    {
       if (CompressionExtensions.empty() == false)
       {
-	 Init(Target.URI, Desc.Description, Desc.ShortDesc);
+	 Init(TargetURI, Desc.Description, Desc.ShortDesc);
 	 Status = StatIdle;
-	 return;
+	 return true;
       }
    }
+   return false;
+}
+void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
+{
+   if (CommonFailed(Target.URI, Target.Description, Message, Cnf))
+      return;
 
    if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
       Status = StatDone;
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index e6e5ea12b..71a11bcde 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -611,6 +611,79 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
    virtual ~pkgAcqBaseIndex();
 };
 									/*}}}*/
+/** \brief An acquire item that is responsible for fetching an index	{{{
+ *  file (e.g., Packages or Sources).
+ *
+ *  \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans
+ *
+ *  \todo Why does pkgAcqIndex have protected members?
+ */
+class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
+{
+   void * const d;
+
+   protected:
+
+   /** \brief The stages the method goes through
+    *
+    *  The method first downloads the indexfile, then its decompressed (or
+    *  copied) and verified
+    */
+   enum AllStages {
+      STAGE_DOWNLOAD,
+      STAGE_DECOMPRESS_AND_VERIFY,
+   };
+   AllStages Stage;
+
+   /** \brief Handle what needs to be done when the download is done */
+   void StageDownloadDone(std::string const &Message);
+
+   /** \brief Handle what needs to be done when the decompression/copy is
+    *         done
+    */
+   void StageDecompressDone();
+
+   /** \brief If \b set, this partially downloaded file will be
+    *  removed when the download completes.
+    */
+   std::string EraseFileName;
+
+   /** \brief The compression-related file extensions that are being
+    *  added to the downloaded file one by one if first fails (e.g., "gz bz2").
+    */
+   std::string CompressionExtensions;
+
+   /** \brief The actual compression extension currently used */
+   std::string CurrentCompressionExtension;
+
+   /** \brief Do the changes needed to fetch via AptByHash (if needed) */
+   void InitByHashIfNeeded();
+
+   /** \brief Get the full pathname of the final file for the current URI */
+   virtual std::string GetFinalFilename() const APT_OVERRIDE;
+
+   virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE;
+
+   public:
+   // Specialized action members
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
+   virtual std::string Custom600Headers() const APT_OVERRIDE;
+   virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;};
+   virtual std::string GetMetaKey() const APT_OVERRIDE;
+
+   pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager,
+               IndexTarget const &Target, bool const Derived = false) APT_NONNULL(2, 3);
+   virtual ~pkgAcqIndex();
+
+   protected:
+   APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
+             std::string const &ShortDesc);
+   APT_HIDDEN bool CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
+             std::string const &Message, pkgAcquire::MethodConfig const * const Cnf);
+};
+									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
  *  package list diffs and starting the package list's download.
  *
@@ -620,7 +693,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
  *
  *  \sa pkgAcqIndexDiffs, pkgAcqIndex
  */
-class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
+class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqIndex
 {
    void * const d;
    std::vector<pkgAcqIndexMergeDiffs*> * diffs;
@@ -646,7 +719,6 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
    virtual void Done(std::string const &Message, HashStringList const &Hashes,
 		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";};
-   virtual std::string Custom600Headers() const APT_OVERRIDE;
    virtual std::string GetMetaKey() const APT_OVERRIDE;
 
    /** \brief Parse the Index file for a set of Packages diffs.
@@ -887,77 +959,6 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
    virtual ~pkgAcqIndexDiffs();
 };
 									/*}}}*/
-/** \brief An acquire item that is responsible for fetching an index	{{{
- *  file (e.g., Packages or Sources).
- *
- *  \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans
- *
- *  \todo Why does pkgAcqIndex have protected members?
- */
-class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
-{
-   void * const d;
-
-   protected:
-
-   /** \brief The stages the method goes through
-    *
-    *  The method first downloads the indexfile, then its decompressed (or
-    *  copied) and verified
-    */
-   enum AllStages {
-      STAGE_DOWNLOAD,
-      STAGE_DECOMPRESS_AND_VERIFY,
-   };
-   AllStages Stage;
-
-   /** \brief Handle what needs to be done when the download is done */
-   void StageDownloadDone(std::string const &Message);
-
-   /** \brief Handle what needs to be done when the decompression/copy is
-    *         done 
-    */
-   void StageDecompressDone();
-
-   /** \brief If \b set, this partially downloaded file will be
-    *  removed when the download completes.
-    */
-   std::string EraseFileName;
-
-   /** \brief The compression-related file extensions that are being
-    *  added to the downloaded file one by one if first fails (e.g., "gz bz2").
-    */
-   std::string CompressionExtensions;
-
-   /** \brief The actual compression extension currently used */
-   std::string CurrentCompressionExtension;
-
-   /** \brief Do the changes needed to fetch via AptByHash (if needed) */
-   void InitByHashIfNeeded();
-
-   /** \brief Get the full pathname of the final file for the current URI */
-   virtual std::string GetFinalFilename() const APT_OVERRIDE;
-
-   virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE;
-
-   public:
-   // Specialized action members
-   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
-   virtual void Done(std::string const &Message, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
-   virtual std::string Custom600Headers() const APT_OVERRIDE;
-   virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;};
-   virtual std::string GetMetaKey() const APT_OVERRIDE;
-
-   pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager,
-               IndexTarget const &Target) APT_NONNULL(2, 3);
-   virtual ~pkgAcqIndex();
-
-   private:
-   APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
-             std::string const &ShortDesc);
-};
-									/*}}}*/
 /** \brief An item that is responsible for fetching a package file.	{{{
  *
  *  If the package file already exists in the cache, nothing will be
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 7a4f8177f..a4fbc7651 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -464,6 +464,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       }
 	       else
 	       {
+		  auto SavedDesc = Owner->GetItemDesc();
 		  if (isDoomedItem(Owner) == false)
 		  {
 		     if (Message.find("\nFailReason:") == std::string::npos)
@@ -476,7 +477,7 @@ bool pkgAcquire::Worker::RunMessages()
 		     Owner->Failed(Message,Config);
 		  }
 		  if (Log != nullptr)
-		     Log->Fail(Owner->GetItemDesc());
+		     Log->Fail(SavedDesc);
 	       }
 	    }
 	    ItemDone();
@@ -524,11 +525,11 @@ bool pkgAcquire::Worker::RunMessages()
 		  Owner->Status = pkgAcquire::Item::StatAuthError;
 	       else if (errTransient)
 		  Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
-
+	       auto SavedDesc = Owner->GetItemDesc();
 	       if (isDoomedItem(Owner) == false)
 		  Owner->Failed(Message,Config);
 	       if (Log != nullptr)
-		  Log->Fail(Owner->GetItemDesc());
+		  Log->Fail(SavedDesc);
 	    }
 	    ItemDone();
 
@@ -685,6 +686,25 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
       return true;
    }
 
+   if (Item->Owner->IsRedirectionLoop(Item->URI))
+   {
+      std::string const Message = "400 URI Failure"
+	 "\nURI: " + Item->URI +
+	 "\nFilename: " + Item->Owner->DestFile +
+	 "\nFailReason: RedirectionLoop";
+
+      auto const ItmOwners = Item->Owners;
+      for (auto &O: ItmOwners)
+      {
+	 O->Status = pkgAcquire::Item::StatError;
+	 O->Failed(Message, Config);
+	 if (Log != nullptr)
+	    Log->Fail(O->GetItemDesc());
+      }
+      // "queued" successfully, the item just instantly failed
+      return true;
+   }
+
    string Message = "600 URI Acquire\n";
    Message.reserve(300);
    Message += "URI: " + Item->URI;
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index a9d51a6bf..fd13b45dc 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -1908,11 +1908,12 @@ public:
 	       " but was forced to ignore it in favor of an external binary – which isn't installed.", compressor.Name.c_str());
 
       bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly;
-      if (Comp == false)
+      if (Comp == false && filefd->iFd != -1)
       {
 	 // Handle 'decompression' of empty files
 	 struct stat Buf;
-	 fstat(filefd->iFd, &Buf);
+	 if (fstat(filefd->iFd, &Buf) != 0)
+	    return filefd->FileFdErrno("fstat", "Could not stat fd %d for file %s", filefd->iFd, filefd->FileName.c_str());
 	 if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false)
 	    return true;
 
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index 4a1676dc2..15665f8b5 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -164,7 +164,7 @@ bool RemoveFile(char const * const Function, std::string const &FileName);
 int GetLock(std::string File,bool Errors = true);
 bool FileExists(std::string File);
 bool RealFileExists(std::string File);
-bool DirectoryExists(std::string const &Path) APT_CONST;
+bool DirectoryExists(std::string const &Path);
 bool CreateDirectory(std::string const &Parent, std::string const &Path);
 time_t GetModificationTime(std::string const &Path);
 bool Rename(std::string From, std::string To);
diff --git a/apt-pkg/contrib/srvrec.cc b/apt-pkg/contrib/srvrec.cc
index be159bad9..327e59937 100644
--- a/apt-pkg/contrib/srvrec.cc
+++ b/apt-pkg/contrib/srvrec.cc
@@ -185,7 +185,7 @@ SrvRec PopFromSrvRecs(std::vector<SrvRec> &Recs)
 	 [&I](SrvRec const &J) { return I->priority != J.priority; });
 
    // clock seems random enough.
-   I += clock() % std::distance(I, J);
+   I += std::max(static_cast<clock_t>(0), clock()) % std::distance(I, J);
    SrvRec const selected = std::move(*I);
    Recs.erase(I);
 
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 6a23b2c00..65bd3e6ee 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -80,7 +80,11 @@ debPackagesIndex::debPackagesIndex(IndexTarget const &Target, bool const Trusted
 std::string debPackagesIndex::ArchiveInfo(pkgCache::VerIterator const &Ver) const
 {
    std::string Res = Target.Description;
-   Res.erase(Target.Description.rfind(' '));
+   {
+      auto const space = Target.Description.rfind(' ');
+      if (space != std::string::npos)
+	 Res.erase(space);
+   }
 
    Res += " ";
    Res += Ver.ParentPkg().Name();
diff --git a/apt-pkg/deb/debsystem.cc b/apt-pkg/deb/debsystem.cc
index fd702b3c6..f7968ec47 100644
--- a/apt-pkg/deb/debsystem.cc
+++ b/apt-pkg/deb/debsystem.cc
@@ -184,26 +184,24 @@ pkgPackageManager *debSystem::CreatePM(pkgDepCache *Cache) const
 // System::Initialize - Setup the configuration space..			/*{{{*/
 // ---------------------------------------------------------------------
 /* These are the Debian specific configuration variables.. */
-static std::string getDpkgStatusLocation(Configuration &Cnf) {
+static std::string getDpkgStatusLocation(Configuration const &Cnf) {
+   Configuration PathCnf;
+   PathCnf.Set("Dir", Cnf.Find("Dir", "/"));
+   PathCnf.Set("Dir::State::status", "status");
    auto const cnfstatedir = Cnf.Find("Dir::State", "var/lib/apt/");
+   // if the state dir ends in apt, replace it with dpkg -
+   // for the default this gives us the same as the fallback below.
+   // This can't be a ../dpkg as that would play bad with symlinks
    std::string statedir;
    if (APT::String::Endswith(cnfstatedir, "/apt/"))
       statedir.assign(cnfstatedir, 0, cnfstatedir.length() - 5);
    else if (APT::String::Endswith(cnfstatedir, "/apt"))
       statedir.assign(cnfstatedir, 0, cnfstatedir.length() - 4);
    if (statedir.empty())
-      Cnf.Set("Dir::State", "var/lib/dpkg");
+      PathCnf.Set("Dir::State", "var/lib/dpkg");
    else
-      Cnf.Set("Dir::State", flCombine(statedir, "dpkg"));
-   auto const cnfrootdir = Cnf.Find("RootDir");
-   if (Cnf.Exists("RootDir") == true)
-      Cnf.Set("RootDir", "");
-   Cnf.Set("Dir::State::status", "status");
-   auto const statusfile = Cnf.FindFile("Dir::State::status");
-   if (cnfrootdir.empty() == false)
-      Cnf.Set("RootDir", cnfrootdir);
-   Cnf.Set("Dir::State", cnfstatedir);
-   return statusfile;
+      PathCnf.Set("Dir::State", flCombine(statedir, "dpkg"));
+   return PathCnf.FindFile("Dir::State::status");
 }
 bool debSystem::Initialize(Configuration &Cnf)
 {
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index b1435d32c..934943205 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -151,7 +151,13 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(ALLOW_WEAK);
       APT_CASE(ALLOW_DOWNGRADE_TO_INSECURE);
 #undef APT_CASE
-      case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
+      case FILENAME:
+      {
+	 auto const M = Options.find("FILENAME");
+	 if (M == Options.end())
+	    return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
+	 return M->second;
+      }
       case EXISTING_FILENAME:
 	 std::string const filename = Option(FILENAME);
 	 std::vector<std::string> const types = VectorizeString(Option(COMPRESSIONTYPES), ' ');
diff --git a/apt-pkg/install-progress.cc b/apt-pkg/install-progress.cc
index c77c240c3..be5b8b5c7 100644
--- a/apt-pkg/install-progress.cc
+++ b/apt-pkg/install-progress.cc
@@ -14,6 +14,7 @@
 #include <algorithm>
 #include <stdio.h>
 #include <sstream>
+#include <cmath>
 
 #include <apti18n.h>
 
@@ -321,19 +322,14 @@ std::string
 PackageManagerFancy::GetTextProgressStr(float Percent, int OutputSize)
 {
    std::string output;
-   int i;
-   
-   // should we raise a exception here instead?
-   if (Percent < 0.0 || Percent > 1.0 || OutputSize < 3)
+   if (unlikely(OutputSize < 3))
       return output;
-   
-   int BarSize = OutputSize - 2; // bar without the leading "[" and trailing "]"
-   output += "[";
-   for(i=0; i < BarSize*Percent; i++)
-      output += "#";
-   for (/*nothing*/; i < BarSize; i++)
-      output += ".";
-   output += "]";
+
+   int const BarSize = OutputSize - 2; // bar without the leading "[" and trailing "]"
+   int const BarDone = std::max(0, std::min(BarSize, static_cast<int>(std::floor(Percent * BarSize))));
+   output.append("[");
+   std::fill_n(std::fill_n(std::back_inserter(output), BarDone, '#'), BarSize - BarDone, '.');
+   output.append("]");
    return output;
 }
 
diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index 000539582..cfd824978 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -328,6 +328,9 @@ bool pkgSourceList::ReadMainList()
       // Only warn if there is no sources.list file.
       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 
+   for (auto && file: _config->FindVector("APT::Sources::With"))
+      Res &= AddVolatileFile(file, nullptr);
+
    return Res;
 }
 									/*}}}*/
@@ -420,18 +423,18 @@ bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 /* Returns: the number of stanzas parsed*/
 bool pkgSourceList::ParseFileDeb822(string const &File)
 {
-   unsigned int i = 1;
-
    // see if we can read the file
    FileFd Fd(File, FileFd::ReadOnly);
    pkgTagFile Sources(&Fd, pkgTagFile::SUPPORT_COMMENTS);
    if (Fd.IsOpen() == false || Fd.Failed())
-      return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
+      return _error->Error(_("Malformed stanza %u in source list %s (type)"),0,File.c_str());
 
    // read step by step
    pkgTagSection Tags;
+   unsigned int i = 0;
    while (Sources.Step(Tags) == true)
    {
+      ++i;
       if(Tags.Exists("Types") == false)
 	 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 
@@ -449,8 +452,6 @@ bool pkgSourceList::ParseFileDeb822(string const &File)
 
          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
             return false;
-
-         ++i;
       }
    }
    return true;
@@ -544,6 +545,22 @@ void pkgSourceList::AddVolatileFile(pkgIndexFile * const File)		/*{{{*/
       VolatileFiles.push_back(File);
 }
 									/*}}}*/
+static bool fileNameMatches(std::string const &filename, std::string const &idxtype)/*{{{*/
+{
+   for (auto && type: APT::Configuration::getCompressionTypes())
+   {
+      if (type == "uncompressed")
+      {
+	 if (filename == idxtype || APT::String::Endswith(filename, '_' + idxtype))
+	    return true;
+      }
+      else if (filename == idxtype + '.' + type ||
+	    APT::String::Endswith(filename, '_' + idxtype + '.' + type))
+	 return true;
+   }
+   return false;
+}
+									/*}}}*/
 bool pkgSourceList::AddVolatileFile(std::string const &File, std::vector<std::string> * const VolatileCmdL)/*{{{*/
 {
    // Note: FileExists matches directories and links, too!
@@ -574,7 +591,20 @@ bool pkgSourceList::AddVolatileFile(std::string const &File, std::vector<std::st
       return true;
    }
    else
-      return false;
+   {
+      auto const filename = flNotDir(File);
+      auto const Target = IndexTarget(File, filename, File, "file:" + File, false, true, {
+	 { "FILENAME", File },
+	 { "REPO_URI", "file:" + flAbsPath(flNotFile(File)) + '/' },
+	 { "COMPONENT", "volatile-packages-file" },
+      });
+      if (fileNameMatches(filename, "Packages"))
+	 AddVolatileFile(new debPackagesIndex(Target, true));
+      else if (fileNameMatches(filename, "Sources"))
+	 AddVolatileFile(new debSourcesIndex(Target, true));
+      else
+	 return false;
+   }
 
    if (VolatileCmdL != nullptr)
       VolatileCmdL->push_back(File);