Merge remote-tracking branch 'upstream/debian/experimental' into feature/srv-records

17 files changed, 733 insertions, 689 deletions

diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index 0c606e48e..62b2c13d0 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -49,6 +49,16 @@ void AcqTextStatus::Start()
    ID = 1;
 }
 									/*}}}*/
+void AcqTextStatus::AssignItemID(pkgAcquire::ItemDesc &Itm)		/*{{{*/
+{
+   /* In theory calling it from Fetch() would be enough, but to be
+      safe we call it from IMSHit and Fail as well.
+      Also, an Item can pass through multiple stages, so ensure
+      that it keeps the same number */
+   if (Itm.Owner->ID == 0)
+      Itm.Owner->ID = ID++;
+}
+									/*}}}*/
 // AcqTextStatus::IMSHit - Called when an item got a HIT response	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -57,9 +67,11 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
+   AssignItemID(Itm);
    clearLastLine();
 
-   out << _("Hit ") << Itm.Description;
+   // TRANSLATOR: Very short word to be displayed before unchanged files in 'apt-get update'
+   ioprintf(out, _("Hit:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
    out << std::endl;
    Update = true;
 }
@@ -72,15 +84,16 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
    Update = true;
    if (Itm.Owner->Complete == true)
       return;
-
-   Itm.Owner->ID = ID++;
+   AssignItemID(Itm);
 
    if (Quiet > 1)
       return;
 
    clearLastLine();
 
-   out << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
+   // TRANSLATOR: Very short word to be displayed for files processed in 'apt-get update'
+   // Potentially replaced later by "Hit:", "Ign:" or "Err:" if something (bad) happens
+   ioprintf(out, _("Get:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
    if (Itm.Owner->FileSize != 0)
       out << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
    out << std::endl;
@@ -89,9 +102,10 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
 // AcqTextStatus::Done - Completed a download				/*{{{*/
 // ---------------------------------------------------------------------
 /* We don't display anything... */
-void AcqTextStatus::Done(pkgAcquire::ItemDesc &/*Itm*/)
+void AcqTextStatus::Done(pkgAcquire::ItemDesc &Itm)
 {
    Update = true;
+   AssignItemID(Itm);
 }
 									/*}}}*/
 // AcqTextStatus::Fail - Called when an item fails to download		/*{{{*/
@@ -102,23 +116,25 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   // Ignore certain kinds of transient failures (bad code)
-   if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
-      return;
-
+   AssignItemID(Itm);
    clearLastLine();
 
-   if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
+   if (Itm.Owner->Status == pkgAcquire::Item::StatDone || Itm.Owner->Status == pkgAcquire::Item::StatIdle)
    {
-      out << _("Ign ") << Itm.Description << std::endl;
+      // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
+      // which failed to download, but the error is ignored (compare "Err:")
+      ioprintf(out, _("Ign:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
       if (Itm.Owner->ErrorText.empty() == false &&
 	    _config->FindB("Acquire::Progress::Ignore::ShowErrorText", false) == true)
-	 out << "  " << Itm.Owner->ErrorText << std::endl;
+	 out << std::endl << "  " << Itm.Owner->ErrorText;
+      out << std::endl;
    }
    else
    {
-      out << _("Err ") << Itm.Description << std::endl;
-      out << "  " << Itm.Owner->ErrorText << std::endl;
+      // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
+      // which failed to download and the error is critical (compare "Ign:")
+      ioprintf(out, _("Err:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
+      out << std::endl << "  " << Itm.Owner->ErrorText << std::endl;
    }
 
    Update = true;
@@ -280,7 +296,7 @@ bool AcqTextStatus::MediaChange(std::string Media, std::string Drive)
    clearLastLine();
    ioprintf(out,_("Media change: please insert the disc labeled\n"
 		   " '%s'\n"
-		   "in the drive '%s' and press enter\n"),
+		   "in the drive '%s' and press [Enter]\n"),
 	    Media.c_str(),Drive.c_str());
 
    char C = 0;
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index 7cf990c65..6b6d555b1 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -23,19 +23,20 @@ class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
    unsigned long ID;
    unsigned long Quiet;
 
-   void clearLastLine();
+   APT_HIDDEN void clearLastLine();
+   APT_HIDDEN void AssignItemID(pkgAcquire::ItemDesc &Itm);
 
    public:
 
-   virtual bool MediaChange(std::string Media,std::string Drive);
-   virtual void IMSHit(pkgAcquire::ItemDesc &Itm);
-   virtual void Fetch(pkgAcquire::ItemDesc &Itm);
-   virtual void Done(pkgAcquire::ItemDesc &Itm);
-   virtual void Fail(pkgAcquire::ItemDesc &Itm);
-   virtual void Start();
-   virtual void Stop();
+   virtual bool MediaChange(std::string Media,std::string Drive) APT_OVERRIDE;
+   virtual void IMSHit(pkgAcquire::ItemDesc &Itm) APT_OVERRIDE;
+   virtual void Fetch(pkgAcquire::ItemDesc &Itm) APT_OVERRIDE;
+   virtual void Done(pkgAcquire::ItemDesc &Itm) APT_OVERRIDE;
+   virtual void Fail(pkgAcquire::ItemDesc &Itm) APT_OVERRIDE;
+   virtual void Start() APT_OVERRIDE;
+   virtual void Stop() APT_OVERRIDE;
 
-   bool Pulse(pkgAcquire *Owner);
+   bool Pulse(pkgAcquire *Owner) APT_OVERRIDE;
 
    AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet);
 };
diff --git a/apt-private/private-cachefile.cc b/apt-private/private-cachefile.cc
index 29e665245..32cad1c33 100644
--- a/apt-private/private-cachefile.cc
+++ b/apt-private/private-cachefile.cc
@@ -7,7 +7,7 @@
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/depcache.h>
 #include <apt-pkg/pkgcache.h>
-#include <apt-pkg/cacheiterators.h>
+#include <apt-pkg/cacheset.h>
 
 #include <apt-private/private-output.h>
 #include <apt-private/private-cachefile.h>
@@ -21,39 +21,40 @@
 
 using namespace std;
 
-// CacheFile::NameComp - QSort compare by name				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-pkgCache *CacheFile::SortCache = 0;
-int CacheFile::NameComp(const void *a,const void *b)
+static bool SortPackagesByName(pkgCache * const Owner,
+      map_pointer_t const A, map_pointer_t const B)
 {
-   if (*(pkgCache::Package **)a == 0 || *(pkgCache::Package **)b == 0)
-      return *(pkgCache::Package **)a - *(pkgCache::Package **)b;
-   
-   const pkgCache::Package &A = **(pkgCache::Package **)a;
-   const pkgCache::Package &B = **(pkgCache::Package **)b;
-   const pkgCache::Group * const GA = SortCache->GrpP + A.Group;
-   const pkgCache::Group * const GB = SortCache->GrpP + B.Group;
-
-   return strcmp(SortCache->StrP + GA->Name,SortCache->StrP + GB->Name);
+   if (A == 0)
+      return false;
+   if (B == 0 || A == B)
+      return true;
+   pkgCache::Group const * const GA = Owner->GrpP + A;
+   pkgCache::Group const * const GB = Owner->GrpP + B;
+   return strcmp(Owner->StrP + GA->Name, Owner->StrP + GB->Name) <= 0;
 }
-									/*}}}*/
-// CacheFile::Sort - Sort by name					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void CacheFile::Sort()
+SortedPackageUniverse::SortedPackageUniverse(CacheFile &Cache) :
+      PackageUniverse{Cache}, List(Cache.UniverseList)
 {
-   delete [] List;
-   List = new pkgCache::Package *[Cache->Head().PackageCount];
-   memset(List,0,sizeof(*List)*Cache->Head().PackageCount);
-   pkgCache::PkgIterator I = Cache->PkgBegin();
-   for (;I.end() != true; ++I)
-      List[I->ID] = I;
-
-   SortCache = *this;
-   qsort(List,Cache->Head().PackageCount,sizeof(*List),NameComp);
 }
-									/*}}}*/
+void SortedPackageUniverse::LazyInit() const
+{
+   if (List.empty() == false)
+      return;
+   pkgCache * const Owner = data();
+   // In Multi-Arch systems Grps are easier to sort than Pkgs
+   std::vector<map_pointer_t> GrpList;
+   List.reserve(Owner->Head().GroupCount);
+   for (pkgCache::GrpIterator I{Owner->GrpBegin()}; I.end() != true; ++I)
+      GrpList.emplace_back(I - Owner->GrpP);
+   std::stable_sort(GrpList.begin(), GrpList.end(), std::bind( &SortPackagesByName, Owner, std::placeholders::_1, std::placeholders::_2 ));
+   List.reserve(Owner->Head().PackageCount);
+   for (auto G : GrpList)
+   {
+      pkgCache::GrpIterator const Grp(*Owner, Owner->GrpP + G);
+      for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() != true; P = Grp.NextPkg(P))
+	 List.emplace_back(P - Owner->PkgP);
+   }
+}
 // CacheFile::CheckDeps - Open the cache file				/*{{{*/
 // ---------------------------------------------------------------------
 /* This routine generates the caches and then opens the dependency cache
diff --git a/apt-private/private-cachefile.h b/apt-private/private-cachefile.h
index 1fddabfbd..51703b0ad 100644
--- a/apt-private/private-cachefile.h
+++ b/apt-private/private-cachefile.h
@@ -7,32 +7,16 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/macros.h>
 #include <apt-pkg/sourcelist.h>
-#include <apti18n.h>
-
-// FIXME: we need to find a way to export this 
-class APT_PUBLIC SourceList : public pkgSourceList
-{
-   
- public:
-   // Add custom metaIndex (e.g. local files)
-   void AddMetaIndex(metaIndex *mi) {
-      SrcList.push_back(mi);
-   }
+#include <apt-pkg/cacheset.h>
 
-};
+#include <apti18n.h>
 
 // class CacheFile - Cover class for some dependency cache functions	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 class APT_PUBLIC CacheFile : public pkgCacheFile
 {
-   static pkgCache *SortCache;
-   APT_HIDDEN static int NameComp(const void *a,const void *b) APT_PURE;
-
    public:
-   pkgCache::Package **List;
-   
-   void Sort();
+   std::vector<map_pointer_t> UniverseList;
+
    bool CheckDeps(bool AllowBroken = false);
    bool BuildCaches(bool WithLock = true)
    {
@@ -41,24 +25,10 @@ class APT_PUBLIC CacheFile : public pkgCacheFile
 	 return false;
       return true;
    }
-   // FIXME: this can go once the "libapt-pkg" pkgSourceList has a way
-   //        to add custom metaIndexes (or custom local files or so)
-   bool BuildSourceList(OpProgress */*Progress*/ = NULL) {
-      if (SrcList != NULL)
-         return true;
-      SrcList = new SourceList();
-      if (SrcList->ReadMainList() == false)
-         return _error->Error(_("The list of sources could not be read."));
-      return true;
-   }
-   bool Open(bool WithLock = true) 
+   bool Open(bool WithLock = true)
    {
       OpTextProgress Prog(*_config);
-      if (pkgCacheFile::Open(&Prog,WithLock) == false)
-	 return false;
-      Sort();
-      
-      return true;
+      return pkgCacheFile::Open(&Prog,WithLock);
    };
    bool OpenForInstall()
    {
@@ -67,11 +37,38 @@ class APT_PUBLIC CacheFile : public pkgCacheFile
       else
 	 return Open(true);
    }
-   CacheFile() : List(0) {};
-   ~CacheFile() {
-      delete[] List;
-   }
 };
 									/*}}}*/
 
+class SortedPackageUniverse : public APT::PackageUniverse
+{
+   std::vector<map_pointer_t> &List;
+   void LazyInit() const;
+
+public:
+   SortedPackageUniverse(CacheFile &Cache);
+
+   class const_iterator : public APT::Container_iterator_base<APT::PackageContainerInterface, SortedPackageUniverse, SortedPackageUniverse::const_iterator, std::vector<map_pointer_t>::const_iterator, pkgCache::PkgIterator>
+   {
+      pkgCache * const Cache;
+      public:
+	 inline pkgCache::PkgIterator getType(void) const
+	 {
+	    if (*_iter == 0) return pkgCache::PkgIterator(*Cache);
+	    return pkgCache::PkgIterator(*Cache, Cache->PkgP + *_iter);
+	 }
+	 explicit const_iterator(pkgCache * const Owner, std::vector<map_pointer_t>::const_iterator i):
+	    Container_iterator_base<APT::PackageContainerInterface, SortedPackageUniverse, SortedPackageUniverse::const_iterator, std::vector<map_pointer_t>::const_iterator, pkgCache::PkgIterator>(i), Cache(Owner) {}
+
+   };
+   typedef const_iterator iterator;
+
+   const_iterator begin() const { LazyInit(); return const_iterator(data(), List.begin()); }
+   const_iterator end() const { LazyInit(); return const_iterator(data(), List.end()); }
+   const_iterator cbegin() const { LazyInit(); return const_iterator(data(), List.begin()); }
+   const_iterator cend() const { LazyInit(); return const_iterator(data(), List.end()); }
+   iterator begin() { LazyInit(); return iterator(data(), List.begin()); }
+   iterator end() { LazyInit(); return iterator(data(), List.end()); }
+};
+
 #endif
diff --git a/apt-private/private-cacheset.cc b/apt-private/private-cacheset.cc
index cb68024db..8db736507 100644
--- a/apt-private/private-cacheset.cc
+++ b/apt-private/private-cacheset.cc
@@ -4,9 +4,11 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/depcache.h>
 #include <apt-pkg/cacheiterators.h>
+#include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/progress.h>
 #include <apt-pkg/policy.h>
+#include <apt-pkg/strutl.h>
 
 #include <apt-private/private-cacheset.h>
 
@@ -14,7 +16,7 @@
 
 #include <apti18n.h>
 
-bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,
+bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,		/*{{{*/
                                  APT::VersionContainerInterface * const vci,
                                  OpProgress * const progress)
 {
@@ -22,7 +24,6 @@ bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,
     return GetLocalitySortedVersionSet(CacheFile, vci,
                                        null_matcher, progress);
 }
-
 bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,
                                  APT::VersionContainerInterface * const vci,
                                  Matcher &matcher,
@@ -88,3 +89,207 @@ bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,
       progress->Done();
    return true;
 }
+									/*}}}*/
+
+// CacheSetHelper saving virtual packages				/*{{{*/
+pkgCache::VerIterator CacheSetHelperVirtuals::canNotGetVersion(
+      enum CacheSetHelper::VerSelector const select,
+      pkgCacheFile &Cache,
+      pkgCache::PkgIterator const &Pkg)
+{
+   if (select == NEWEST || select == CANDIDATE || select == ALL)
+      virtualPkgs.insert(Pkg);
+   return CacheSetHelper::canNotGetVersion(select, Cache, Pkg);
+}
+void CacheSetHelperVirtuals::canNotFindVersion(
+      enum CacheSetHelper::VerSelector const select,
+      APT::VersionContainerInterface * vci,
+      pkgCacheFile &Cache,
+      pkgCache::PkgIterator const &Pkg)
+{
+   if (select == NEWEST || select == CANDIDATE || select == ALL)
+      virtualPkgs.insert(Pkg);
+   return CacheSetHelper::canNotFindVersion(select, vci, Cache, Pkg);
+}
+CacheSetHelperVirtuals::CacheSetHelperVirtuals(bool const ShowErrors, GlobalError::MsgType const &ErrorType) :
+   CacheSetHelper{ShowErrors, ErrorType}
+{}
+									/*}}}*/
+
+// CacheSetHelperAPTGet - responsible for message telling from the CacheSets/*{{{*/
+CacheSetHelperAPTGet::CacheSetHelperAPTGet(std::ostream &out) :
+   APT::CacheSetHelper{true}, out(out)
+{
+   explicitlyNamed = true;
+}
+void CacheSetHelperAPTGet::showTaskSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern)
+{
+   ioprintf(out, _("Note, selecting '%s' for task '%s'\n"),
+	 Pkg.FullName(true).c_str(), pattern.c_str());
+   explicitlyNamed = false;
+}
+void CacheSetHelperAPTGet::showFnmatchSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern)
+{
+   ioprintf(out, _("Note, selecting '%s' for glob '%s'\n"),
+	 Pkg.FullName(true).c_str(), pattern.c_str());
+   explicitlyNamed = false;
+}
+void CacheSetHelperAPTGet::showRegExSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern)
+{
+   ioprintf(out, _("Note, selecting '%s' for regex '%s'\n"),
+	 Pkg.FullName(true).c_str(), pattern.c_str());
+   explicitlyNamed = false;
+}
+void CacheSetHelperAPTGet::showSelectedVersion(pkgCache::PkgIterator const &/*Pkg*/, pkgCache::VerIterator const Ver,
+      std::string const &ver, bool const /*verIsRel*/)
+{
+   if (ver == Ver.VerStr())
+      return;
+   selectedByRelease.push_back(make_pair(Ver, ver));
+}
+bool CacheSetHelperAPTGet::showVirtualPackageErrors(pkgCacheFile &Cache)
+{
+   if (virtualPkgs.empty() == true)
+      return true;
+   for (APT::PackageSet::const_iterator Pkg = virtualPkgs.begin();
+	 Pkg != virtualPkgs.end(); ++Pkg) {
+      if (Pkg->ProvidesList != 0) {
+	 ioprintf(c1out,_("Package %s is a virtual package provided by:\n"),
+	       Pkg.FullName(true).c_str());
+
+	 pkgCache::PrvIterator I = Pkg.ProvidesList();
+	 unsigned short provider = 0;
+	 for (; I.end() == false; ++I) {
+	    pkgCache::PkgIterator Pkg = I.OwnerPkg();
+
+	    if (Cache[Pkg].CandidateVerIter(Cache) == I.OwnerVer()) {
+	       c1out << "  " << Pkg.FullName(true) << " " << I.OwnerVer().VerStr();
+	       if (Cache[Pkg].Install() == true && Cache[Pkg].NewInstall() == false)
+		  c1out << _(" [Installed]");
+	       c1out << std::endl;
+	       ++provider;
+	    }
+	 }
+	 // if we found no candidate which provide this package, show non-candidates
+	 if (provider == 0)
+	    for (I = Pkg.ProvidesList(); I.end() == false; ++I)
+	       c1out << "  " << I.OwnerPkg().FullName(true) << " " << I.OwnerVer().VerStr()
+		  << _(" [Not candidate version]") << std::endl;
+	 else
+	    out << _("You should explicitly select one to install.") << std::endl;
+      } else {
+	 ioprintf(c1out,
+	       _("Package %s is not available, but is referred to by another package.\n"
+		  "This may mean that the package is missing, has been obsoleted, or\n"
+		  "is only available from another source\n"),Pkg.FullName(true).c_str());
+
+	 std::string List;
+	 std::string VersionsList;
+	 std::vector<bool> Seen(Cache.GetPkgCache()->Head().PackageCount, false);
+	 APT::PackageList pkglist;
+	 for (pkgCache::DepIterator Dep = Pkg.RevDependsList();
+	       Dep.end() == false; ++Dep) {
+	    if (Dep->Type != pkgCache::Dep::Replaces)
+	       continue;
+	    pkgCache::PkgIterator const DP = Dep.ParentPkg();
+	    if (Seen[DP->ID] == true)
+	       continue;
+	    Seen[DP->ID] = true;
+	    pkglist.insert(DP);
+	 }
+	 ShowList(c1out, _("However the following packages replace it:"), pkglist,
+	       &AlwaysTrue, &PrettyFullName, &EmptyString);
+      }
+      c1out << std::endl;
+   }
+   return false;
+}
+pkgCache::VerIterator CacheSetHelperAPTGet::canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg)
+{
+   APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, CacheSetHelper::CANDIDATE);
+   if (verset.empty() == false)
+      return *(verset.begin());
+   else if (ShowError == true) {
+      _error->Error(_("Package '%s' has no installation candidate"),Pkg.FullName(true).c_str());
+      virtualPkgs.insert(Pkg);
+   }
+   return pkgCache::VerIterator(Cache, 0);
+}
+pkgCache::VerIterator CacheSetHelperAPTGet::canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg)
+{
+   if (Pkg->ProvidesList != 0)
+   {
+      APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, CacheSetHelper::NEWEST);
+      if (verset.empty() == false)
+	 return *(verset.begin());
+      if (ShowError == true)
+	 ioprintf(out, _("Virtual packages like '%s' can't be removed\n"), Pkg.FullName(true).c_str());
+   }
+   else
+   {
+      pkgCache::GrpIterator Grp = Pkg.Group();
+      pkgCache::PkgIterator P = Grp.PackageList();
+      for (; P.end() != true; P = Grp.NextPkg(P))
+      {
+	 if (P == Pkg)
+	    continue;
+	 if (P->CurrentVer != 0) {
+	    // TRANSLATORS: Note, this is not an interactive question
+	    ioprintf(c1out,_("Package '%s' is not installed, so not removed. Did you mean '%s'?\n"),
+		  Pkg.FullName(true).c_str(), P.FullName(true).c_str());
+	    break;
+	 }
+      }
+      if (P.end() == true)
+	 ioprintf(c1out,_("Package '%s' is not installed, so not removed\n"),Pkg.FullName(true).c_str());
+   }
+   return pkgCache::VerIterator(Cache, 0);
+}
+APT::VersionSet CacheSetHelperAPTGet::tryVirtualPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg,
+      CacheSetHelper::VerSelector const select)
+{
+   /* This is a pure virtual package and there is a single available
+      candidate providing it. */
+   if (unlikely(Cache[Pkg].CandidateVer != 0) || Pkg->ProvidesList == 0)
+      return APT::VersionSet();
+
+   pkgCache::PkgIterator Prov;
+   bool found_one = false;
+   for (pkgCache::PrvIterator P = Pkg.ProvidesList(); P; ++P) {
+      pkgCache::VerIterator const PVer = P.OwnerVer();
+      pkgCache::PkgIterator const PPkg = PVer.ParentPkg();
+
+      /* Ignore versions that are not a candidate. */
+      if (Cache[PPkg].CandidateVer != PVer)
+	 continue;
+
+      if (found_one == false) {
+	 Prov = PPkg;
+	 found_one = true;
+      } else if (PPkg != Prov) {
+	 // same group, so it's a foreign package
+	 if (PPkg->Group == Prov->Group) {
+	    // do we already have the requested arch?
+	    if (strcmp(Pkg.Arch(), Prov.Arch()) == 0 ||
+		  strcmp(Prov.Arch(), "all") == 0 ||
+		  unlikely(strcmp(PPkg.Arch(), Prov.Arch()) == 0)) // packages have only on candidate, but just to be sure
+	       continue;
+	    // see which architecture we prefer more and switch to it
+	    std::vector<std::string> archs = APT::Configuration::getArchitectures();
+	    if (std::find(archs.begin(), archs.end(), PPkg.Arch()) < std::find(archs.begin(), archs.end(), Prov.Arch()))
+	       Prov = PPkg;
+	    continue;
+	 }
+	 found_one = false; // we found at least two
+	 break;
+      }
+   }
+
+   if (found_one == true) {
+      ioprintf(out, _("Note, selecting '%s' instead of '%s'\n"),
+	    Prov.FullName(true).c_str(), Pkg.FullName(true).c_str());
+      return APT::VersionSet::FromPackage(Cache, Prov, select, *this);
+   }
+   return APT::VersionSet();
+}
+									/*}}}*/
diff --git a/apt-private/private-cacheset.h b/apt-private/private-cacheset.h
index 059c7637e..892993e58 100644
--- a/apt-private/private-cacheset.h
+++ b/apt-private/private-cacheset.h
@@ -1,55 +1,48 @@
 #ifndef APT_PRIVATE_CACHESET_H
 #define APT_PRIVATE_CACHESET_H
 
-#include <apt-pkg/aptconfiguration.h>
-#include <apt-pkg/cachefile.h>
 #include <apt-pkg/cacheset.h>
-#include <apt-pkg/sptr.h>
-#include <apt-pkg/strutl.h>
-#include <apt-pkg/depcache.h>
-#include <apt-pkg/error.h>
-#include <apt-pkg/pkgcache.h>
-#include <apt-pkg/cacheiterators.h>
 #include <apt-pkg/macros.h>
 
-#include <algorithm>
+#include <apt-private/private-output.h>
+
 #include <vector>
-#include <string.h>
 #include <list>
-#include <ostream>
 #include <set>
 #include <string>
-#include <utility>
-
-#include "private-output.h"
 
 #include <apti18n.h>
 
 class OpProgress;
 
-struct VersionSortDescriptionLocality
+struct APT_PUBLIC VersionSortDescriptionLocality			/*{{{*/
 {
-   bool operator () (const pkgCache::VerIterator &v_lhs, 
-                     const pkgCache::VerIterator &v_rhs)
-    {
-        pkgCache::DescFile *A = v_lhs.TranslatedDescription().FileList();
-        pkgCache::DescFile *B = v_rhs.TranslatedDescription().FileList();
-        if (A == 0 && B == 0)
-           return false;
+   bool operator () (const pkgCache::VerIterator &v_lhs,
+	 const pkgCache::VerIterator &v_rhs)
+   {
+      pkgCache::DescFile const *A = nullptr;
+      pkgCache::DescFile const *B = nullptr;
+      if (v_lhs->DescriptionList != 0)
+	 A = v_lhs.TranslatedDescription().FileList();
+      if (v_rhs->DescriptionList != 0)
+	 B = v_rhs.TranslatedDescription().FileList();
 
-       if (A == 0)
-          return true;
+      if (A == nullptr && B == nullptr)
+	 return false;
 
-       if (B == 0)
-          return false;
+      if (A == nullptr)
+	 return true;
 
-       if (A->File == B->File)
-          return A->Offset < B->Offset;
+      if (B == nullptr)
+	 return false;
 
-       return A->File < B->File;
-    }
-};
+      if (A->File == B->File)
+	 return A->Offset < B->Offset;
 
+      return A->File < B->File;
+   }
+};
+									/*}}}*/
 // sorted by locality which makes iterating much faster
 typedef APT::VersionContainer<
    std::set<pkgCache::VerIterator,
@@ -72,28 +65,19 @@ bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,
 
 
 // CacheSetHelper saving virtual packages				/*{{{*/
-class CacheSetHelperVirtuals: public APT::CacheSetHelper {
+class APT_PUBLIC CacheSetHelperVirtuals: public APT::CacheSetHelper {
 public:
    APT::PackageSet virtualPkgs;
 
-   virtual pkgCache::VerIterator canNotGetVersion(enum CacheSetHelper::VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) {
-      if (select == NEWEST || select == CANDIDATE || select == ALL)
-	 virtualPkgs.insert(Pkg);
-      return CacheSetHelper::canNotGetVersion(select, Cache, Pkg);
-   }
+   virtual pkgCache::VerIterator canNotGetVersion(enum CacheSetHelper::VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) APT_OVERRIDE;
+   virtual void canNotFindVersion(enum CacheSetHelper::VerSelector const select, APT::VersionContainerInterface * vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) APT_OVERRIDE;
 
-   virtual void canNotFindVersion(enum CacheSetHelper::VerSelector const select, APT::VersionContainerInterface * vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) {
-      if (select == NEWEST || select == CANDIDATE || select == ALL)
-	 virtualPkgs.insert(Pkg);
-      return CacheSetHelper::canNotFindVersion(select, vci, Cache, Pkg);
-   }
-
-   CacheSetHelperVirtuals(bool const ShowErrors = true, GlobalError::MsgType const &ErrorType = GlobalError::NOTICE) : CacheSetHelper(ShowErrors, ErrorType) {}
+   CacheSetHelperVirtuals(bool const ShowErrors = true, GlobalError::MsgType const &ErrorType = GlobalError::NOTICE);
 };
 									/*}}}*/
 
 // CacheSetHelperAPTGet - responsible for message telling from the CacheSets/*{{{*/
-class CacheSetHelperAPTGet : public APT::CacheSetHelper {
+class APT_PUBLIC CacheSetHelperAPTGet : public APT::CacheSetHelper {
 	/** \brief stream message should be printed to */
 	std::ostream &out;
 	/** \brief were things like Task or RegEx used to select packages? */
@@ -104,178 +88,22 @@ class CacheSetHelperAPTGet : public APT::CacheSetHelper {
 public:
 	std::list<std::pair<pkgCache::VerIterator, std::string> > selectedByRelease;
 
-	CacheSetHelperAPTGet(std::ostream &out) : APT::CacheSetHelper(true), out(out) {
-		explicitlyNamed = true;
-	}
+	CacheSetHelperAPTGet(std::ostream &out);
 
-	virtual void showTaskSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern) {
-		ioprintf(out, _("Note, selecting '%s' for task '%s'\n"),
-				Pkg.FullName(true).c_str(), pattern.c_str());
-		explicitlyNamed = false;
-	}
-        virtual void showFnmatchSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern) {
-		ioprintf(out, _("Note, selecting '%s' for glob '%s'\n"),
-				Pkg.FullName(true).c_str(), pattern.c_str());
-		explicitlyNamed = false;
-	}
-	virtual void showRegExSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern) {
-		ioprintf(out, _("Note, selecting '%s' for regex '%s'\n"),
-				Pkg.FullName(true).c_str(), pattern.c_str());
-		explicitlyNamed = false;
-	}
+	virtual void showTaskSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern) APT_OVERRIDE;
+        virtual void showFnmatchSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern) APT_OVERRIDE;
+	virtual void showRegExSelection(pkgCache::PkgIterator const &Pkg, std::string const &pattern) APT_OVERRIDE;
 	virtual void showSelectedVersion(pkgCache::PkgIterator const &/*Pkg*/, pkgCache::VerIterator const Ver,
-				 std::string const &ver, bool const /*verIsRel*/) {
-		if (ver == Ver.VerStr())
-			return;
-		selectedByRelease.push_back(make_pair(Ver, ver));
-	}
-
-	bool showVirtualPackageErrors(pkgCacheFile &Cache) {
-		if (virtualPkgs.empty() == true)
-			return true;
-		for (APT::PackageSet::const_iterator Pkg = virtualPkgs.begin();
-		     Pkg != virtualPkgs.end(); ++Pkg) {
-			if (Pkg->ProvidesList != 0) {
-				ioprintf(c1out,_("Package %s is a virtual package provided by:\n"),
-					 Pkg.FullName(true).c_str());
+				 std::string const &ver, bool const /*verIsRel*/) APT_OVERRIDE;
+	bool showVirtualPackageErrors(pkgCacheFile &Cache);
 
-				pkgCache::PrvIterator I = Pkg.ProvidesList();
-				unsigned short provider = 0;
-				for (; I.end() == false; ++I) {
-					pkgCache::PkgIterator Pkg = I.OwnerPkg();
-
-					if (Cache[Pkg].CandidateVerIter(Cache) == I.OwnerVer()) {
-						c1out << "  " << Pkg.FullName(true) << " " << I.OwnerVer().VerStr();
-						if (Cache[Pkg].Install() == true && Cache[Pkg].NewInstall() == false)
-							c1out << _(" [Installed]");
-						c1out << std::endl;
-						++provider;
-					}
-				}
-				// if we found no candidate which provide this package, show non-candidates
-				if (provider == 0)
-					for (I = Pkg.ProvidesList(); I.end() == false; ++I)
-						c1out << "  " << I.OwnerPkg().FullName(true) << " " << I.OwnerVer().VerStr()
-                                                      << _(" [Not candidate version]") << std::endl;
-				else
-                                   out << _("You should explicitly select one to install.") << std::endl;
-			} else {
-				ioprintf(c1out,
-					_("Package %s is not available, but is referred to by another package.\n"
-					  "This may mean that the package is missing, has been obsoleted, or\n"
-					  "is only available from another source\n"),Pkg.FullName(true).c_str());
-
-				std::string List;
-				std::string VersionsList;
-				SPtrArray<bool> Seen = new bool[Cache.GetPkgCache()->Head().PackageCount];
-				memset(Seen,0,Cache.GetPkgCache()->Head().PackageCount*sizeof(*Seen));
-				for (pkgCache::DepIterator Dep = Pkg.RevDependsList();
-				     Dep.end() == false; ++Dep) {
-					if (Dep->Type != pkgCache::Dep::Replaces)
-						continue;
-					if (Seen[Dep.ParentPkg()->ID] == true)
-						continue;
-					Seen[Dep.ParentPkg()->ID] = true;
-					List += Dep.ParentPkg().FullName(true) + " ";
-					//VersionsList += std::string(Dep.ParentPkg().CurVersion) + "\n"; ???
-				}
-				ShowList(c1out,_("However the following packages replace it:"),List,VersionsList);
-			}
-			c1out << std::endl;
-		}
-		return false;
-	}
-
-	virtual pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) {
-		APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, CacheSetHelper::CANDIDATE);
-		if (verset.empty() == false)
-			return *(verset.begin());
-		else if (ShowError == true) {
-			_error->Error(_("Package '%s' has no installation candidate"),Pkg.FullName(true).c_str());
-			virtualPkgs.insert(Pkg);
-		}
-		return pkgCache::VerIterator(Cache, 0);
-	}
-
-	virtual pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) {
-		if (Pkg->ProvidesList != 0)
-		{
-			APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, CacheSetHelper::NEWEST);
-			if (verset.empty() == false)
-				return *(verset.begin());
-			if (ShowError == true)
-				ioprintf(out, _("Virtual packages like '%s' can't be removed\n"), Pkg.FullName(true).c_str());
-		}
-		else
-		{
-			pkgCache::GrpIterator Grp = Pkg.Group();
-			pkgCache::PkgIterator P = Grp.PackageList();
-			for (; P.end() != true; P = Grp.NextPkg(P))
-			{
-				if (P == Pkg)
-					continue;
-				if (P->CurrentVer != 0) {
-					// TRANSLATORS: Note, this is not an interactive question
-					ioprintf(c1out,_("Package '%s' is not installed, so not removed. Did you mean '%s'?\n"),
-						 Pkg.FullName(true).c_str(), P.FullName(true).c_str());
-					break;
-				}
-			}
-			if (P.end() == true)
-				ioprintf(c1out,_("Package '%s' is not installed, so not removed\n"),Pkg.FullName(true).c_str());
-		}
-		return pkgCache::VerIterator(Cache, 0);
-	}
+	virtual pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) APT_OVERRIDE;
+	virtual pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) APT_OVERRIDE;
 
 	APT::VersionSet tryVirtualPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg,
-						CacheSetHelper::VerSelector const select) {
-		/* This is a pure virtual package and there is a single available
-		   candidate providing it. */
-		if (unlikely(Cache[Pkg].CandidateVer != 0) || Pkg->ProvidesList == 0)
-			return APT::VersionSet();
-
-		pkgCache::PkgIterator Prov;
-		bool found_one = false;
-		for (pkgCache::PrvIterator P = Pkg.ProvidesList(); P; ++P) {
-			pkgCache::VerIterator const PVer = P.OwnerVer();
-			pkgCache::PkgIterator const PPkg = PVer.ParentPkg();
-
-			/* Ignore versions that are not a candidate. */
-			if (Cache[PPkg].CandidateVer != PVer)
-				continue;
-
-			if (found_one == false) {
-				Prov = PPkg;
-				found_one = true;
-			} else if (PPkg != Prov) {
-				// same group, so it's a foreign package
-				if (PPkg->Group == Prov->Group) {
-					// do we already have the requested arch?
-					if (strcmp(Pkg.Arch(), Prov.Arch()) == 0 ||
-					    strcmp(Prov.Arch(), "all") == 0 ||
-					    unlikely(strcmp(PPkg.Arch(), Prov.Arch()) == 0)) // packages have only on candidate, but just to be sure
-						continue;
-					// see which architecture we prefer more and switch to it
-					std::vector<std::string> archs = APT::Configuration::getArchitectures();
-					if (std::find(archs.begin(), archs.end(), PPkg.Arch()) < std::find(archs.begin(), archs.end(), Prov.Arch()))
-						Prov = PPkg;
-					continue;
-				}
-				found_one = false; // we found at least two
-				break;
-			}
-		}
-
-		if (found_one == true) {
-			ioprintf(out, _("Note, selecting '%s' instead of '%s'\n"),
-				 Prov.FullName(true).c_str(), Pkg.FullName(true).c_str());
-			return APT::VersionSet::FromPackage(Cache, Prov, select, *this);
-		}
-		return APT::VersionSet();
-	}
+						CacheSetHelper::VerSelector const select);
 
 	inline bool allPkgNamedExplicitly() const { return explicitlyNamed; }
-
 };
 									/*}}}*/
 
diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 41aab81f6..3a1564b23 100644
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -51,6 +51,7 @@ static bool addArgumentsAPTCache(std::vector<CommandLine::Args> &Args, char cons
       addArg(0, "conflicts", "APT::Cache::ShowConflicts", 0);
       addArg(0, "enhances", "APT::Cache::ShowEnhances", 0);
       addArg(0, "recurse", "APT::Cache::RecurseDepends", 0);
+      addArg(0, "implicit", "APT::Cache::ShowImplicit", 0);
    }
    else if (CmdMatches("search"))
    {
@@ -163,14 +164,19 @@ static bool addArgumentsAPTGet(std::vector<CommandLine::Args> &Args, char const
       // once sbuild is fixed, this option can be removed
       addArg('f', "fix-broken", "APT::Get::Fix-Broken", 0);
    }
-   else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
+   else if (CmdMatches("indextargets"))
+   {
+      addArg(0,"format","APT::Get::IndexTargets::Format", CommandLine::HasArg);
+      addArg(0,"release-info","APT::Get::IndexTargets::ReleaseInfo", 0);
+   }
+   else if (CmdMatches("clean", "autoclean", "auto-clean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands
       ;
    else if (CmdMatches("moo"))
       addArg(0, "color", "APT::Moo::Color", 0);
 
    if (CmdMatches("install", "remove", "purge", "upgrade", "dist-upgrade",
-	    "dselect-upgrade", "autoremove", "clean", "autoclean", "check",
+	    "dselect-upgrade", "autoremove", "auto-remove", "clean", "autoclean", "auto-clean", "check",
 	    "build-dep", "full-upgrade", "source"))
    {
       addArg('s', "simulate", "APT::Get::Simulate", 0);
@@ -196,6 +202,9 @@ static bool addArgumentsAPTGet(std::vector<CommandLine::Args> &Args, char const
    addArg(0,"ignore-hold","APT::Ignore-Hold",0);
    addArg(0,"upgrade","APT::Get::upgrade",0);
    addArg(0,"only-upgrade","APT::Get::Only-Upgrade",0);
+   addArg(0,"allow-change-held-packages","APT::Get::allow-change-held-packages",CommandLine::Boolean);
+   addArg(0,"allow-remove-essential","APT::Get::allow-remove-essential",CommandLine::Boolean);
+   addArg(0,"allow-downgrades","APT::Get::allow-downgrades",CommandLine::Boolean);
    addArg(0,"force-yes","APT::Get::force-yes",0);
    addArg(0,"print-uris","APT::Get::Print-URIs",0);
    addArg(0,"trivial-only","APT::Get::Trivial-Only",0);
@@ -236,6 +245,7 @@ static bool addArgumentsAPT(std::vector<CommandLine::Args> &Args, char const * c
    if (CmdMatches("list"))
    {
       addArg(0,"installed","APT::Cmd::Installed",0);
+      addArg(0,"upgradeable","APT::Cmd::Upgradable",0);
       addArg(0,"upgradable","APT::Cmd::Upgradable",0);
       addArg(0,"manual-installed","APT::Cmd::Manual-Installed",0);
       addArg('v', "verbose", "APT::Cmd::List-Include-Summary", 0);
diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc
index 37fae18e9..18a9b1fbc 100644
--- a/apt-private/private-download.cc
+++ b/apt-private/private-download.cc
@@ -78,20 +78,23 @@ bool CheckDropPrivsMustBeDisabled(pkgAcquire &Fetcher)			/*{{{*/
 // CheckAuth - check if each download comes form a trusted source	/*{{{*/
 bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser)
 {
-   std::string UntrustedList;
+   std::vector<std::string> UntrustedList;
    for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); I < Fetcher.ItemsEnd(); ++I)
       if (!(*I)->IsTrusted())
-          UntrustedList += std::string((*I)->ShortDesc()) + " ";
+	 UntrustedList.push_back((*I)->ShortDesc());
 
-   if (UntrustedList == "")
+   if (UntrustedList.empty())
       return true;
 
    return AuthPrompt(UntrustedList, PromptUser);
 }
 
-bool AuthPrompt(std::string const &UntrustedList, bool const PromptUser)
+bool AuthPrompt(std::vector<std::string> const &UntrustedList, bool const PromptUser)
 {
-   ShowList(c2out,_("WARNING: The following packages cannot be authenticated!"),UntrustedList,"");
+   ShowList(c2out,_("WARNING: The following packages cannot be authenticated!"), UntrustedList,
+	 [](std::string const&) { return true; },
+	 [](std::string const&str) { return str; },
+	 [](std::string const&) { return ""; });
 
    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
    {
@@ -111,10 +114,12 @@ bool AuthPrompt(std::string const &UntrustedList, bool const PromptUser)
 
       return true;
    }
-   else if (_config->FindB("APT::Get::Force-Yes",false) == true)
+   else if (_config->FindB("APT::Get::Force-Yes",false) == true) {
+      _error->Warning(_("--force-yes is deprecated, use one of the options starting with --allow instead."));
       return true;
+   }
 
-   return _error->Error(_("There are problems and -y was used without --force-yes"));
+   return _error->Error(_("There were unauthenticated packages and -y was used without --allow-unauthenticated"));
 }
 									/*}}}*/
 bool AcquireRun(pkgAcquire &Fetcher, int const PulseInterval, bool * const Failure, bool * const TransientNetworkFailure)/*{{{*/
diff --git a/apt-private/private-download.h b/apt-private/private-download.h
index 0a0ac6b95..0f3db5e7a 100644
--- a/apt-private/private-download.h
+++ b/apt-private/private-download.h
@@ -4,6 +4,7 @@
 #include <apt-pkg/macros.h>
 
 #include <string>
+#include <vector>
 
 class pkgAcquire;
 
@@ -14,7 +15,7 @@ APT_PUBLIC bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser);
 
 // show a authentication warning prompt and return true if the system
 // should continue
-APT_PUBLIC bool AuthPrompt(std::string const &UntrustedList, bool const PromptUser);
+APT_PUBLIC bool AuthPrompt(std::vector<std::string> const &UntrustedList, bool const PromptUser);
 
 APT_PUBLIC bool AcquireRun(pkgAcquire &Fetcher, int const PulseInterval, bool * const Failure, bool * const TransientNetworkFailure);
 
diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc
index acc6d42c2..844fcbc7e 100644
--- a/apt-private/private-install.cc
+++ b/apt-private/private-install.cc
@@ -21,6 +21,7 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/upgrade.h>
 #include <apt-pkg/install-progress.h>
+#include <apt-pkg/debindexfile.h>
 
 #include <stdlib.h>
 #include <string.h>
@@ -57,7 +58,8 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
       }
    }
    
-   bool Fail = false;
+   bool Hold = false;
+   bool Downgrade = false;
    bool Essential = false;
    
    // Show all the various warning indicators
@@ -65,13 +67,17 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
    ShowNew(c1out,Cache);
    if (ShwKept == true)
       ShowKept(c1out,Cache);
-   Fail |= !ShowHold(c1out,Cache);
+   Hold = !ShowHold(c1out,Cache);
    if (_config->FindB("APT::Get::Show-Upgraded",true) == true)
       ShowUpgraded(c1out,Cache);
-   Fail |= !ShowDowngraded(c1out,Cache);
+   Downgrade = !ShowDowngraded(c1out,Cache);
+
    if (_config->FindB("APT::Get::Download-Only",false) == false)
         Essential = !ShowEssential(c1out,Cache);
-   Fail |= Essential;
+
+   // All kinds of failures
+   bool Fail = (Essential || Downgrade || Hold);
+
    Stats(c1out,Cache);
 
    // Sanity check
@@ -88,20 +94,33 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
    // No remove flag
    if (Cache->DelCount() != 0 && _config->FindB("APT::Get::Remove",true) == false)
       return _error->Error(_("Packages need to be removed but remove is disabled."));
-       
+
+   // Fail safe check
+   if (_config->FindI("quiet",0) >= 2 ||
+       _config->FindB("APT::Get::Assume-Yes",false) == true)
+   {
+      if (_config->FindB("APT::Get::Force-Yes",false) == true) {
+	 _error->Warning(_("--force-yes is deprecated, use one of the options starting with --allow instead."));
+      }
+
+      if (Fail == true && _config->FindB("APT::Get::Force-Yes",false) == false) {
+	 if (Essential == true && _config->FindB("APT::Get::allow-remove-essential", false) == false)
+	    return _error->Error(_("Essential packages were removed and -y was used without --allow-remove-essential."));
+	 if (Downgrade == true && _config->FindB("APT::Get::allow-downgrades", false) == false)
+	    return _error->Error(_("Packages were downgraded and -y was used without --allow-downgrades."));
+	 if (Hold == true && _config->FindB("APT::Get::allow-change-held-packages", false) == false)
+	    return _error->Error(_("Held packages were changed and -y was used without --allow-change-held-packages."));
+      }
+   }
+
    // Run the simulator ..
    if (_config->FindB("APT::Get::Simulate") == true)
    {
       pkgSimulate PM(Cache);
 
-#if APT_PKG_ABI >= 413
       APT::Progress::PackageManager *progress = APT::Progress::PackageManagerProgressFactory();
       pkgPackageManager::OrderResult Res = PM.DoInstall(progress);
       delete progress;
-#else
-      int status_fd = _config->FindI("APT::Status-Fd",-1);
-      pkgPackageManager::OrderResult Res = PM.DoInstall(status_fd);
-#endif
 
       if (Res == pkgPackageManager::Failed)
 	 return false;
@@ -132,7 +151,7 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
    pkgSourceList *List = Cache.GetSourceList();
    
    // Create the package manager and prepare to download
-   SPtr<pkgPackageManager> PM= _system->CreatePM(Cache);
+   std::unique_ptr<pkgPackageManager> PM(_system->CreatePM(Cache));
    if (PM->GetArchives(&Fetcher,List,&Recs) == false || 
        _error->PendingError() == true)
       return false;
@@ -177,15 +196,7 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
    if (CheckFreeSpaceBeforeDownload(_config->FindDir("Dir::Cache::Archives"), (FetchBytes - FetchPBytes)) == false)
       return false;
 
-   // Fail safe check
-   if (_config->FindI("quiet",0) >= 2 ||
-       _config->FindB("APT::Get::Assume-Yes",false) == true)
-   {
-      if (Fail == true && _config->FindB("APT::Get::Force-Yes",false) == false)
-	 return _error->Error(_("There are problems and -y was used without --force-yes"));
-   }         
-
-   if (Essential == true && Safety == true)
+   if (Essential == true && Safety == true && _config->FindB("APT::Get::allow-remove-essential", false) == false)
    {
       if (_config->FindB("APT::Get::Trivial-Only",false) == true)
 	 return _error->Error(_("Trivial Only specified but this is not a trivial operation."));
@@ -307,14 +318,9 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
 
       _system->UnLock();
 
-#if APT_PKG_ABI >= 413
       APT::Progress::PackageManager *progress = APT::Progress::PackageManagerProgressFactory();
       pkgPackageManager::OrderResult Res = PM->DoInstall(progress);
       delete progress;
-#else
-      int status_fd = _config->FindI("APT::Status-Fd", -1);
-      pkgPackageManager::OrderResult Res = PM->DoInstall(status_fd);
-#endif
 
       if (Res == pkgPackageManager::Failed || _error->PendingError() == true)
 	 return false;
@@ -330,19 +336,17 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
    }
 
    std::set<std::string> const disappearedPkgs = PM->GetDisappearedPackages();
-   if (disappearedPkgs.empty() == true)
-      return true;
-
-   std::string disappear;
-   for (std::set<std::string>::const_iterator d = disappearedPkgs.begin();
-	d != disappearedPkgs.end(); ++d)
-      disappear.append(*d).append(" ");
-
-   ShowList(c1out, P_("The following package disappeared from your system as\n"
-	"all files have been overwritten by other packages:",
-	"The following packages disappeared from your system as\n"
-	"all files have been overwritten by other packages:", disappearedPkgs.size()), disappear, "");
-   c0out << _("Note: This is done automatically and on purpose by dpkg.") << std::endl;
+   if (disappearedPkgs.empty() == false)
+   {
+      ShowList(c1out, P_("The following package disappeared from your system as\n"
+	       "all files have been overwritten by other packages:",
+	       "The following packages disappeared from your system as\n"
+	       "all files have been overwritten by other packages:", disappearedPkgs.size()), disappearedPkgs,
+	    [](std::string const &Pkg) { return Pkg.empty() == false; },
+	    [](std::string const &Pkg) { return Pkg; },
+	    [](std::string const &) { return std::string(); });
+      c0out << _("Note: This is done automatically and on purpose by dpkg.") << std::endl;
+   }
 
    return true;
 }
@@ -374,11 +378,10 @@ static bool DoAutomaticRemove(CacheFile &Cache)
 
    unsigned long autoRemoveCount = 0;
    APT::PackageSet tooMuch;
-   APT::PackageList autoRemoveList;
+   SortedPackageUniverse Universe(Cache);
    // look over the cache to see what can be removed
-   for (unsigned J = 0; J < Cache->Head().PackageCount; ++J)
+   for (auto const &Pkg: Universe)
    {
-      pkgCache::PkgIterator Pkg(Cache,Cache.List[J]);
       if (Cache[Pkg].Garbage)
       {
 	 if(Pkg.CurrentVer() != 0 || Cache[Pkg].Install())
@@ -395,8 +398,6 @@ static bool DoAutomaticRemove(CacheFile &Cache)
 	 }
 	 else
 	 {
-	    if (hideAutoRemove == false && Cache[Pkg].Delete() == false)
-	       autoRemoveList.insert(Pkg);
 	    // if the package is a new install and already garbage we don't need to
 	    // install it in the first place, so nuke it instead of show it
 	    if (Cache[Pkg].Install() == true && Pkg.CurrentVer() == 0)
@@ -419,7 +420,7 @@ static bool DoAutomaticRemove(CacheFile &Cache)
       bool Changed;
       do {
 	 Changed = false;
-	 for (APT::PackageSet::const_iterator Pkg = tooMuch.begin();
+	 for (APT::PackageSet::iterator Pkg = tooMuch.begin();
 	      Pkg != tooMuch.end(); ++Pkg)
 	 {
 	    APT::PackageSet too;
@@ -456,18 +457,6 @@ static bool DoAutomaticRemove(CacheFile &Cache)
       } while (Changed == true);
    }
 
-   std::string autoremovelist, autoremoveversions;
-   if (smallList == false && autoRemoveCount != 0)
-   {
-      for (APT::PackageList::const_iterator Pkg = autoRemoveList.begin(); Pkg != autoRemoveList.end(); ++Pkg)
-      {
-	 if (Cache[Pkg].Garbage == false)
-	    continue;
-	 autoremovelist += Pkg.FullName(true) + " ";
-	 autoremoveversions += std::string(Cache[Pkg].CandVersion) + "\n";
-      }
-   }
-
    // Now see if we had destroyed anything (if we had done anything)
    if (Cache->BrokenCount() != 0)
    {
@@ -482,12 +471,17 @@ static bool DoAutomaticRemove(CacheFile &Cache)
    }
 
    // if we don't remove them, we should show them!
-   if (doAutoRemove == false && (autoremovelist.empty() == false || autoRemoveCount != 0))
+   if (doAutoRemove == false && autoRemoveCount != 0)
    {
       if (smallList == false)
+      {
+	 SortedPackageUniverse Universe(Cache);
 	 ShowList(c1out, P_("The following package was automatically installed and is no longer required:",
 	          "The following packages were automatically installed and are no longer required:",
-	          autoRemoveCount), autoremovelist, autoremoveversions);
+	          autoRemoveCount), Universe,
+	       [&Cache](pkgCache::PkgIterator const &Pkg) { return (*Cache)[Pkg].Garbage == true && (*Cache)[Pkg].Delete() == false; },
+	       &PrettyFullName, CandidateVersion(&Cache));
+      }
       else
 	 ioprintf(c1out, P_("%lu package was automatically installed and is no longer required.\n",
 	          "%lu packages were automatically installed and are no longer required.\n", autoRemoveCount), autoRemoveCount);
@@ -513,9 +507,9 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache,
    if (Cache->BrokenCount() != 0)
       BrokenFix = true;
 
-   SPtr<pkgProblemResolver> Fix;
+   std::unique_ptr<pkgProblemResolver> Fix(nullptr);
    if (_config->FindB("APT::Get::CallResolver", true) == true)
-      Fix = new pkgProblemResolver(Cache);
+      Fix.reset(new pkgProblemResolver(Cache));
 
    unsigned short fallback = MOD_INSTALL;
    if (strcasecmp(CmdL.FileList[0],"remove") == 0)
@@ -525,7 +519,8 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache,
       _config->Set("APT::Get::Purge", true);
       fallback = MOD_REMOVE;
    }
-   else if (strcasecmp(CmdL.FileList[0], "autoremove") == 0)
+   else if (strcasecmp(CmdL.FileList[0], "autoremove") == 0 ||
+	    strcasecmp(CmdL.FileList[0], "auto-remove") == 0)
    {
       _config->Set("APT::Get::AutomaticRemove", "true");
       fallback = MOD_REMOVE;
@@ -547,8 +542,8 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache,
    }
 
 
-  TryToInstall InstallAction(Cache, Fix, BrokenFix);
-  TryToRemove RemoveAction(Cache, Fix);
+  TryToInstall InstallAction(Cache, Fix.get(), BrokenFix);
+  TryToRemove RemoveAction(Cache, Fix.get());
 
    // new scope for the ActionGroup
    {
@@ -651,30 +646,26 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache,
 // DoInstall - Install packages from the command line			/*{{{*/
 // ---------------------------------------------------------------------
 /* Install named packages */
+struct PkgIsExtraInstalled {
+   pkgCacheFile * const Cache;
+   APT::VersionSet const * const verset;
+   PkgIsExtraInstalled(pkgCacheFile * const Cache, APT::VersionSet const * const Container) : Cache(Cache), verset(Container) {}
+   bool operator() (pkgCache::PkgIterator const Pkg)
+   {
+        if ((*Cache)[Pkg].Install() == false)
+           return false;
+        pkgCache::VerIterator const Cand = (*Cache)[Pkg].CandidateVerIter(*Cache);
+        return verset->find(Cand) == verset->end();
+   }
+};
 bool DoInstall(CommandLine &CmdL)
 {
    CacheFile Cache;
    // first check for local pkgs and add them to the cache
    for (const char **I = CmdL.FileList; *I != 0; I++)
    {
-      if(FileExists(*I))
-      {
-         // FIXME: make this more elegant
-         std::string TypeStr = flExtension(*I) + "-file";
-         pkgSourceList::Type *Type = pkgSourceList::Type::GetType(TypeStr.c_str());
-         if(Type != 0)
-         {
-            std::vector<metaIndex *> List;
-            std::map<std::string, std::string> Options;
-            if(Type->CreateItem(List, *I, "", "", Options))
-            {
-               // we have our own CacheFile that gives us a SourceList
-               // with superpowerz
-               SourceList *sources = (SourceList*)Cache.GetSourceList();
-               sources->AddMetaIndex(List[0]);
-            }
-         }
-      }
+      if(FileExists(*I) && flExtension(*I) == "deb")
+	 Cache.GetSourceList()->AddVolatileFile(new debDebPkgFileIndex(*I));
    }
 
    // then open the cache
@@ -689,35 +680,17 @@ bool DoInstall(CommandLine &CmdL)
 
    /* Print out a list of packages that are going to be installed extra
       to what the user asked */
+   SortedPackageUniverse Universe(Cache);
    if (Cache->InstCount() != verset[MOD_INSTALL].size())
-   {
-      std::string List;
-      std::string VersionsList;
-      for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-      {
-	 pkgCache::PkgIterator I(Cache,Cache.List[J]);
-	 if ((*Cache)[I].Install() == false)
-	    continue;
-	 pkgCache::VerIterator Cand = Cache[I].CandidateVerIter(Cache);
-
-	 if (verset[MOD_INSTALL].find(Cand) != verset[MOD_INSTALL].end())
-	    continue;
-
-	 List += I.FullName(true) + " ";
-	 VersionsList += std::string(Cache[I].CandVersion) + "\n";
-      }
-      
-      ShowList(c1out,_("The following extra packages will be installed:"),List,VersionsList);
-   }
+      ShowList(c1out, _("The following additional packages will be installed:"), Universe,
+	    PkgIsExtraInstalled(&Cache, &verset[MOD_INSTALL]),
+	    &PrettyFullName, CandidateVersion(&Cache));
 
    /* Print out a list of suggested and recommended packages */
    {
-      std::string SuggestsList, RecommendsList;
-      std::string SuggestsVersions, RecommendsVersions;
-      for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
+      std::list<std::string> Recommends, Suggests, SingleRecommends, SingleSuggests;
+      for (auto const &Pkg: Universe)
       {
-	 pkgCache::PkgIterator Pkg(Cache,Cache.List[J]);
-
 	 /* Just look at the ones we want to install */
 	 if ((*Cache)[Pkg].Install() == false)
 	   continue;
@@ -729,77 +702,79 @@ bool DoInstall(CommandLine &CmdL)
 	    pkgCache::DepIterator Start;
 	    pkgCache::DepIterator End;
 	    D.GlobOr(Start,End); // advances D
+	    if (Start->Type != pkgCache::Dep::Recommends && Start->Type != pkgCache::Dep::Suggests)
+	       continue;
 
-	    // FIXME: we really should display a or-group as a or-group to the user
-	    //        the problem is that ShowList is incapable of doing this
-            std::string RecommendsOrList,RecommendsOrVersions;
-            std::string SuggestsOrList,SuggestsOrVersions;
-	    bool foundInstalledInOrGroup = false;
-	    for(;;)
 	    {
-	       /* Skip if package is  installed already, or is about to be */
-               std::string target = Start.TargetPkg().FullName(true) + " ";
-	       pkgCache::PkgIterator const TarPkg = Start.TargetPkg();
-	       if (TarPkg->SelectedState == pkgCache::State::Install ||
-		   TarPkg->SelectedState == pkgCache::State::Hold ||
-		   Cache[Start.TargetPkg()].Install())
-	       {
-		  foundInstalledInOrGroup=true;
-		  break;
-	       }
-
-	       /* Skip if we already saw it */
-	       if (int(SuggestsList.find(target)) != -1 || int(RecommendsList.find(target)) != -1)
+	       // Skip if we already saw this
+	       std::string target;
+	       for (pkgCache::DepIterator I = Start; I != D; ++I)
 	       {
-		  foundInstalledInOrGroup=true;
-		  break; 
+		  if (target.empty() == false)
+		     target.append(" | ");
+		  target.append(I.TargetPkg().FullName(true));
 	       }
+	       std::list<std::string> &Type = Start->Type == pkgCache::Dep::Recommends ? SingleRecommends : SingleSuggests;
+	       if (std::find(Type.begin(), Type.end(), target) != Type.end())
+		  continue;
+	       Type.push_back(target);
+	    }
 
-	       // this is a dep on a virtual pkg, check if any package that provides it
-	       // should be installed
-	       if(Start.TargetPkg().ProvidesList() != 0)
+	    std::list<std::string> OrList;
+	    bool foundInstalledInOrGroup = false;
+	    for (pkgCache::DepIterator I = Start; I != D; ++I)
+	    {
 	       {
-		  pkgCache::PrvIterator I = Start.TargetPkg().ProvidesList();
-		  for (; I.end() == false; ++I)
+		  // satisfying package is installed and not marked for deletion
+		  APT::VersionList installed = APT::VersionList::FromDependency(Cache, I, APT::CacheSetHelper::INSTALLED);
+		  if (std::find_if(installed.begin(), installed.end(),
+			   [&Cache](pkgCache::VerIterator const &Ver) { return Cache[Ver.ParentPkg()].Delete() == false; }) != installed.end())
 		  {
-		     pkgCache::PkgIterator Pkg = I.OwnerPkg();
-		     if (Cache[Pkg].CandidateVerIter(Cache) == I.OwnerVer() && 
-			 Pkg.CurrentVer() != 0)
-			foundInstalledInOrGroup=true;
+		     foundInstalledInOrGroup = true;
+		     break;
 		  }
 	       }
 
-	       if (Start->Type == pkgCache::Dep::Suggests) 
-	       {
-		  SuggestsOrList += target;
-		  SuggestsOrVersions += std::string(Cache[Start.TargetPkg()].CandVersion) + "\n";
-	       }
-	       
-	       if (Start->Type == pkgCache::Dep::Recommends) 
 	       {
-		  RecommendsOrList += target;
-		  RecommendsOrVersions += std::string(Cache[Start.TargetPkg()].CandVersion) + "\n";
+		  // satisfying package is upgraded to/new install
+		  APT::VersionList upgrades = APT::VersionList::FromDependency(Cache, I, APT::CacheSetHelper::CANDIDATE);
+		  if (std::find_if(upgrades.begin(), upgrades.end(),
+			   [&Cache](pkgCache::VerIterator const &Ver) { return Cache[Ver.ParentPkg()].Upgrade(); }) != upgrades.end())
+		  {
+		     foundInstalledInOrGroup = true;
+		     break;
+		  }
 	       }
 
-	       if (Start >= End)
-		  break;
-	       ++Start;
+	       if (OrList.empty())
+		  OrList.push_back(I.TargetPkg().FullName(true));
+	       else
+		  OrList.push_back("| " + I.TargetPkg().FullName(true));
 	    }
-	    
+
 	    if(foundInstalledInOrGroup == false)
 	    {
-	       RecommendsList += RecommendsOrList;
-	       RecommendsVersions += RecommendsOrVersions;
-	       SuggestsList += SuggestsOrList;
-	       SuggestsVersions += SuggestsOrVersions;
+	       std::list<std::string> &Type = Start->Type == pkgCache::Dep::Recommends ? Recommends : Suggests;
+	       std::move(OrList.begin(), OrList.end(), std::back_inserter(Type));
 	    }
-	       
 	 }
       }
-
-      ShowList(c1out,_("Suggested packages:"),SuggestsList,SuggestsVersions);
-      ShowList(c1out,_("Recommended packages:"),RecommendsList,RecommendsVersions);
-
+      auto always_true = [](std::string const&) { return true; };
+      auto string_ident = [](std::string const&str) { return str; };
+      auto verbose_show_candidate =
+	 [&Cache](std::string str)
+	 {
+	    if (APT::String::Startswith(str, "| "))
+	       str.erase(0, 2);
+	    pkgCache::PkgIterator const Pkg = Cache->FindPkg(str);
+	    if (Pkg.end() == true)
+	       return "";
+	    return (*Cache)[Pkg].CandVersion;
+	 };
+      ShowList(c1out,_("Suggested packages:"), Suggests,
+	    always_true, string_ident, verbose_show_candidate);
+      ShowList(c1out,_("Recommended packages:"), Recommends,
+	    always_true, string_ident, verbose_show_candidate);
    }
 
    // See if we need to prompt
@@ -807,7 +782,7 @@ bool DoInstall(CommandLine &CmdL)
    if (Cache->InstCount() == verset[MOD_INSTALL].size() && Cache->DelCount() == 0)
       return InstallPackages(Cache,false,false);
 
-   return InstallPackages(Cache,false);   
+   return InstallPackages(Cache,false);
 }
 									/*}}}*/
 
@@ -841,8 +816,9 @@ void TryToInstall::operator() (pkgCache::VerIterator const &Ver) {
 	    else
 	       Cache->GetDepCache()->SetReInstall(Pkg, true);
 	 } else
-	    ioprintf(c1out,_("%s is already the newest version.\n"),
-		  Pkg.FullName(true).c_str());
+	    // TRANSLATORS: First string is package name, second is version
+	    ioprintf(c1out,_("%s is already the newest version (%s).\n"),
+		  Pkg.FullName(true).c_str(), Pkg.CurrentVer().VerStr());
       }
 
       // Install it with autoinstalling enabled (if we not respect the minial
diff --git a/apt-private/private-list.cc b/apt-private/private-list.cc
index aa3a2c24b..c4d5e8bc3 100644
--- a/apt-private/private-list.cc
+++ b/apt-private/private-list.cc
@@ -59,7 +59,7 @@ class PackageNameMatcher : public Matcher
       for(J=filters.begin(); J != filters.end(); ++J)
          delete *J;
    }
-   virtual bool operator () (const pkgCache::PkgIterator &P) 
+   virtual bool operator () (const pkgCache::PkgIterator &P) APT_OVERRIDE
    {
       for(J=filters.begin(); J != filters.end(); ++J)
       {
diff --git a/apt-private/private-main.cc b/apt-private/private-main.cc
index 668b1733a..3886c7df6 100644
--- a/apt-private/private-main.cc
+++ b/apt-private/private-main.cc
@@ -22,13 +22,15 @@ void InitSignals()
 
 void CheckSimulateMode(CommandLine &CmdL)
 {
-   // simulate user-friendly if apt-get has no root privileges
-   if (getuid() != 0 && _config->FindB("APT::Get::Simulate") == true &&
+   // disable locking in simulation, but show the message only for users
+   // as root hasn't the same problems like unreadable files which can heavily
+   // distort the simulation.
+   if (_config->FindB("APT::Get::Simulate") == true &&
 	(CmdL.FileSize() == 0 ||
 	 (strcmp(CmdL.FileList[0], "source") != 0 && strcmp(CmdL.FileList[0], "download") != 0 &&
 	  strcmp(CmdL.FileList[0], "changelog") != 0)))
    {
-      if (_config->FindB("APT::Get::Show-User-Simulation-Note",true) == true)
+      if (getuid() != 0 && _config->FindB("APT::Get::Show-User-Simulation-Note",true) == true)
          std::cout << _("NOTE: This is only a simulation!\n"
 	    "      apt-get needs root privileges for real execution.\n"
 	    "      Keep also in mind that locking is deactivated,\n"
diff --git a/apt-private/private-moo.h b/apt-private/private-moo.h
index b8e1cfed6..bc8b3e7dd 100644
--- a/apt-private/private-moo.h
+++ b/apt-private/private-moo.h
@@ -1,6 +1,8 @@
 #ifndef APT_PRIVATE_MOO_H
 #define APT_PRIVATE_MOO_H
 
+#include <apt-pkg/macros.h>
+
 class CommandLine;
 
 APT_PUBLIC bool DoMoo(CommandLine &CmdL);
diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc
index 4e18030ab..b8e6dec02 100644
--- a/apt-private/private-output.cc
+++ b/apt-private/private-output.cc
@@ -25,6 +25,8 @@
 #include <signal.h>
 #include <sys/ioctl.h>
 
+#include <sstream>
+
 #include <apti18n.h>
 									/*}}}*/
 
@@ -199,10 +201,12 @@ static std::string GetShortDescription(pkgCacheFile &CacheFile, pkgRecords &reco
    std::string ShortDescription = "(none)";
    if(ver)
    {
-      pkgCache::DescIterator Desc = ver.TranslatedDescription();
-      pkgRecords::Parser & parser = records.Lookup(Desc.FileList());
-
-      ShortDescription = parser.ShortDesc();
+      pkgCache::DescIterator const Desc = ver.TranslatedDescription();
+      if (Desc.end() == false)
+      {
+	 pkgRecords::Parser & parser = records.Lookup(Desc.FileList());
+	 ShortDescription = parser.ShortDesc();
+      }
    }
    return ShortDescription;
 }
@@ -222,11 +226,14 @@ static std::string GetLongDescription(pkgCacheFile &CacheFile, pkgRecords &recor
       return EmptyDescription;
 
    pkgCache::DescIterator const Desc = ver.TranslatedDescription();
-   pkgRecords::Parser & parser = records.Lookup(Desc.FileList());
-   std::string const longdesc = parser.LongDesc();
-   if (longdesc.empty() == true)
-      return EmptyDescription;
-   return SubstVar(longdesc, "\n ", "\n  ");
+   if (Desc.end() == false)
+   {
+      pkgRecords::Parser & parser = records.Lookup(Desc.FileList());
+      std::string const longdesc = parser.LongDesc();
+      if (longdesc.empty() == false)
+	 return SubstVar(longdesc, "\n ", "\n  ");
+   }
+   return EmptyDescription;
 }
 									/*}}}*/
 void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records,	/*{{{*/
@@ -293,66 +300,6 @@ void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records,	/*{{{*/
    out << output;
 }
 									/*}}}*/
-// ShowList - Show a list						/*{{{*/
-// ---------------------------------------------------------------------
-/* This prints out a string of space separated words with a title and 
-   a two space indent line wraped to the current screen width. */
-bool ShowList(ostream &out,string Title,string List,string VersionsList)
-{
-   if (List.empty() == true)
-      return true;
-   // trim trailing space
-   int NonSpace = List.find_last_not_of(' ');
-   if (NonSpace != -1)
-   {
-      List = List.erase(NonSpace + 1);
-      if (List.empty() == true)
-	 return true;
-   }
-
-   // Acount for the leading space
-   int ScreenWidth = ::ScreenWidth - 3;
-      
-   out << Title << endl;
-   string::size_type Start = 0;
-   string::size_type VersionsStart = 0;
-   while (Start < List.size())
-   {
-      if(_config->FindB("APT::Get::Show-Versions",false) == true &&
-         VersionsList.size() > 0) {
-         string::size_type End;
-         string::size_type VersionsEnd;
-         
-         End = List.find(' ',Start);
-         VersionsEnd = VersionsList.find('\n', VersionsStart);
-
-         out << "   " << string(List,Start,End - Start) << " (" << 
-            string(VersionsList,VersionsStart,VersionsEnd - VersionsStart) << 
-            ")" << endl;
-
-	 if (End == string::npos || End < Start)
-	    End = Start + ScreenWidth;
-
-         Start = End + 1;
-         VersionsStart = VersionsEnd + 1;
-      } else {
-         string::size_type End;
-
-         if (Start + ScreenWidth >= List.size())
-            End = List.size();
-         else
-            End = List.rfind(' ',Start+ScreenWidth);
-
-         if (End == string::npos || End < Start)
-            End = Start + ScreenWidth;
-         out << "  " << string(List,Start,End - Start) << endl;
-         Start = End + 1;
-      }
-   }   
-
-   return false;
-}
-									/*}}}*/
 // ShowBroken - Debugging aide						/*{{{*/
 // ---------------------------------------------------------------------
 /* This prints out the names of all the packages that are broken along
@@ -486,11 +433,9 @@ void ShowBroken(ostream &out, CacheFile &Cache, bool const Now)
       return;
 
    out << _("The following packages have unmet dependencies:") << endl;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {
-      pkgCache::PkgIterator const I(Cache,Cache.List[J]);
-      ShowBrokenPackage(out, &Cache, I, Now);
-   }
+   SortedPackageUniverse Universe(Cache);
+   for (auto const &Pkg: Universe)
+      ShowBrokenPackage(out, &Cache, Pkg, Now);
 }
 void ShowBroken(ostream &out, pkgCacheFile &Cache, bool const Now)
 {
@@ -498,98 +443,64 @@ void ShowBroken(ostream &out, pkgCacheFile &Cache, bool const Now)
       return;
 
    out << _("The following packages have unmet dependencies:") << endl;
-   for (pkgCache::PkgIterator Pkg = Cache->PkgBegin(); Pkg.end() == false; ++Pkg)
+   APT::PackageUniverse Universe(Cache);
+   for (auto const &Pkg: Universe)
       ShowBrokenPackage(out, &Cache, Pkg, Now);
 }
 									/*}}}*/
 // ShowNew - Show packages to newly install				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void ShowNew(ostream &out,CacheFile &Cache)
 {
-   /* Print out a list of packages that are going to be installed extra
-      to what the user asked */
-   string List;
-   string VersionsList;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
-      if (Cache[I].NewInstall() == true) {
-         List += I.FullName(true) + " ";
-         VersionsList += string(Cache[I].CandVersion) + "\n";
-      }
-   }
-   
-   ShowList(out,_("The following NEW packages will be installed:"),List,VersionsList);
+   SortedPackageUniverse Universe(Cache);
+   ShowList(out,_("The following NEW packages will be installed:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].NewInstall(); },
+	 &PrettyFullName,
+	 CandidateVersion(&Cache));
 }
 									/*}}}*/
 // ShowDel - Show packages to delete					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void ShowDel(ostream &out,CacheFile &Cache)
 {
-   /* Print out a list of packages that are going to be removed extra
-      to what the user asked */
-   string List;
-   string VersionsList;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
-      if (Cache[I].Delete() == true)
-      {
-	 if ((Cache[I].iFlags & pkgDepCache::Purge) == pkgDepCache::Purge)
-	    List += I.FullName(true) + "* ";
-	 else
-	    List += I.FullName(true) + " ";
-     
-     VersionsList += string(Cache[I].CandVersion)+ "\n";
-      }
-   }
-   
-   ShowList(out,_("The following packages will be REMOVED:"),List,VersionsList);
+   SortedPackageUniverse Universe(Cache);
+   ShowList(out,_("The following packages will be REMOVED:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].Delete(); },
+	 [&Cache](pkgCache::PkgIterator const &Pkg)
+	 {
+	    std::string str = PrettyFullName(Pkg);
+	    if (((*Cache)[Pkg].iFlags & pkgDepCache::Purge) == pkgDepCache::Purge)
+	       str.append("*");
+	    return str;
+	 },
+	 CandidateVersion(&Cache));
 }
 									/*}}}*/
 // ShowKept - Show kept packages					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void ShowKept(ostream &out,CacheFile &Cache)
 {
-   string List;
-   string VersionsList;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {	 
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
-      
-      // Not interesting
-      if (Cache[I].Upgrade() == true || Cache[I].Upgradable() == false ||
-	  I->CurrentVer == 0 || Cache[I].Delete() == true)
-	 continue;
-      
-      List += I.FullName(true) + " ";
-      VersionsList += string(Cache[I].CurVersion) + " => " + Cache[I].CandVersion + "\n";
-   }
-   ShowList(out,_("The following packages have been kept back:"),List,VersionsList);
+   SortedPackageUniverse Universe(Cache);
+   ShowList(out,_("The following packages have been kept back:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg)
+	 {
+	    return Cache[Pkg].Upgrade() == false &&
+		   Cache[Pkg].Upgradable() == true &&
+		   Pkg->CurrentVer != 0 &&
+		   Cache[Pkg].Delete() == false;
+	 },
+	 &PrettyFullName,
+	 CurrentToCandidateVersion(&Cache));
 }
 									/*}}}*/
 // ShowUpgraded - Show upgraded packages				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void ShowUpgraded(ostream &out,CacheFile &Cache)
 {
-   string List;
-   string VersionsList;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
-      
-      // Not interesting
-      if (Cache[I].Upgrade() == false || Cache[I].NewInstall() == true)
-	 continue;
-
-      List += I.FullName(true) + " ";
-      VersionsList += string(Cache[I].CurVersion) + " => " + Cache[I].CandVersion + "\n";
-   }
-   ShowList(out,_("The following packages will be upgraded:"),List,VersionsList);
+   SortedPackageUniverse Universe(Cache);
+   ShowList(out,_("The following packages will be upgraded:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg)
+	 {
+	    return Cache[Pkg].Upgrade() == true && Cache[Pkg].NewInstall() == false;
+	 },
+	 &PrettyFullName,
+	 CurrentToCandidateVersion(&Cache));
 }
 									/*}}}*/
 // ShowDowngraded - Show downgraded packages				/*{{{*/
@@ -597,74 +508,73 @@ void ShowUpgraded(ostream &out,CacheFile &Cache)
 /* */
 bool ShowDowngraded(ostream &out,CacheFile &Cache)
 {
-   string List;
-   string VersionsList;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
-      
-      // Not interesting
-      if (Cache[I].Downgrade() == false || Cache[I].NewInstall() == true)
-	 continue;
-
-      List += I.FullName(true) + " ";
-      VersionsList += string(Cache[I].CurVersion) + " => " + Cache[I].CandVersion + "\n";
-   }
-   return ShowList(out,_("The following packages will be DOWNGRADED:"),List,VersionsList);
+   SortedPackageUniverse Universe(Cache);
+   return ShowList(out,_("The following packages will be DOWNGRADED:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg)
+	 {
+	    return Cache[Pkg].Downgrade() == true && Cache[Pkg].NewInstall() == false;
+	 },
+	 &PrettyFullName,
+	 CurrentToCandidateVersion(&Cache));
 }
 									/*}}}*/
 // ShowHold - Show held but changed packages				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 bool ShowHold(ostream &out,CacheFile &Cache)
 {
-   string List;
-   string VersionsList;
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
-   {
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
-      if (Cache[I].InstallVer != (pkgCache::Version *)I.CurrentVer() &&
-          I->SelectedState == pkgCache::State::Hold) {
-         List += I.FullName(true) + " ";
-		 VersionsList += string(Cache[I].CurVersion) + " => " + Cache[I].CandVersion + "\n";
-      }
-   }
-
-   return ShowList(out,_("The following held packages will be changed:"),List,VersionsList);
+   SortedPackageUniverse Universe(Cache);
+   return ShowList(out,_("The following held packages will be changed:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg)
+	 {
+	    return Pkg->SelectedState == pkgCache::State::Hold &&
+		   Cache[Pkg].InstallVer != (pkgCache::Version *)Pkg.CurrentVer();
+	 },
+	 &PrettyFullName,
+	 CurrentToCandidateVersion(&Cache));
 }
 									/*}}}*/
 // ShowEssential - Show an essential package warning			/*{{{*/
 // ---------------------------------------------------------------------
 /* This prints out a warning message that is not to be ignored. It shows
-   all essential packages and their dependents that are to be removed. 
+   all essential packages and their dependents that are to be removed.
    It is insanely risky to remove the dependents of an essential package! */
+struct APT_HIDDEN PrettyFullNameWithDue {
+   std::map<unsigned long long, pkgCache::PkgIterator> due;
+   PrettyFullNameWithDue() {}
+   std::string operator() (pkgCache::PkgIterator const &Pkg)
+   {
+      std::string const A = PrettyFullName(Pkg);
+      std::map<unsigned long long, pkgCache::PkgIterator>::const_iterator d = due.find(Pkg->ID);
+      if (d == due.end())
+        return A;
+
+      std::string const B = PrettyFullName(d->second);
+      std::ostringstream outstr;
+      ioprintf(outstr, _("%s (due to %s)"), A.c_str(), B.c_str());
+      return outstr.str();
+   }
+};
 bool ShowEssential(ostream &out,CacheFile &Cache)
 {
-   string List;
-   string VersionsList;
-   bool *Added = new bool[Cache->Head().PackageCount];
-   for (unsigned int I = 0; I != Cache->Head().PackageCount; I++)
-      Added[I] = false;
-   
-   for (unsigned J = 0; J < Cache->Head().PackageCount; J++)
+   std::vector<bool> Added(Cache->Head().PackageCount, false);
+   APT::PackageDeque pkglist;
+   PrettyFullNameWithDue withdue;
+
+   SortedPackageUniverse Universe(Cache);
+   for (pkgCache::PkgIterator const &I: Universe)
    {
-      pkgCache::PkgIterator I(Cache,Cache.List[J]);
       if ((I->Flags & pkgCache::Flag::Essential) != pkgCache::Flag::Essential &&
 	  (I->Flags & pkgCache::Flag::Important) != pkgCache::Flag::Important)
 	 continue;
-      
+
       // The essential package is being removed
-      if (Cache[I].Delete() == true)
+      if (Cache[I].Delete() == false)
+	 continue;
+
+      if (Added[I->ID] == false)
       {
-	 if (Added[I->ID] == false)
-	 {
-	    Added[I->ID] = true;
-	    List += I.FullName(true) + " ";
-        //VersionsList += string(Cache[I].CurVersion) + "\n"; ???
-	 }
+	 Added[I->ID] = true;
+	 pkglist.insert(I);
       }
-      else
-	 continue;
 
       if (I->CurrentVer == 0)
 	 continue;
@@ -676,27 +586,23 @@ bool ShowEssential(ostream &out,CacheFile &Cache)
 	 if (D->Type != pkgCache::Dep::PreDepends &&
 	     D->Type != pkgCache::Dep::Depends)
 	    continue;
-	 
+
 	 pkgCache::PkgIterator P = D.SmartTargetPkg();
 	 if (Cache[P].Delete() == true)
 	 {
 	    if (Added[P->ID] == true)
 	       continue;
 	    Added[P->ID] = true;
-	    
-	    char S[300];
-	    snprintf(S,sizeof(S),_("%s (due to %s) "),P.FullName(true).c_str(),I.FullName(true).c_str());
-	    List += S;
-        //VersionsList += "\n"; ???
-	 }	 
-      }      
+
+	    pkglist.insert(P);
+	    withdue.due[P->ID] = I;
+	 }
+      }
    }
-   
-   delete [] Added;
    return ShowList(out,_("WARNING: The following essential packages will be removed.\n"
-			 "This should NOT be done unless you know exactly what you are doing!"),List,VersionsList);
+			 "This should NOT be done unless you know exactly what you are doing!"),
+	 pkglist, &AlwaysTrue, withdue, &EmptyString);
 }
-
 									/*}}}*/
 // Stats - Show some statistics						/*{{{*/
 // ---------------------------------------------------------------------
@@ -824,3 +730,33 @@ bool AnalPrompt(const char *Text)
    return false;
 }
 									/*}}}*/
+
+std::string PrettyFullName(pkgCache::PkgIterator const &Pkg)
+{
+   return Pkg.FullName(true);
+}
+std::string CandidateVersion(pkgCacheFile * const Cache, pkgCache::PkgIterator const &Pkg)
+{
+   return (*Cache)[Pkg].CandVersion;
+}
+std::function<std::string(pkgCache::PkgIterator const &)> CandidateVersion(pkgCacheFile * const Cache)
+{
+   return std::bind(static_cast<std::string(*)(pkgCacheFile * const, pkgCache::PkgIterator const&)>(&CandidateVersion), Cache, std::placeholders::_1);
+}
+std::string CurrentToCandidateVersion(pkgCacheFile * const Cache, pkgCache::PkgIterator const &Pkg)
+{
+   return std::string((*Cache)[Pkg].CurVersion) + " => " + (*Cache)[Pkg].CandVersion;
+}
+std::function<std::string(pkgCache::PkgIterator const &)> CurrentToCandidateVersion(pkgCacheFile * const Cache)
+{
+   return std::bind(static_cast<std::string(*)(pkgCacheFile * const, pkgCache::PkgIterator const&)>(&CurrentToCandidateVersion), Cache, std::placeholders::_1);
+}
+bool AlwaysTrue(pkgCache::PkgIterator const &)
+{
+      return true;
+}
+std::string EmptyString(pkgCache::PkgIterator const &)
+{
+   return std::string();
+}
+
diff --git a/apt-private/private-output.h b/apt-private/private-output.h
index d5b57adec..4930fd981 100644
--- a/apt-private/private-output.h
+++ b/apt-private/private-output.h
@@ -1,9 +1,11 @@
 #ifndef APT_PRIVATE_OUTPUT_H
 #define APT_PRIVATE_OUTPUT_H
 
+#include <apt-pkg/configuration.h>
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/macros.h>
 
+#include <functional>
 #include <fstream>
 #include <string>
 #include <iostream>
@@ -32,8 +34,61 @@ void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records,
 APT_PUBLIC void ShowBroken(std::ostream &out, CacheFile &Cache, bool const Now);
 APT_PUBLIC void ShowBroken(std::ostream &out, pkgCacheFile &Cache, bool const Now);
 
-APT_PUBLIC bool ShowList(std::ostream &out, std::string Title, std::string List,
-              std::string VersionsList);
+template<class Container, class PredicateC, class DisplayP, class DisplayV> APT_PUBLIC bool ShowList(std::ostream &out, std::string const &Title,
+      Container const &cont,
+      PredicateC Predicate,
+      DisplayP PkgDisplay,
+      DisplayV VerboseDisplay)
+{
+   size_t const ScreenWidth = (::ScreenWidth > 3) ? ::ScreenWidth - 3 : 0;
+   int ScreenUsed = 0;
+   bool const ShowVersions = _config->FindB("APT::Get::Show-Versions", false);
+   bool printedTitle = false;
+
+   for (auto const &Pkg: cont)
+   {
+      if (Predicate(Pkg) == false)
+	 continue;
+
+      if (printedTitle == false)
+      {
+	 out << Title;
+	 printedTitle = true;
+      }
+
+      if (ShowVersions == true)
+      {
+	 out << std::endl << "   " << PkgDisplay(Pkg);
+	 std::string const verbose = VerboseDisplay(Pkg);
+	 if (verbose.empty() == false)
+	    out << " (" << verbose << ")";
+      }
+      else
+      {
+	 std::string const PkgName = PkgDisplay(Pkg);
+	 if (ScreenUsed == 0 || (ScreenUsed + PkgName.length()) >= ScreenWidth)
+	 {
+	    out << std::endl << "  ";
+	    ScreenUsed = 0;
+	 }
+	 else if (ScreenUsed != 0)
+	 {
+	    out << " ";
+	    ++ScreenUsed;
+	 }
+	 out << PkgName;
+	 ScreenUsed += PkgName.length();
+      }
+   }
+
+   if (printedTitle == true)
+   {
+      out << std::endl;
+      return false;
+   }
+   return true;
+}
+
 void ShowNew(std::ostream &out,CacheFile &Cache);
 void ShowDel(std::ostream &out,CacheFile &Cache);
 void ShowKept(std::ostream &out,CacheFile &Cache);
@@ -49,4 +104,12 @@ void Stats(std::ostream &out, pkgDepCache &Dep);
 bool YnPrompt(bool Default=true);
 bool AnalPrompt(const char *Text);
 
+std::string PrettyFullName(pkgCache::PkgIterator const &Pkg);
+std::string CandidateVersion(pkgCacheFile * const Cache, pkgCache::PkgIterator const &Pkg);
+std::function<std::string(pkgCache::PkgIterator const &)> CandidateVersion(pkgCacheFile * const Cache);
+std::string CurrentToCandidateVersion(pkgCacheFile * const Cache, pkgCache::PkgIterator const &Pkg);
+std::function<std::string(pkgCache::PkgIterator const &)> CurrentToCandidateVersion(pkgCacheFile * const Cache);
+std::string EmptyString(pkgCache::PkgIterator const &);
+bool AlwaysTrue(pkgCache::PkgIterator const &);
+
 #endif
diff --git a/apt-private/private-show.cc b/apt-private/private-show.cc
index 790bc0092..3a393b746 100644
--- a/apt-private/private-show.cc
+++ b/apt-private/private-show.cc
@@ -94,7 +94,7 @@ static bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V,
    if (is_installed)
       manual_installed = !(state.Flags & pkgCache::Flag::Auto) ? "yes" : "no";
    else
-      manual_installed = 0;
+      manual_installed = "";
 
    // FIXME: add verbose that does not do the removal of the tags?
    std::vector<pkgTagSection::Tag> RW;
diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc
index 73a82e988..1323771f0 100644
--- a/apt-private/private-update.cc
+++ b/apt-private/private-update.cc
@@ -84,7 +84,8 @@ bool DoUpdate(CommandLine &CmdL)
    if (_config->FindB("APT::Cmd::Show-Update-Stats", false) == true)
    {
       int upgradable = 0;
-      Cache.Open();
+      if (Cache.Open() == false)
+         return false;
       for (pkgCache::PkgIterator I = Cache->PkgBegin(); I.end() != true; ++I)
       {
          pkgDepCache::StateCache &state = Cache[I];