fail instead of segfault on unreadable config files

The report mentions "apt list --upgradable", but there are others which
have inconsistent behavior ranging from segfaulting to doing something
with the partial (and hence incomplete) data. We had a recent report
about sources.list (#818628), this one mentions prefences, the obvious
next step is conf files… so the testcase is adapted to check for all
three in file and directory versions and run a bunch of commands each
time which should all have more or less the same behavior in such a case
(aka error out).

Closes: 824503
(cherry picked from commit fdf9eef4d96a18d0167708499c993e1174251e88)

8 files changed, 111 insertions, 51 deletions

diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc
index 0e2192973..b5f32fc29 100644
--- a/apt-pkg/cachefile.cc
+++ b/apt-pkg/cachefile.cc
@@ -173,6 +173,9 @@ bool pkgCacheFile::BuildPolicy(OpProgress * /*Progress*/)
 /* */
 bool pkgCacheFile::BuildDepCache(OpProgress *Progress)
 {
+   if (BuildCaches(Progress, false) == false)
+      return false;
+
    std::unique_ptr<pkgDepCache> DCache;
    if (this->DCache != NULL)
       return true;
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index b36f4d4b7..06c516e53 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -414,7 +414,12 @@ bool ReadPinDir(pkgPolicy &Plcy,string Dir)
       return true;
    }
 
+   _error->PushToStack();
    vector<string> const List = GetListOfFilesInDir(Dir, "pref", true, true);
+   bool const PendingErrors = _error->PendingError();
+   _error->MergeWithStack();
+   if (PendingErrors)
+      return false;
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
diff --git a/apt-private/private-cacheset.cc b/apt-private/private-cacheset.cc
index 981766cdf..52cd22d2a 100644
--- a/apt-private/private-cacheset.cc
+++ b/apt-private/private-cacheset.cc
@@ -30,13 +30,18 @@ bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile,
                                  Matcher &matcher,
                                  OpProgress * const progress)
 {
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   pkgDepCache *DepCache = CacheFile.GetDepCache();
+   pkgCache * const Cache = CacheFile.GetPkgCache();
+   if (unlikely(Cache == nullptr))
+      return false;
+   if (progress != nullptr)
+      progress->SubProgress(Cache->Head().PackageCount, _("Sorting"));
+
+   pkgDepCache * const DepCache = CacheFile.GetDepCache();
+   if (unlikely(DepCache == nullptr))
+      return false;
    APT::CacheSetHelper helper(false);
 
    int Done=0;
-   if (progress != NULL)
-      progress->SubProgress(Cache->Head().PackageCount, _("Sorting"));
 
    bool const insertCurrentVer = _config->FindB("APT::Cmd::Installed", false);
    bool const insertUpgradable = _config->FindB("APT::Cmd::Upgradable", false);
diff --git a/apt-private/private-depends.cc b/apt-private/private-depends.cc
index 955f3dc67..3a3a2737a 100644
--- a/apt-private/private-depends.cc
+++ b/apt-private/private-depends.cc
@@ -26,8 +26,8 @@
 static bool ShowDepends(CommandLine &CmdL, bool const RevDepends)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   if (unlikely(Cache == NULL))
+   pkgCache * const Cache = CacheFile.GetPkgCache();
+   if (unlikely(Cache == nullptr || CacheFile.GetDepCache() == nullptr))
       return false;
 
    CacheSetHelperVirtuals helper(false);
diff --git a/apt-private/private-list.cc b/apt-private/private-list.cc
index a948c7d9f..32c8eeaa2 100644
--- a/apt-private/private-list.cc
+++ b/apt-private/private-list.cc
@@ -93,8 +93,8 @@ static void ListAllVersions(pkgCacheFile &CacheFile, pkgRecords &records,/*{{{*/
 bool DoList(CommandLine &Cmd)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   if (unlikely(Cache == NULL))
+   pkgCache * const Cache = CacheFile.GetPkgCache();
+   if (unlikely(Cache == nullptr || CacheFile.GetDepCache() == nullptr))
       return false;
    pkgRecords records(CacheFile);
 
diff --git a/apt-private/private-show.cc b/apt-private/private-show.cc
index 792c455a8..cd7a2a064 100644
--- a/apt-private/private-show.cc
+++ b/apt-private/private-show.cc
@@ -258,6 +258,8 @@ bool ShowPackage(CommandLine &CmdL)					/*{{{*/
    CacheSetHelperVirtuals helper(true, GlobalError::NOTICE);
    APT::CacheSetHelper::VerSelector const select = _config->FindB("APT::Cache::AllVersions", true) ?
 			APT::CacheSetHelper::ALL : APT::CacheSetHelper::CANDIDATE;
+   if (select == APT::CacheSetHelper::CANDIDATE && CacheFile.GetDepCache() == nullptr)
+      return false;
    APT::VersionList const verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, select, helper);
    int const ShowVersion = _config->FindI("APT::Cache::Show::Version", 1);
    for (APT::VersionList::const_iterator Ver = verset.begin(); Ver != verset.end(); ++Ver)
@@ -356,10 +358,14 @@ bool ShowSrcPackage(CommandLine &CmdL)					/*{{{*/
 bool Policy(CommandLine &CmdL)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   pkgPolicy *Plcy = CacheFile.GetPolicy();
-   pkgSourceList *SrcList = CacheFile.GetSourceList();
-   if (unlikely(Cache == NULL || Plcy == NULL || SrcList == NULL))
+   pkgSourceList const * const SrcList = CacheFile.GetSourceList();
+   if (unlikely(SrcList == nullptr))
+      return false;
+   pkgCache * const Cache = CacheFile.GetPkgCache();
+   if (unlikely(Cache == nullptr))
+      return false;
+   pkgPolicy * const Plcy = CacheFile.GetPolicy();
+   if (unlikely(Plcy == nullptr))
       return false;
 
    bool OldPolicy = _config->FindI("APT::Policy", 1) < 1;
diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc
index 8b615a83b..4e86edc77 100644
--- a/cmdline/apt-mark.cc
+++ b/cmdline/apt-mark.cc
@@ -48,9 +48,8 @@ using namespace std;
 static bool DoAuto(CommandLine &CmdL)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   pkgDepCache *DepCache = CacheFile.GetDepCache();
-   if (unlikely(Cache == NULL || DepCache == NULL))
+   pkgDepCache * const DepCache = CacheFile.GetDepCache();
+   if (unlikely(DepCache == nullptr))
       return false;
 
    APT::PackageList pkgset = APT::PackageList::FromCommandLine(CacheFile, CmdL.FileList + 1);
@@ -95,9 +94,8 @@ static bool DoAuto(CommandLine &CmdL)
 static bool DoMarkAuto(CommandLine &CmdL)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   pkgDepCache *DepCache = CacheFile.GetDepCache();
-   if (unlikely(Cache == NULL || DepCache == NULL))
+   pkgDepCache * const DepCache = CacheFile.GetDepCache();
+   if (unlikely(DepCache == nullptr))
       return false;
 
    APT::PackageList pkgset = APT::PackageList::FromCommandLine(CacheFile, CmdL.FileList + 1);
@@ -132,9 +130,8 @@ static bool DoMarkAuto(CommandLine &CmdL)
 static bool ShowAuto(CommandLine &CmdL)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   pkgDepCache *DepCache = CacheFile.GetDepCache();
-   if (unlikely(Cache == NULL || DepCache == NULL))
+   pkgDepCache * const DepCache = CacheFile.GetDepCache();
+   if (unlikely(DepCache == nullptr))
       return false;
 
    std::vector<string> packages;
@@ -143,8 +140,8 @@ static bool ShowAuto(CommandLine &CmdL)
 
    if (CmdL.FileList[1] == 0)
    {
-      packages.reserve(Cache->HeaderP->PackageCount / 3);
-      for (pkgCache::PkgIterator P = Cache->PkgBegin(); P.end() == false; ++P)
+      packages.reserve(DepCache->Head().PackageCount / 3);
+      for (pkgCache::PkgIterator P = DepCache->PkgBegin(); P.end() == false; ++P)
 	 if (P->CurrentVer != 0 &&
 	     (((*DepCache)[P].Flags & pkgCache::Flag::Auto) == pkgCache::Flag::Auto) == ShowAuto)
 	    packages.push_back(P.FullName(true));
@@ -172,8 +169,8 @@ static bool ShowAuto(CommandLine &CmdL)
 static bool DoSelection(CommandLine &CmdL)
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   if (unlikely(Cache == NULL))
+   pkgCache * const Cache = CacheFile.GetPkgCache();
+   if (unlikely(Cache == nullptr))
       return false;
 
    APT::VersionVector pkgset = APT::VersionVector::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::CacheSetHelper::INSTCAND);
@@ -238,8 +235,8 @@ static bool DoSelection(CommandLine &CmdL)
 static bool ShowSelection(CommandLine &CmdL)				/*{{{*/
 {
    pkgCacheFile CacheFile;
-   pkgCache *Cache = CacheFile.GetPkgCache();
-   if (unlikely(Cache == NULL))
+   pkgCache * const Cache = CacheFile.GetPkgCache();
+   if (unlikely(Cache == nullptr))
       return false;
 
    pkgCache::State::PkgSelectedState selector;
diff --git a/test/integration/test-bug-818628-unreadable-source b/test/integration/test-bug-818628-unreadable-source
index 84776dd8d..cddc79398 100755
--- a/test/integration/test-bug-818628-unreadable-source
+++ b/test/integration/test-bug-818628-unreadable-source
@@ -4,39 +4,83 @@ set -e
 TESTDIR="$(readlink -f "$(dirname "$0")")"
 . "$TESTDIR/framework"
 setupenvironment
-configarchitecture 'amd64' 'i386'
+configarchitecture 'amd64'
 
-if [ "$(id -u)" = "0" ]; then
-    msgskip "Tests for unreadable sources.list do not work as root"
+if [ "$(id -u)" = '0' ]; then
+    msgskip 'Tests for unreadable files do not work as root'
     exit 0
 fi
 
-insertinstalledpackage 'apt' 'i386' '1'
-buildsimplenativepackage 'apt' 'i386' '2' 'unstable'
+insertinstalledpackage 'foo' 'amd64' '1'
+insertpackage 'unstable' 'foo' 'amd64' '2'
 
-setupaptarchive
+setupaptarchive --no-update
 
-# Test unreadable sources.list files
-chmod -r rootdir/etc/apt/sources.list.d
+touch rootdir/etc/apt/apt.conf.d/unreadable.conf
+touch rootdir/etc/apt/preferences.d/unreadable.pref
 
-testfailureequal "E: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/ - opendir (13: Permission denied)
-W: You may want to run apt-get update to correct these problems
-E: The package cache file is corrupted" aptcache policy apt
-testfailureequal "E: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/ - opendir (13: Permission denied)
-W: You may want to run apt-get update to correct these problems
-E: The package cache file is corrupted" apt search apt
+addtrap 'prefix' "chmod -R +r '$(escape_shell "$TMPWORKINGDIRECTORY")/rootdir/etc/apt';"
+chmod -r rootdir/etc/apt/trusted.gpg.d
+testfailure apt update
+chmod +r rootdir/etc/apt/trusted.gpg.d
+testsuccess apt update
+testsuccess apt policy foo
+testsuccess apt search foo
+testsuccessequal "Listing...
+foo/unstable 2 amd64 [upgradable from: 1]
+N: There is 1 additional version. Please use the '-a' switch to see it" apt list --upgradable
 
-chmod +r rootdir/etc/apt/sources.list.d
+runthemall() {
+	local ERR1="$1"
+	local ERR2="$1$2"
+	testfailureequal "$ERR1" aptcache policy
+	testfailureequal "$ERR1" aptcache policy foo
+	testfailureequal "$ERR2" aptcache depends foo
+	testfailureequal "$ERR2" aptcache rdepends foo
+	testfailureequal "$ERR2" aptcache search foo
+	testfailureequal "$ERR1" apt policy
+	testfailureequal "$ERR1" apt policy foo
+	testfailureequal "$ERR2" apt depends foo
+	testfailureequal "$ERR2" apt rdepends foo
+	testfailureequal "$ERR2" apt search foo
+	testfailureequal "$ERR2" apt list --upgradable
+	testfailureequal "$ERR2" apt show foo
+	testfailureequal "$ERR2" aptcache show foo --no-all-versions
+	testfailureequal "$ERR2" aptmark auto foo
+	testfailureequal "$ERR2" aptmark manual foo
+	testfailureequal "$ERR2" aptmark auto foo
+}
+echo 'Apt::Cmd::Disable-Script-Warning "true";' >> aptconfig.conf
 
-
-# Test unreadable sources.list files
+msgmsg 'Unreadable sources file'
 chmod -r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list
+runthemall "E: Opening $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list - ifstream::ifstream (13: Permission denied)
+E: The list of sources could not be read."
+chmod +r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list
 
-testfailureequal "E: Opening $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list - ifstream::ifstream (13: Permission denied)
-E: The list of sources could not be read.
-E: Opening $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list - ifstream::ifstream (13: Permission denied)
-E: The list of sources could not be read." aptcache policy apt
-testfailureequal "E: Opening $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list - ifstream::ifstream (13: Permission denied)
-E: The list of sources could not be read." apt search apt
+msgmsg 'Unreadable config file'
+chmod -r rootdir/etc/apt/apt.conf.d/unreadable.conf
+runthemall "E: Opening configuration file ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/unreadable.conf - ifstream::ifstream (13: Permission denied)"
+chmod +r rootdir/etc/apt/apt.conf.d/unreadable.conf
 
-chmod +r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list
+msgmsg 'Unreadable preferences file'
+chmod -r rootdir/etc/apt/preferences.d/unreadable.pref
+runthemall "E: Could not open file ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/preferences.d/unreadable.pref - open (13: Permission denied)"
+chmod +r rootdir/etc/apt/preferences.d/unreadable.pref
+
+msgmsg 'Unreadable sources directory'
+chmod -r rootdir/etc/apt/sources.list.d
+runthemall "E: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/ - opendir (13: Permission denied)" "
+W: You may want to run apt-get update to correct these problems
+E: The package cache file is corrupted"
+chmod +r rootdir/etc/apt/sources.list.d
+
+msgmsg 'Unreadable config directory'
+chmod -r rootdir/etc/apt/apt.conf.d
+runthemall "E: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/ - opendir (13: Permission denied)"
+chmod +r rootdir/etc/apt/apt.conf.d
+
+msgmsg 'Unreadable preferences directory'
+chmod -r rootdir/etc/apt/preferences.d
+runthemall "E: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/preferences.d/ - opendir (13: Permission denied)"
+chmod +r rootdir/etc/apt/preferences.d