allow the apt/lists/auxfiles/ directory to be missing

apt 1.6~alpha6 introduced aux requests to revamp the implementation of
a-t-mirror. This already included the potential of running as non-root,
but the detection wasn't complete resulting in errors or could produce
spurious warnings along the way if the directory didn't exist yet.

References: ef9677831f62a1554a888ebc7b162517d7881116
Closes: 887624

4 files changed, 76 insertions, 22 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 7fcc5867a..792465b90 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -3979,7 +3979,7 @@ static std::string GetAuxFileNameFromURIInLists(std::string const &uri)
    std::string const tmpfile_tpl = flCombine(dirname, filetag);
    std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
    int const fd = mkstemp(tmpfile.get());
-   if (fd == -1 && errno == EACCES)
+   if (fd == -1)
       return "";
    RemoveFile("GetAuxFileNameFromURI", tmpfile.get());
    close(fd);
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index d964171a6..ba895ed44 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -81,27 +81,41 @@ void pkgAcquire::Initialize()
 static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode)
 {
    std::string const partial = parent + postfix;
-   mode_t const old_umask = umask(S_IWGRP | S_IWOTH);
-   bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
-	 CreateAPTDirectoryIfNeeded(parent, partial) == false);
-   umask(old_umask);
-   if (creation_fail == true)
-      return false;
+   bool const partialExists = DirectoryExists(partial);
+   if (partialExists == false)
+   {
+      mode_t const old_umask = umask(S_IWGRP | S_IWOTH);
+      bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
+	    CreateAPTDirectoryIfNeeded(parent, partial) == false);
+      umask(old_umask);
+      if (creation_fail == true)
+	 return false;
+   }
 
    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
-   if (getuid() == 0 && SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
+   if (getuid() == 0)
    {
-      struct passwd const * const pw = getpwnam(SandboxUser.c_str());
-      struct group const * const gr = getgrnam(ROOT_GROUP);
-      if (pw != NULL && gr != NULL)
+      if (SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it
       {
-         // chown the partial dir
-         if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
-            _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
+	 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
+	 struct group const * const gr = getgrnam(ROOT_GROUP);
+	 if (pw != NULL && gr != NULL)
+	 {
+	    // chown the partial dir
+	    if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
+	       _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
+	 }
       }
+      if (chmod(partial.c_str(), mode) != 0)
+	 _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
+
+   }
+   else if (chmod(partial.c_str(), mode) != 0)
+   {
+      // if we haven't created the dir and aren't root, it is kinda expected that chmod doesn't work
+      if (partialExists == false)
+	 _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
    }
-   if (chmod(partial.c_str(), mode) != 0)
-      _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
 
    _error->PushToStack();
    // remove 'old' FAILED files to stop us from collecting them for no reason
@@ -120,7 +134,9 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("List directory %s is missing."), (listDir + "partial").c_str());
       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
-	 return _error->Errno("Acquire", _("List directory %s is missing."), (listDir + "auxfiles").c_str());
+      {
+	 // not being able to create lists/auxfiles isn't critical as we will use a tmpdir then
+      }
       string const archivesDir = _config->FindDir("Dir::Cache::Archives");
       if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("Archives directory %s is missing."), (archivesDir + "partial").c_str());
@@ -150,7 +166,9 @@ bool pkgAcquire::GetLock(std::string const &Lock)
    if (Lock == listDir || Lock == archivesDir)
    {
       if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
-	 return _error->Errno("Acquire", _("List directory %s is missing."), (listDir + "auxfiles").c_str());
+      {
+	 // not being able to create lists/auxfiles isn't critical as we will use a tmpdir then
+      }
    }
 
    if (_config->FindB("Debug::NoLocking", false) == true)
diff --git a/test/integration/framework b/test/integration/framework
index ff7a7c514..ecce46d2e 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -177,7 +177,11 @@ runapt() {
 	sh|aptitude|*/*|command) ;;
 	*) CMD="${BUILDDIRECTORY}/$CMD";;
 	esac
-	MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH="${LIBRARYPATH}:${LD_LIBRARY_PATH}" "$CMD" "$@"
+	if [ "$CMD" = 'aptitude' ]; then
+		MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH="${LIBRARYPATH}:${LD_LIBRARY_PATH}" command "$CMD" "$@"
+	else
+		MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH="${LIBRARYPATH}:${LD_LIBRARY_PATH}" "$CMD" "$@"
+	fi
 }
 runpython3() { runapt command python3 "$@"; }
 aptconfig() { runapt apt-config "$@"; }
diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror
index d00118405..99d0c9700 100755
--- a/test/integration/test-method-mirror
+++ b/test/integration/test-method-mirror
@@ -14,15 +14,37 @@ changetowebserver
 webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "http://localhost:${APTHTTPPORT}/"
 addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/lib/apt/lists || true;"
 
-testrundownload_internal() {
-	cd downloaded
-	testsuccess apt download "$@"
+testrundownload_internal_debs() {
+	local WORKDIR="$1"
+	shift
+	if [ "$WORKDIR" != '.' ]; then cd "$WORKDIR"; fi
 	while [ -n "$1" ]; do
+		if [ "$1" = 'foo=1' ]; then shift; continue; fi
 		local fndeb="$(echo "$1" | tr '=' '_')_all.deb"
 		testsuccess test -e "$fndeb"
 		rm -f "$fndeb"
 		shift
 	done
+	if [ "$WORKDIR" != '.' ]; then cd - >/dev/null; fi
+}
+testrundownload_internal() {
+	cd downloaded
+	testsuccess apt install --download-only -o Debug::NoLocking=true -y "$@"
+	testrundownload_internal_debs '../rootdir/var/cache/apt/archives' "$@"
+	msgtest 'Repeat the download only install test with' 'aptitude'
+	if dpkg-checkbuilddeps -d 'aptitude' /dev/null >/dev/null 2>&1; then
+		if aptitude install --download-only -o Debug::NoLocking=true -y "$@" > ../rootdir/tmp/aptitude.log 2>&1; then
+			msgpass
+		else
+			msgfail
+			cat >&2 ../rootdir/tmp/aptitude.log
+		fi
+		testrundownload_internal_debs '../rootdir/var/cache/apt/archives' "$@"
+	else
+		msgskip 'not installed'
+	fi
+	testsuccess apt download "$@" -o Debug::pkgAcquire::Worker=1
+	testrundownload_internal_debs '.' "$@"
 	testempty find . -name '*mirror*'
 	cd ..
 }
@@ -37,6 +59,10 @@ testrundownload() {
 		chmod -f -R -w rootdir/var/lib/apt/lists/auxfiles
 		testrundownload_internal "$@"
 		chmod -f -R +w rootdir/var/lib/apt/lists
+		rm -rf rootdir/var/lib/apt/lists/auxfiles
+		chmod -f -R -w rootdir/var/lib/apt/lists
+		testrundownload_internal "$@"
+		chmod -f -R +w rootdir/var/lib/apt/lists
 	fi
 }
 testrun() {
@@ -161,6 +187,7 @@ Hit:2 http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
 Reading package lists...
 Building dependency tree...
 All packages are up to date." apt update
+testrundownload 'foo=2'
 sed -i -e 's# mirror+# foo+mirror+#' rootdir/etc/apt/sources.list.d/*
 testfailure apt update
 testsuccess grep 'package apt-transport-foo installed' rootdir/tmp/testfailure.output
@@ -170,6 +197,8 @@ Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
 Reading package lists...
 Building dependency tree...
 All packages are up to date." apt update
+testrundownload 'foo=2'
+
 echo "file:/nonexistent/apt/archive	priority:1
 http://localhost:${APTHTTPPORT}/redirectme
 " > aptarchive/mirror.txt
@@ -183,8 +212,11 @@ Building dependency tree...
 All packages are up to date." apt update
 sed -i -e "s#+file:${APTARCHIVE}#+http://localhost:${APTHTTPPORT}#" rootdir/etc/apt/sources.list.d/*
 testsuccess apt update
+testrundownload 'foo=2'
+
 testsuccessequal "Get:1 foo+http://localhost:${APTHTTPPORT}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
 Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
 Reading package lists...
 Building dependency tree...
 All packages are up to date." apt update
+testrundownload 'foo=2'