diff options
-rw-r--r-- | apt-pkg/acquire-item.cc | 119 | ||||
-rw-r--r-- | apt-pkg/acquire-item.h | 19 | ||||
-rw-r--r-- | apt-pkg/acquire-worker.cc | 60 | ||||
-rw-r--r-- | apt-pkg/acquire-worker.h | 1 | ||||
-rw-r--r-- | apt-pkg/acquire.cc | 40 | ||||
-rw-r--r-- | apt-pkg/clean.cc | 11 | ||||
-rw-r--r-- | apt-pkg/init.cc | 1 | ||||
-rw-r--r-- | apt-private/private-download.cc | 3 | ||||
-rw-r--r-- | doc/method.dbk | 23 | ||||
-rw-r--r-- | test/integration/framework | 2 | ||||
-rwxr-xr-x | test/integration/test-apt-get-update-unauth-warning | 3 | ||||
-rwxr-xr-x | test/integration/test-ubuntu-bug-346386-apt-get-update-paywall | 3 |
12 files changed, 258 insertions, 27 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 9b163081b..611876dd2 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -16,6 +16,7 @@ #include <config.h> #include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> #include <apt-pkg/acquire.h> #include <apt-pkg/aptconfiguration.h> #include <apt-pkg/configuration.h> @@ -34,6 +35,7 @@ #include <algorithm> #include <ctime> #include <iostream> +#include <memory> #include <numeric> #include <random> #include <sstream> @@ -1070,6 +1072,12 @@ bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/ /*}}}*/ int pkgAcquire::Item::Priority() /*{{{*/ { + // Stage 0: Files requested by methods + // - they will usually not end up here, but if they do we make sure + // to get them as soon as possible as they are probably blocking + // the processing of files by the requesting method + if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr) + return 5000; // Stage 1: Meta indices and diff indices // - those need to be fetched first to have progress reporting working // for the rest @@ -3892,3 +3900,114 @@ string pkgAcqFile::Custom600Headers() const /*{{{*/ } /*}}}*/ pkgAcqFile::~pkgAcqFile() {} + +void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + pkgAcqFile::Failed(Message, Cnf); + if (Status == StatIdle) + return; + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); + Worker->ReplyAux(Desc); +} + /*}}}*/ +void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/ + pkgAcquire::MethodConfig const *const Cnf) +{ + pkgAcqFile::Done(Message, CalcHashes, Cnf); + if (Status == StatDone) + Worker->ReplyAux(Desc); + else if (Status == StatAuthError || Status == StatError) + Worker->ReplyAux(Desc); +} + /*}}}*/ +std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/ +{ + if (MaximumSize == 0) + return pkgAcqFile::Custom600Headers(); + std::string maxsize; + strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize); + return pkgAcqFile::Custom600Headers().append(maxsize); +} + /*}}}*/ +void pkgAcqAuxFile::Finished() /*{{{*/ +{ + auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + if (APT::String::Startswith(DestFile, dirname)) + { + // the file is never returned by method requesting it, so fix up the permission now + if (FileExists(DestFile)) + { + ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644); + if (Status == StatDone) + return; + } + } + else + { + dirname = flNotFile(DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED"); + rmdir(dirname.c_str()); + } + DestFile.clear(); +} + /*}}}*/ +// GetAuxFileNameFromURI /*{{{*/ +static std::string GetAuxFileNameFromURIInLists(std::string const &uri) +{ + // check if we have write permission for our usual location. + auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + char const * const filetag = ".apt-acquire-privs-test.XXXXXX"; + std::string const tmpfile_tpl = flCombine(dirname, filetag); + std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free }; + int const fd = mkstemp(tmpfile.get()); + if (fd == -1 && errno == EACCES) + return ""; + RemoveFile("GetAuxFileNameFromURI", tmpfile.get()); + close(fd); + return flCombine(dirname, URItoFileName(uri)); +} +static std::string GetAuxFileNameFromURI(std::string const &uri) +{ + auto const lists = GetAuxFileNameFromURIInLists(uri); + if (lists.empty() == false) + return lists; + + std::string tmpdir_tpl; + strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str()); + std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free }; + if (mkdtemp(tmpdir.get()) == nullptr) + { + _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get()); + return flCombine("/nonexistent/auxfiles/", URItoFileName(uri)); + } + chmod(tmpdir.get(), 0755); + auto const filename = flCombine(tmpdir.get(), URItoFileName(uri)); + _error->PushToStack(); + FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly); + if (in.IsOpen()) + { + FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive); + CopyFile(in, out); + } + _error->RevertToStack(); + return filename; +} + /*}}}*/ +pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false), + Owner(Owner), Worker(Worker), MaximumSize(MaximumSize) +{ + /* very bad failures can happen while constructing which causes + us to hang as the aux request is never answered (e.g. method not available) + Ideally we catch failures earlier, but a safe guard can't hurt. */ + if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching) + return; + Failed(std::string("400 URI Failure\n") + + "URI: " + URI + "\n" + + "Filename: " + DestFile, + nullptr); +} +pkgAcqAuxFile::~pkgAcqAuxFile() {} diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index cf227d1b5..46d79df92 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -1238,6 +1238,25 @@ class pkgAcqFile : public pkgAcquire::Item virtual ~pkgAcqFile(); }; /*}}}*/ +class APT_HIDDEN pkgAcqAuxFile : public pkgAcqFile /*{{{*/ +{ + pkgAcquire::Item *const Owner; + pkgAcquire::Worker *const Worker; + unsigned long long MaximumSize; + + public: + virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + + pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize); + virtual ~pkgAcqAuxFile(); +}; + /*}}}*/ /** @} */ #endif diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 995750dea..1b1f4dc4c 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -192,7 +192,8 @@ bool pkgAcquire::Worker::ReadMessages() // --------------------------------------------------------------------- /* This takes the messages from the message queue and runs them through the parsers in order. */ -enum class APT_HIDDEN MessageType { +enum class APT_HIDDEN MessageType +{ CAPABILITIES = 100, LOG = 101, STATUS = 102, @@ -200,6 +201,7 @@ enum class APT_HIDDEN MessageType { WARNING = 104, URI_START = 200, URI_DONE = 201, + AUX_REQUEST = 351, URI_FAILURE = 400, GENERAL_FAILURE = 401, MEDIA_CHANGE = 403 @@ -512,6 +514,22 @@ bool pkgAcquire::Worker::RunMessages() break; } + case MessageType::AUX_REQUEST: + { + if (Itm == nullptr) + { + _error->Error("Method gave invalid Aux Request message"); + break; + } + + auto maxsizestr = LookupTag(Message, "MaximumSize", ""); + unsigned long long const MaxSize = maxsizestr.empty() ? 0 : strtoull(maxsizestr.c_str(), nullptr, 10); + new pkgAcqAuxFile(Itm->Owner, this, LookupTag(Message, "Aux-ShortDesc", ""), + LookupTag(Message, "Aux-Description", ""), LookupTag(Message, "Aux-URI", ""), + GetHashesFromMessage("Aux-", Message), MaxSize); + break; + } + case MessageType::URI_FAILURE: { if (Itm == nullptr) @@ -772,6 +790,46 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) return true; } /*}}}*/ +// Worker::ReplyAux - reply to an aux request from this worker /*{{{*/ +bool pkgAcquire::Worker::ReplyAux(pkgAcquire::ItemDesc const &Item) +{ + if (OutFd == -1) + return false; + + if (isDoomedItem(Item.Owner)) + return true; + + string Message = "600 URI Acquire\n"; + Message.reserve(200); + Message += "URI: " + Item.URI; + if (RealFileExists(Item.Owner->DestFile)) + { + if (Item.Owner->Status == pkgAcquire::Item::StatDone) + { + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("Worker::ReplyAux", Item.Owner->DestFile.c_str(), + SandboxUser.c_str(), ROOT_GROUP, 0600); + Message += "\nFilename: " + Item.Owner->DestFile; + } + else + { + // we end up here in case we would need root-rights to delete a file, + // but we run the command as non-root… (yes, it is unlikely) + Message += "\nFilename: " + flCombine("/nonexistent", Item.Owner->DestFile); + } + } + else + Message += "\nFilename: " + Item.Owner->DestFile; + Message += "\n\n"; + + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Message, "\n") << endl; + OutQueue += Message; + OutReady = true; + + return true; +} + /*}}}*/ // Worker::OutFdRead - Out bound FD is ready /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h index 8fc686880..3c5c1cef6 100644 --- a/apt-pkg/acquire-worker.h +++ b/apt-pkg/acquire-worker.h @@ -276,6 +276,7 @@ class pkgAcquire::Worker : public WeakPointable * queue. */ bool QueueItem(pkgAcquire::Queue::QItem *Item); + APT_HIDDEN bool ReplyAux(pkgAcquire::ItemDesc const &Item); /** \brief Start up the worker and fill in #Config. * diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index aabcb0aba..5fa456ce3 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -78,13 +78,13 @@ void pkgAcquire::Initialize() } /*}}}*/ // Acquire::GetLock - lock directory and prepare for action /*{{{*/ -static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode) { - std::string const partial = parent + "partial"; - mode_t const mode = umask(S_IWGRP | S_IWOTH); + std::string const partial = parent + postfix; + mode_t const old_umask = umask(S_IWGRP | S_IWOTH); bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false && CreateAPTDirectoryIfNeeded(parent, partial) == false); - umask(mode); + umask(old_umask); if (creation_fail == true) return false; @@ -100,7 +100,7 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); } } - if (chmod(partial.c_str(), 0700) != 0) + if (chmod(partial.c_str(), mode) != 0) _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); _error->PushToStack(); @@ -117,10 +117,12 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) if (Lock.empty()) { string const listDir = _config->FindDir("Dir::State::lists"); - if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false) return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false) + return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str()); string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false) return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); return true; } @@ -137,14 +139,19 @@ bool pkgAcquire::GetLock(std::string const &Lock) if (Lock == listDir) { - if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false) return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); } if (Lock == archivesDir) { - if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false) return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); } + if (Lock == listDir || Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false) + return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str()); + } if (_config->FindB("Debug::NoLocking", false) == true) return true; @@ -288,7 +295,6 @@ static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item, "\nFilename: " + Item->DestFile + "\nFailReason: WeakHashSums"; - auto SavedDesc = Item->GetItemDesc(); Item->Status = pkgAcquire::Item::StatAuthError; Item->Failed(Message, Config); if (Log != nullptr) @@ -303,7 +309,10 @@ void pkgAcquire::Enqueue(ItemDesc &Item) const MethodConfig *Config; string Name = QueueName(Item.URI,Config); if (Name.empty() == true) + { + Item.Owner->Status = pkgAcquire::Item::StatError; return; + } /* the check for running avoids that we produce errors in logging before we actually have started, which would @@ -769,11 +778,12 @@ bool pkgAcquire::Clean(string Dir) for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D)) { // Skip some entries - if (strcmp(E->d_name,"lock") == 0 || - strcmp(E->d_name,"partial") == 0 || - strcmp(E->d_name,"lost+found") == 0 || - strcmp(E->d_name,".") == 0 || - strcmp(E->d_name,"..") == 0) + if (strcmp(E->d_name, "lock") == 0 || + strcmp(E->d_name, "partial") == 0 || + strcmp(E->d_name, "auxfiles") == 0 || + strcmp(E->d_name, "lost+found") == 0 || + strcmp(E->d_name, ".") == 0 || + strcmp(E->d_name, "..") == 0) continue; // Look in the get list and if not found nuke diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc index 04a7c4910..1abc638ee 100644 --- a/apt-pkg/clean.cc +++ b/apt-pkg/clean.cc @@ -62,11 +62,12 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache) for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) { // Skip some files.. - if (strcmp(Dir->d_name,"lock") == 0 || - strcmp(Dir->d_name,"partial") == 0 || - strcmp(Dir->d_name,"lost+found") == 0 || - strcmp(Dir->d_name,".") == 0 || - strcmp(Dir->d_name,"..") == 0) + if (strcmp(Dir->d_name, "lock") == 0 || + strcmp(Dir->d_name, "partial") == 0 || + strcmp(Dir->d_name, "auxfiles") == 0 || + strcmp(Dir->d_name, "lost+found") == 0 || + strcmp(Dir->d_name, ".") == 0 || + strcmp(Dir->d_name, "..") == 0) continue; struct stat St; diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 207f0d902..5c34113e7 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -138,7 +138,6 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.CndSet("Dir::State", STATE_DIR + 1); Cnf.CndSet("Dir::State::lists","lists/"); Cnf.CndSet("Dir::State::cdroms","cdroms.list"); - Cnf.CndSet("Dir::State::mirrors","mirrors/"); // Cache Cnf.CndSet("Dir::Cache", CACHE_DIR + 1); diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc index 77c35f4de..8bd7e33e0 100644 --- a/apt-private/private-download.cc +++ b/apt-private/private-download.cc @@ -225,7 +225,8 @@ bool DoDownload(CommandLine &CmdL) std::string const filename = cwd + flNotDir((*I)->DestFile); if ((*I)->Local == true && filename != (*I)->DestFile && - (*I)->Status == pkgAcquire::Item::StatDone) + (*I)->Status == pkgAcquire::Item::StatDone && + dynamic_cast<pkgAcqArchive*>(*I) != nullptr) { std::ifstream src((*I)->DestFile.c_str(), std::ios::binary); std::ofstream dst(filename.c_str(), std::ios::binary); diff --git a/doc/method.dbk b/doc/method.dbk index e5e035a2b..410d6898c 100644 --- a/doc/method.dbk +++ b/doc/method.dbk @@ -267,6 +267,11 @@ an informational string provided for visual debugging. </listitem> <listitem> <para> +351 Aux Request - Method requests an auxiliary file +</para> +</listitem> +<listitem> +<para> 400 URI Failure - URI has failed to acquire </para> </listitem> @@ -309,9 +314,9 @@ Authorization is User/Pass </itemizedlist> <para> Only the 6xx series of status codes is sent TO the method. Furthermore the -method may not emit status codes in the 6xx range. The Codes 402 and 403 +method may not emit status codes in the 6xx range. The Codes 351, 402 and 403 require that the method continue reading all other 6xx codes until the proper -602/603 code is received. This means the method must be capable of handling an +600/602/603 code is received. This means the method must be capable of handling an unlimited number of 600 messages. </para> <para> @@ -567,6 +572,20 @@ Size, Last-Modified, Filename, MD5-Hash </listitem> </varlistentry> <varlistentry> +<term>351 Aux Request</term> +<listitem> +<para> +Indicates a request for an auxiliary file to be downloaded by the acquire system +(via another method) and made available for the requesting method. The requestor +will get a <emphasis>600 URI Acquire</emphasis> with the URI it requested and the +filename will either be an existing file if the request was success or if the +acquire failed for some reason the file will not exist. +Fields: URI (of the file causing the need for the auxiliary file), MaximumSize, +Aux-ShortDesc, Aux-Description, Aux-URI +</para> +</listitem> +</varlistentry> +<varlistentry> <term>400 URI Failure</term> <listitem> <para> diff --git a/test/integration/framework b/test/integration/framework index f9d98835c..ff7a7c514 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -2031,9 +2031,11 @@ mkdir() { command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" command mkdir -m 700 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" + command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/auxfiles" touch "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/lock" if [ "$(id -u)" = '0' ]; then chown _apt:$(id -gn) "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" + chown _apt:$(id -gn) "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/auxfiles" fi else command mkdir "$@" diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 616e0234c..a0d7a59d9 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -40,7 +40,8 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta # no package foo testsuccessequal 'Listing...' apt list foo -testequal 'lock +testequal 'auxfiles +lock partial' ls rootdir/var/lib/apt/lists filesize() { diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall index 3571a9f25..5f2109db9 100755 --- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall +++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall @@ -41,7 +41,8 @@ runtests() { testsuccess grep "$1" rootdir/tmp/testfailure.output ensure_n_canary_strings_in_dir "$LISTS" 'ni ni ni' 0 - testequal 'lock + testequal 'auxfiles +lock partial' ls "$LISTS" # and again with pre-existing files with "valid data" which should remain |