From 57fa854e4cdb060e87ca265abd5a83364f9fa681 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Oct 2017 18:39:36 +0200 Subject: reimplement and simplify mirror:// method Embedding an entire acquire stack and HTTP logic in the mirror method made it rather heavy weight and fragile. This reimplement goes the other way by doing only the bare minimum in the method itself and instead redirect the actual download of files to their proper methods. The reimplementation drops the (in the real world) unused query-string feature as it isn't really implementable in the new architecture. --- methods/CMakeLists.txt | 8 +- methods/aptmethod.h | 10 + methods/http.cc | 14 +- methods/http_main.cc | 17 -- methods/mirror.cc | 623 ++++++++++++++++++------------------------------- methods/mirror.h | 57 ----- 6 files changed, 251 insertions(+), 478 deletions(-) delete mode 100644 methods/http_main.cc delete mode 100644 methods/mirror.h (limited to 'methods') diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index a25d4b525..cf5ab799d 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories($<$:${SECCOMP_INCLUDE_DIR}>) link_libraries(apt-pkg $<$:${SECCOMP_LIBRARIES}>) -add_library(httplib OBJECT http.cc basehttp.cc) add_library(connectlib OBJECT connect.cc rfc2553emu.cc) add_executable(file file.cc) @@ -10,8 +9,8 @@ add_executable(copy copy.cc) add_executable(store store.cc) add_executable(gpgv gpgv.cc) add_executable(cdrom cdrom.cc) -add_executable(http http_main.cc $ $) -add_executable(mirror mirror.cc $ $) +add_executable(http http.cc basehttp.cc $) +add_executable(mirror mirror.cc) add_executable(ftp ftp.cc $) add_executable(rred rred.cc) add_executable(rsh rsh.cc) @@ -21,14 +20,13 @@ target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR}) # Additional libraries to link against for networked stuff target_link_libraries(http ${GNUTLS_LIBRARIES}) -target_link_libraries(mirror ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES}) target_link_libraries(ftp ${GNUTLS_LIBRARIES}) # Install the library install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) -add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store) +add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+http mirror+https mirror+file) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh) diff --git a/methods/aptmethod.h b/methods/aptmethod.h index 88d325cba..331411571 100644 --- a/methods/aptmethod.h +++ b/methods/aptmethod.h @@ -448,6 +448,16 @@ protected: return true; } + // This is a copy of #pkgAcqMethod::Dequeue which is private & hidden + void Dequeue() + { + FetchItem const *const Tmp = Queue; + Queue = Queue->Next; + if (Tmp == QueueBack) + QueueBack = Queue; + delete Tmp; + } + aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) : pkgAcqMethod(Ver, Flags), Binary(Binary), SeccompFlags(0), methodNames({Binary}) { diff --git a/methods/http.cc b/methods/http.cc index 2d23b1646..5d286bcb4 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -1034,7 +1035,7 @@ BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &R return FILE_IS_OPEN; } /*}}}*/ -HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ +HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig) /*{{{*/ { SeccompFlags = aptMethod::BASE | aptMethod::NETWORK; @@ -1051,3 +1052,14 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2 } } /*}}}*/ + +int main(int, const char *argv[]) +{ + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") + Binary.append("+http"); + return HttpMethod(std::move(Binary)).Loop(); +} diff --git a/methods/http_main.cc b/methods/http_main.cc deleted file mode 100644 index 792b5e22f..000000000 --- a/methods/http_main.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include -#include -#include - -#include "http.h" - -int main(int, const char *argv[]) -{ - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - std::string Binary = flNotDir(argv[0]); - if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") - Binary.append("+http"); - return HttpMethod(std::move(Binary)).Loop(); -} diff --git a/methods/mirror.cc b/methods/mirror.cc index b551802e4..ad8867836 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -1,18 +1,17 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ /* ###################################################################### - Mirror Acquire Method - This is the Mirror acquire method for APT. - + Mirror URI – This method helps avoiding hardcoding of mirrors in the + sources.lists by looking up a list of mirrors first to which the + following requests are redirected. + ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ #include -#include -#include -#include +#include "aptmethod.h" #include #include #include @@ -20,447 +19,275 @@ #include #include -#include -#include -#include +#include +#include +#include +#include -#include -#include -#include -#include -#include -#include #include -#include - -using namespace std; -#include - -#include "http.h" -#include "mirror.h" #include /*}}}*/ -/* Done: - * - works with http (only!) - * - always picks the first mirror from the list - * - call out to problem reporting script - * - supports "deb mirror://host/path/to/mirror-list/// dist component" - * - uses pkgAcqMethod::FailReason() to have a string representation - * of the failure that is also send to LP - * - * TODO: - * - deal with running as non-root because we can't write to the lists - dir then -> use the cached mirror file - * - better method to download than having a pkgAcquire interface here - * and better error handling there! - * - support more than http - * - testing :) - */ - -MirrorMethod::MirrorMethod() - : HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false) +static void sortByLength(std::vector &vec) /*{{{*/ { -} - -// HttpMethod::Configuration - Handle a configuration message /*{{{*/ -// --------------------------------------------------------------------- -/* We stash the desired pipeline depth */ -bool MirrorMethod::Configuration(string Message) -{ - if (HttpMethod::Configuration(Message) == false) - return false; - Debug = DebugEnabled(); - - return true; + // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected + // by checking for the longest matches first + std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) { + return a.length() > b.length(); + }); } /*}}}*/ - -// clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string Dir) +class MirrorMethod : public aptMethod /*{{{*/ { - vector::const_iterator I; - - if(Debug) - clog << "MirrorMethod::Clean(): " << Dir << endl; - - if(Dir == "/") - return _error->Error("will not clean: '/'"); - - // read sources.list - pkgSourceList list; - list.ReadMainList(); - - int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); - if (dirfd == -1) - return _error->Errno("open",_("Unable to read %s"), Dir.c_str()); - DIR * const D = fdopendir(dirfd); - if (D == nullptr) - return _error->Errno("fdopendir",_("Unable to read %s"),Dir.c_str()); - - for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + std::vector sourceslist; + enum MirrorFileState { - // Skip some files.. - if (strcmp(Dir->d_name,"lock") == 0 || - strcmp(Dir->d_name,"partial") == 0 || - strcmp(Dir->d_name,"lost+found") == 0 || - strcmp(Dir->d_name,".") == 0 || - strcmp(Dir->d_name,"..") == 0) - continue; - - // see if we have that uri - for(I=list.begin(); I != list.end(); ++I) + REQUESTED, + FAILED, + AVAILABLE + }; + struct MirrorInfo + { + MirrorFileState state; + std::string baseuri; + std::vector list; + }; + std::unordered_map mirrorfilestate; + unsigned int seedvalue; + + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; + + void RedirectItem(MirrorInfo const &info, FetchItem *const Itm); + bool MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm); + std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm); + void DealWithPendingItems(std::vector const &baseuris, MirrorInfo const &info, FetchItem *const Itm, std::function handler); + + public: + MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig) + { + SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; + + // we want the file to be random for each different machine, but also + // "stable" on the same machine to avoid issues like picking different + // mirrors in different states for indexes and deb downloads + struct utsname buf; + seedvalue = 1; + if (uname(&buf) == 0) { - string uri = (*I)->GetURI(); - if(uri.compare(0, strlen("mirror://"), "mirror://") != 0) - continue; - string BaseUri = uri.substr(0,uri.size()-1); - if (URItoFileName(BaseUri) == Dir->d_name) - break; + for (size_t i = 0; buf.nodename[i] != '\0'; ++i) + seedvalue = seedvalue * 31 + buf.nodename[i]; } - // nothing found, nuke it - if (I == list.end()) - RemoveFileAt("mirror", dirfd, Dir->d_name); } - closedir(D); - return true; -} - - -bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/) +}; + /*}}}*/ +void MirrorMethod::RedirectItem(MirrorInfo const &info, FetchItem *const Itm) /*{{{*/ { - // not that great to use pkgAcquire here, but we do not have - // any other way right now - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - -#if 0 // no need for this, the getArchitectures() will also include the main - // arch - // append main architecture - fetch += "?arch=" + _config->Find("Apt::Architecture"); -#endif - - // append all architectures - std::vector vec = APT::Configuration::getArchitectures(); - for (std::vector::const_iterator I = vec.begin(); - I != vec.end(); ++I) - if (I == vec.begin()) - fetch += "?arch=" + (*I); + std::string const path = Itm->Uri.substr(info.baseuri.length()); + std::string altMirrors; + std::unordered_map fields; + fields.emplace("URI", Queue->Uri); + for (auto curMirror = info.list.cbegin(); curMirror != info.list.cend(); ++curMirror) + { + std::string mirror = *curMirror; + if (APT::String::Endswith(mirror, "/") == false) + mirror.append("/"); + mirror.append(path); + if (curMirror == info.list.cbegin()) + fields.emplace("New-URI", mirror); + else if (altMirrors.empty()) + altMirrors.append(mirror); else - fetch += "&arch=" + (*I); - - // append the dist as a query string - if (Dist != "") - fetch += "&dist=" + Dist; - - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'" - << " to " << MirrorFile << endl; - - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) { - DownloadedMirrorFile = true; - chmod(MirrorFile.c_str(), 0644); - } - Fetcher.Shutdown(); - - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl; - - return res; -} - -// Randomizes the lines in the mirror file, this is used so that -// we spread the load on the mirrors evenly -bool MirrorMethod::RandomizeMirrorFile(string mirror_file) -{ - vector content; - string line; - - if (!FileExists(mirror_file)) - return false; - - // read - ifstream in(mirror_file.c_str()); - while ( !in.eof() ) { - getline(in, line); - content.push_back(line); + altMirrors.append("\n").append(mirror); } - - // we want the file to be random for each different machine, but also - // "stable" on the same machine. this is to avoid running into out-of-sync - // issues (i.e. Release/Release.gpg different on each mirror) - struct utsname buf; - int seed=1; - if(uname(&buf) == 0) { - for(int i=0,seed=1; buf.nodename[i] != 0; ++i) { - seed = seed * 31 + buf.nodename[i]; - } - } - srand( seed ); - random_shuffle(content.begin(), content.end()); - - // write - ofstream out(mirror_file.c_str()); - while ( !content.empty()) { - line = content.back(); - content.pop_back(); - out << line << "\n"; - } - - return true; + fields.emplace("Alternate-URIs", altMirrors); + SendMessage("103 Redirect", std::move(fields)); + Dequeue(); } - -/* convert a the Queue->Uri back to the mirror base uri and look - * at all mirrors we have for this, this is needed as queue->uri - * may point to different mirrors (if TryNextMirror() was run) - */ -void MirrorMethod::CurrentQueueUriToMirror() + /*}}}*/ +void MirrorMethod::DealWithPendingItems(std::vector const &baseuris, /*{{{*/ + MirrorInfo const &info, FetchItem *const Itm, + std::function handler) { - // already in mirror:// style so nothing to do - if(Queue->Uri.find("mirror://") == 0) - return; - - // find current mirror and select next one - for (vector::const_iterator mirror = AllMirrors.begin(); - mirror != AllMirrors.end(); ++mirror) + FetchItem **LastItm = &Itm->Next; + while (*LastItm != nullptr) + LastItm = &((*LastItm)->Next); + while (Queue != Itm) { - if (Queue->Uri.find(*mirror) == 0) + if (APT::String::Startswith(Queue->Uri, info.baseuri) == false || + std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); })) + { + // move the item behind the aux file not related to it + *LastItm = Queue; + Queue = QueueBack = Queue->Next; + (*LastItm)->Next = nullptr; + LastItm = &((*LastItm)->Next); + } + else { - Queue->Uri.replace(0, mirror->length(), BaseUri); - return; + handler(); } } - _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri.c_str(), BaseUri.c_str()); + // now remove out trigger + QueueBack = Queue = Queue->Next; + delete Itm; } - -bool MirrorMethod::TryNextMirror() + /*}}}*/ +bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm) /*{{{*/ { - // find current mirror and select next one - for (vector::const_iterator mirror = AllMirrors.begin(); - mirror != AllMirrors.end(); ++mirror) + std::vector baseuris; + for (auto const &i : mirrorfilestate) + if (info.baseuri.length() < i.second.baseuri.length() && + i.second.state == REQUESTED && + APT::String::Startswith(i.second.baseuri, info.baseuri)) + baseuris.push_back(i.second.baseuri); + sortByLength(baseuris); + + FileFd mirrorlist; + if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) { - if (Queue->Uri.find(*mirror) != 0) - continue; - - vector::const_iterator nextmirror = mirror + 1; - if (nextmirror == AllMirrors.end()) - break; - Queue->Uri.replace(0, mirror->length(), *nextmirror); - if (Debug) - clog << "TryNextMirror: " << Queue->Uri << endl; - - // inform parent - UsedMirror = *nextmirror; - Log("Switching mirror"); - return true; - } - - if (Debug) - clog << "TryNextMirror could not find another mirror to try" << endl; - - return false; -} + std::string mirror; + while (mirrorlist.ReadLine(mirror)) + { + if (mirror.empty() || mirror[0] == '#') + continue; + info.list.push_back(mirror); + } + mirrorlist.Close(); + // we reseed each time to avoid "races" with multiple mirror://s + std::mt19937 g(seedvalue); + std::shuffle(info.list.begin(), info.list.end(), g); -bool MirrorMethod::InitMirrors() -{ - // if we do not have a MirrorFile, fallback - if(!FileExists(MirrorFile)) - { - // FIXME: fallback to a default mirror here instead - // and provide a config option to define that default - return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + if (info.list.empty()) + { + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); + } + else + { + info.state = AVAILABLE; + DealWithPendingItems(baseuris, info, Itm, [&]() { + RedirectItem(info, Queue); + }); + } } - - if (access(MirrorFile.c_str(), R_OK) != 0) - { - // FIXME: fallback to a default mirror here instead - // and provide a config option to define that default - return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str()); - } - - // FIXME: make the mirror selection more clever, do not - // just use the first one! - // BUT: we can not make this random, the mirror has to be - // stable across session, because otherwise we can - // get into sync issues (got indexfiles from mirror A, - // but packages from mirror B - one might be out of date etc) - ifstream in(MirrorFile.c_str()); - string s; - while (!in.eof()) + else { - getline(in, s); - - // ignore lines that start with # - if (s.find("#") == 0) - continue; - // ignore empty lines - if (s.size() == 0) - continue; - // ignore non http lines - if (s.compare(0, strlen("http://"), "http://") != 0) - continue; - - AllMirrors.push_back(s); - } - if (AllMirrors.empty()) { - return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str()); + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); } - Mirror = AllMirrors[0]; - UsedMirror = Mirror; return true; } - -string MirrorMethod::GetMirrorFileName(string mirror_uri_str) + /*}}}*/ +std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/ { - /* - - a mirror_uri_str looks like this: - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg - - - the matching source.list entry - deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main - - - we actually want to go after: - http://people.ubuntu.com/~mvo/apt/mirror/mirrors - - And we need to save the BaseUri for later: - - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors - - FIXME: what if we have two similar prefixes? - mirror://people.ubuntu.com/~mvo/mirror - mirror://people.ubuntu.com/~mvo/mirror2 - then mirror_uri_str looks like: - mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg - mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg - we search sources.list and find: - mirror://people.ubuntu.com/~mvo/apt/mirror - in both cases! So we need to apply some domain knowledge here :( and - check for /dists/ or /Release.gpg as suffixes - */ - string name; - if(Debug) - std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; - - // read sources.list and find match - vector::const_iterator I; - pkgSourceList list; - list.ReadMainList(); - for(I=list.begin(); I != list.end(); ++I) + if (APT::String::Startswith(Itm->Uri, Binary)) + { + std::string const repouri = LookupTag(Message, "Target-Repo-Uri"); + if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend()) + sourceslist.push_back(repouri); + } + if (sourceslist.empty()) { - string uristr = (*I)->GetURI(); - if(Debug) - std::cerr << "Checking: " << uristr << std::endl; - if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) - continue; - // find matching uri in sources.list - if(mirror_uri_str.substr(0,uristr.size()) == uristr) + // read sources.list and find the matching base uri + pkgSourceList sl; + if (sl.ReadMainList() == false) { - if(Debug) - std::cerr << "found BaseURI: " << uristr << std::endl; - BaseUri = uristr.substr(0,uristr.size()-1); - Dist = (*I)->GetDist(); + _error->Error(_("The list of sources could not be read.")); + return ""; } + std::string const needle = Binary + ":"; + for (auto const &SL : sl) + { + std::string uristr = SL->GetURI(); + if (APT::String::Startswith(uristr, needle)) + sourceslist.push_back(uristr); + } + sortByLength(sourceslist); } - // get new file - name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); - - if(Debug) + for (auto uristr : sourceslist) { - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << name << endl; + if (APT::String::Startswith(Itm->Uri, uristr)) + { + uristr.erase(uristr.length() - 1); // remove the ending '/' + auto const colon = uristr.find(':'); + if (unlikely(colon == std::string::npos)) + continue; + auto const plus = uristr.find("+"); + if (plus < colon) + return uristr.substr(plus + 1); + else + { + uristr.replace(0, strlen("mirror"), "http"); + return uristr; + } + } } - return name; + return ""; } - -// MirrorMethod::Fetch - Fetch an item /*{{{*/ -// --------------------------------------------------------------------- -/* This adds an item to the pipeline. We keep the pipeline at a fixed - depth. */ -bool MirrorMethod::Fetch(FetchItem *Itm) + /*}}}*/ +bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/ { - if(Debug) - clog << "MirrorMethod::Fetch()" << endl; + auto mirrorinfo = mirrorfilestate.find(Itm->Uri); + if (mirrorinfo != mirrorfilestate.end()) + return MirrorListFileRecieved(mirrorinfo->second, Itm); - // the http method uses Fetch(0) as a way to update the pipeline, - // just let it do its work in this case - Fetch() with a valid - // Itm will always run before the first Fetch(0) - if(Itm == NULL) - return HttpMethod::Fetch(Itm); - - // if we don't have the name of the mirror file on disk yet, - // calculate it now (can be derived from the uri) - if(MirrorFile.empty()) - MirrorFile = GetMirrorFileName(Itm->Uri); - - // download mirror file once (if we are after index files) - if(Itm->IndexFile && !DownloadedMirrorFile) + std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm); + if (mirrorfileuri.empty()) { - Clean(_config->FindDir("Dir::State::mirrors")); - if (DownloadMirrorFile(Itm->Uri)) - RandomizeMirrorFile(MirrorFile); + _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str()); + return false; } + if (DebugEnabled()) + std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl; - if(AllMirrors.empty()) { - if(!InitMirrors()) { - // no valid mirror selected, something went wrong downloading - // from the master mirror site most likely and there is - // no old mirror file availalbe + // have we requested this mirror file already? + auto const state = mirrorfilestate.find(mirrorfileuri); + if (state == mirrorfilestate.end()) + { + MirrorInfo info; + info.state = REQUESTED; + info.baseuri = mirrorfileuri + '/'; + auto const colon = info.baseuri.find(':'); + if (unlikely(colon == std::string::npos)) return false; - } + info.baseuri.replace(0, colon, Binary); + mirrorfilestate[mirrorfileuri] = info; + std::unordered_map fields; + fields.emplace("URI", Itm->Uri); + fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone + fields.emplace("Aux-ShortDesc", "Mirrorlist"); + fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist"); + fields.emplace("Aux-Uri", mirrorfileuri); + SendMessage("351 Aux Request", std::move(fields)); + return true; } - if(Itm->Uri.find("mirror://") != string::npos) - Itm->Uri.replace(0,BaseUri.size(), Mirror); - - if(Debug) - clog << "Fetch: " << Itm->Uri << endl << endl; - - // now run the real fetcher - return HttpMethod::Fetch(Itm); -} - -void MirrorMethod::Fail(string Err,bool Transient) -{ - // FIXME: TryNextMirror is not ideal for indexfile as we may - // run into auth issues - - if (Debug) - clog << "Failure to get " << Queue->Uri << endl; - - // try the next mirror on fail (if its not a expected failure, - // e.g. translations are ok to ignore) - if (!Queue->FailIgnore && TryNextMirror()) - return; - - // all mirrors failed, so bail out - string s; - strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); - SetIP(s); - - CurrentQueueUriToMirror(); - pkgAcqMethod::Fail(Err, Transient); -} - -void MirrorMethod::URIStart(FetchResult &Res) -{ - CurrentQueueUriToMirror(); - pkgAcqMethod::URIStart(Res); -} - -void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) -{ - CurrentQueueUriToMirror(); - pkgAcqMethod::URIDone(Res, Alt); + switch (state->second.state) + { + case REQUESTED: + // lets wait for the requested mirror file + return true; + case FAILED: + Fail("Downloading mirror file failed", false); + return true; + case AVAILABLE: + RedirectItem(state->second, Itm); + return true; + } + return false; } + /*}}}*/ - -int main() +int main(int, const char *argv[]) { - return MirrorMethod().Loop(); + return MirrorMethod(flNotDir(argv[0])).Run(); } - - diff --git a/methods/mirror.h b/methods/mirror.h deleted file mode 100644 index 6ebe08e6b..000000000 --- a/methods/mirror.h +++ /dev/null @@ -1,57 +0,0 @@ -// -*- mode: cpp; mode: fold -*- -// Description /*{{{*/ -/* ###################################################################### - - MIRROR Acquire Method - This is the MIRROR acquire method for APT. - - ##################################################################### */ - /*}}}*/ - -#ifndef APT_MIRROR_H -#define APT_MIRROR_H - -#include -#include -#include - -using std::cout; -using std::cerr; -using std::endl; - -#include "http.h" - -class MirrorMethod : public HttpMethod -{ - FetchResult Res; - // we simply transform between BaseUri and Mirror - std::string BaseUri; // the original mirror://... url - std::string Mirror; // the selected mirror uri (http://...) - std::vector AllMirrors; // all available mirrors - std::string MirrorFile; // the file that contains the list of mirrors - bool DownloadedMirrorFile; // already downloaded this session - std::string Dist; // the target distrubtion (e.g. sid, oneiric) - - bool Debug; - - protected: - bool DownloadMirrorFile(std::string uri); - bool RandomizeMirrorFile(std::string file); - std::string GetMirrorFileName(std::string uri); - bool InitMirrors(); - bool TryNextMirror(); - void CurrentQueueUriToMirror(); - bool Clean(std::string dir); - - // we need to overwrite those to transform the url back - virtual void Fail(std::string Why, bool Transient = false) APT_OVERRIDE; - virtual void URIStart(FetchResult &Res) APT_OVERRIDE; - virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0) APT_OVERRIDE; - virtual bool Configuration(std::string Message) APT_OVERRIDE; - - public: - MirrorMethod(); - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; -}; - - -#endif -- cgit v1.2.3 From 04ab37fecaf286f724bef2e0969d2b67ab5ac1b1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 28 Oct 2017 00:01:27 +0200 Subject: require methods to request AuxRequest capability at startup Allowing a method to request work from other methods is a powerful capability which could be misused or exploited, so to slightly limited the surface let method opt-in into this capability on startup. --- methods/mirror.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ad8867836..ee703aaae 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -64,7 +64,7 @@ class MirrorMethod : public aptMethod /*{{{*/ void DealWithPendingItems(std::vector const &baseuris, MirrorInfo const &info, FetchItem *const Itm, std::function handler); public: - MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig) + MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests) { SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; -- cgit v1.2.3 From a839efb126af066dc2e58400fb5e25911cb2a9f1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Oct 2017 18:27:54 +0200 Subject: add tag-based control over mirror choices from the list The old implementation used to construct a query string including the release(s) the mirrorlist should be for, but that is hard to deal with as this rules out that partial mirrors are included in the list and it turns out that nobody ended up implementing it on the server side. Controlling this on the client side allows partial mirrors to be included and as a bonus prevents that we tell the mirrorlist server (this rather generic) user information. --- methods/mirror.cc | 137 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 103 insertions(+), 34 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ee703aaae..4b9cd7384 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -40,7 +40,9 @@ static void sortByLength(std::vector &vec) /*{{{*/ /*}}}*/ class MirrorMethod : public aptMethod /*{{{*/ { + std::mt19937 genrng; std::vector sourceslist; + std::unordered_map msgCache; enum MirrorFileState { REQUESTED, @@ -48,52 +50,108 @@ class MirrorMethod : public aptMethod /*{{{*/ AVAILABLE }; struct MirrorInfo + { + std::string uri; + unsigned long priority = std::numeric_limits::max(); + decltype(genrng)::result_type seed = 0; + std::unordered_map> tags; + MirrorInfo(std::string const &u, std::vector &&ptags = {}) : uri(u) + { + for (auto &&tag : ptags) + { + auto const colonfound = tag.find(':'); + if (unlikely(colonfound == std::string::npos)) + continue; + auto name = tag.substr(0, colonfound); + auto value = tag.substr(colonfound + 1); + if (name == "arch") + tags["Architecture"].emplace_back(std::move(value)); + else if (name == "lang") + tags["Language"].emplace_back(std::move(value)); + else if (name == "priority") + priority = std::strtoul(value.c_str(), nullptr, 10); + else if (likely(name.empty() == false)) + { + if (name == "codename" || name == "suite") + tags["Release"].push_back(value); + name[0] = std::toupper(name[0]); + tags[std::move(name)].emplace_back(std::move(value)); + } + } + } + }; + struct MirrorListInfo { MirrorFileState state; std::string baseuri; - std::vector list; + std::vector list; }; - std::unordered_map mirrorfilestate; - unsigned int seedvalue; + std::unordered_map mirrorfilestate; virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; - void RedirectItem(MirrorInfo const &info, FetchItem *const Itm); - bool MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm); + void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message); + bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm); std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm); - void DealWithPendingItems(std::vector const &baseuris, MirrorInfo const &info, FetchItem *const Itm, std::function handler); + void DealWithPendingItems(std::vector const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function handler); public: - MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests) + MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests), genrng(clock()) { SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; - - // we want the file to be random for each different machine, but also - // "stable" on the same machine to avoid issues like picking different - // mirrors in different states for indexes and deb downloads - struct utsname buf; - seedvalue = 1; - if (uname(&buf) == 0) - { - for (size_t i = 0; buf.nodename[i] != '\0'; ++i) - seedvalue = seedvalue * 31 + buf.nodename[i]; - } } }; /*}}}*/ -void MirrorMethod::RedirectItem(MirrorInfo const &info, FetchItem *const Itm) /*{{{*/ +void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/ { + std::unordered_map matchers; + matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture")); + matchers.emplace("Codename", LookupTag(Message, "Target-Codename")); + matchers.emplace("Component", LookupTag(Message, "Target-Component")); + matchers.emplace("Language", LookupTag(Message, "Target-Language")); + matchers.emplace("Release", LookupTag(Message, "Target-Release")); + matchers.emplace("Suite", LookupTag(Message, "Target-Suite")); + matchers.emplace("Type", LookupTag(Message, "Target-Type")); + decltype(info.list) possMirrors; + for (auto const &mirror : info.list) + { + bool failedMatch = false; + for (auto const &m : matchers) + { + if (m.second.empty()) + continue; + auto const tagsetiter = mirror.tags.find(m.first); + if (tagsetiter == mirror.tags.end()) + continue; + auto const tagset = tagsetiter->second; + if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end()) + { + failedMatch = true; + break; + } + } + if (failedMatch) + continue; + possMirrors.push_back(mirror); + } + for (auto &&mirror : possMirrors) + mirror.seed = genrng(); + std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) { + if (a.priority != b.priority) + return a.priority < b.priority; + return a.seed < b.seed; + }); std::string const path = Itm->Uri.substr(info.baseuri.length()); std::string altMirrors; std::unordered_map fields; fields.emplace("URI", Queue->Uri); - for (auto curMirror = info.list.cbegin(); curMirror != info.list.cend(); ++curMirror) + for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror) { - std::string mirror = *curMirror; + std::string mirror = curMirror->uri; if (APT::String::Endswith(mirror, "/") == false) mirror.append("/"); mirror.append(path); - if (curMirror == info.list.cbegin()) + if (curMirror == possMirrors.cbegin()) fields.emplace("New-URI", mirror); else if (altMirrors.empty()) altMirrors.append(mirror); @@ -106,7 +164,7 @@ void MirrorMethod::RedirectItem(MirrorInfo const &info, FetchItem *const Itm) /* } /*}}}*/ void MirrorMethod::DealWithPendingItems(std::vector const &baseuris, /*{{{*/ - MirrorInfo const &info, FetchItem *const Itm, + MirrorListInfo const &info, FetchItem *const Itm, std::function handler) { FetchItem **LastItm = &Itm->Next; @@ -133,7 +191,7 @@ void MirrorMethod::DealWithPendingItems(std::vector const &baseuris delete Itm; } /*}}}*/ -bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm) /*{{{*/ +bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/ { std::vector baseuris; for (auto const &i : mirrorfilestate) @@ -146,17 +204,25 @@ bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm FileFd mirrorlist; if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) { - std::string mirror; - while (mirrorlist.ReadLine(mirror)) + std::string line; + while (mirrorlist.ReadLine(line)) { - if (mirror.empty() || mirror[0] == '#') + if (line.empty() || line[0] == '#') continue; - info.list.push_back(mirror); + auto const tab = line.find('\t'); + if (tab == std::string::npos) + info.list.emplace_back(std::move(line)); + else + { + auto uri = line.substr(0, tab); + auto tagline = line.substr(tab + 1); + std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' '); + auto tags = VectorizeString(tagline, ' '); + tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end()); + info.list.emplace_back(std::move(uri), std::move(tags)); + } } mirrorlist.Close(); - // we reseed each time to avoid "races" with multiple mirror://s - std::mt19937 g(seedvalue); - std::shuffle(info.list.begin(), info.list.end(), g); if (info.list.empty()) { @@ -171,8 +237,9 @@ bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm { info.state = AVAILABLE; DealWithPendingItems(baseuris, info, Itm, [&]() { - RedirectItem(info, Queue); + RedirectItem(info, Queue, msgCache[Queue->Uri]); }); + msgCache.clear(); } } else @@ -253,7 +320,8 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{* auto const state = mirrorfilestate.find(mirrorfileuri); if (state == mirrorfilestate.end()) { - MirrorInfo info; + msgCache[Itm->Uri] = Message; + MirrorListInfo info; info.state = REQUESTED; info.baseuri = mirrorfileuri + '/'; auto const colon = info.baseuri.find(':'); @@ -275,12 +343,13 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{* { case REQUESTED: // lets wait for the requested mirror file + msgCache[Itm->Uri] = Message; return true; case FAILED: Fail("Downloading mirror file failed", false); return true; case AVAILABLE: - RedirectItem(state->second, Itm); + RedirectItem(state->second, Itm, Message); return true; } return false; -- cgit v1.2.3 From 4df5483994d510290677abab5720445f71babe65 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Nov 2017 12:38:21 +0100 Subject: non-local mirrorlists shouldn't redirect to local A mirror list we get from an non-local source like http shouldn't be able to include e.g. file sources and even with other online sources we need to be careful: They also shouldn't include prefixed methods like 'tor+http'. So apply magic based on how the method is called: mirror+file will be allowed to redirect to any source while tor+mirror+file allows all, but sends them to their tor+ variant. --- methods/CMakeLists.txt | 2 +- methods/mirror.cc | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index cf5ab799d..c4a32b4f5 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(ftp ${GNUTLS_LIBRARIES}) install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) -add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+http mirror+https mirror+file) +add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+ftp mirror+http mirror+https mirror+file mirror+copy) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh) diff --git a/methods/mirror.cc b/methods/mirror.cc index 4b9cd7384..add9f0875 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -28,6 +28,7 @@ #include /*}}}*/ +constexpr char const *const disallowLocal[] = {"ftp", "http", "https"}; static void sortByLength(std::vector &vec) /*{{{*/ { @@ -204,17 +205,47 @@ bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const FileFd mirrorlist; if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) { + auto const accessColon = info.baseuri.find(':'); + auto access = info.baseuri.substr(0, accessColon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + auto const plus = info.baseuri.find('+'); + prefixAccess = info.baseuri.substr(0, plus); + access.erase(0, plus + 1); + } + std::vector limitAccess; + // If the mirror file comes from an online source, allow only other online + // sources, not e.g. file:///. If the mirrorlist comes from there we can assume + // the admin knows what (s)he is doing through and not limit the options. + if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal), + [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) || + access == "mirror") + { + for (auto const &a : disallowLocal) + limitAccess.emplace_back(a); + } std::string line; while (mirrorlist.ReadLine(line)) { if (line.empty() || line[0] == '#') continue; + auto const access = line.substr(0, line.find(':')); + if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end()) + continue; auto const tab = line.find('\t'); if (tab == std::string::npos) - info.list.emplace_back(std::move(line)); + { + if (prefixAccess.empty()) + info.list.emplace_back(std::move(line)); + else + info.list.emplace_back(prefixAccess + '+' + line); + } else { auto uri = line.substr(0, tab); + if (prefixAccess.empty() == false) + uri = prefixAccess + '+' + uri; auto tagline = line.substr(tab + 1); std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' '); auto tags = VectorizeString(tagline, ' '); @@ -290,7 +321,20 @@ std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem continue; auto const plus = uristr.find("+"); if (plus < colon) - return uristr.substr(plus + 1); + { + // started as tor+mirror+http we want to get the file via tor+http + auto access = uristr.substr(0, colon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + prefixAccess = uristr.substr(0, plus); + access.erase(0, plus + 1); + uristr.erase(plus, strlen("mirror") + 1); + return uristr; + } + else + return uristr.substr(plus + 1); + } else { uristr.replace(0, strlen("mirror"), "http"); -- cgit v1.2.3