From a839efb126af066dc2e58400fb5e25911cb2a9f1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Oct 2017 18:27:54 +0200 Subject: add tag-based control over mirror choices from the list The old implementation used to construct a query string including the release(s) the mirrorlist should be for, but that is hard to deal with as this rules out that partial mirrors are included in the list and it turns out that nobody ended up implementing it on the server side. Controlling this on the client side allows partial mirrors to be included and as a bonus prevents that we tell the mirrorlist server (this rather generic) user information. --- methods/mirror.cc | 137 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 103 insertions(+), 34 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ee703aaae..4b9cd7384 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -40,7 +40,9 @@ static void sortByLength(std::vector &vec) /*{{{*/ /*}}}*/ class MirrorMethod : public aptMethod /*{{{*/ { + std::mt19937 genrng; std::vector sourceslist; + std::unordered_map msgCache; enum MirrorFileState { REQUESTED, @@ -48,52 +50,108 @@ class MirrorMethod : public aptMethod /*{{{*/ AVAILABLE }; struct MirrorInfo + { + std::string uri; + unsigned long priority = std::numeric_limits::max(); + decltype(genrng)::result_type seed = 0; + std::unordered_map> tags; + MirrorInfo(std::string const &u, std::vector &&ptags = {}) : uri(u) + { + for (auto &&tag : ptags) + { + auto const colonfound = tag.find(':'); + if (unlikely(colonfound == std::string::npos)) + continue; + auto name = tag.substr(0, colonfound); + auto value = tag.substr(colonfound + 1); + if (name == "arch") + tags["Architecture"].emplace_back(std::move(value)); + else if (name == "lang") + tags["Language"].emplace_back(std::move(value)); + else if (name == "priority") + priority = std::strtoul(value.c_str(), nullptr, 10); + else if (likely(name.empty() == false)) + { + if (name == "codename" || name == "suite") + tags["Release"].push_back(value); + name[0] = std::toupper(name[0]); + tags[std::move(name)].emplace_back(std::move(value)); + } + } + } + }; + struct MirrorListInfo { MirrorFileState state; std::string baseuri; - std::vector list; + std::vector list; }; - std::unordered_map mirrorfilestate; - unsigned int seedvalue; + std::unordered_map mirrorfilestate; virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; - void RedirectItem(MirrorInfo const &info, FetchItem *const Itm); - bool MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm); + void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message); + bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm); std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm); - void DealWithPendingItems(std::vector const &baseuris, MirrorInfo const &info, FetchItem *const Itm, std::function handler); + void DealWithPendingItems(std::vector const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function handler); public: - MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests) + MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests), genrng(clock()) { SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; - - // we want the file to be random for each different machine, but also - // "stable" on the same machine to avoid issues like picking different - // mirrors in different states for indexes and deb downloads - struct utsname buf; - seedvalue = 1; - if (uname(&buf) == 0) - { - for (size_t i = 0; buf.nodename[i] != '\0'; ++i) - seedvalue = seedvalue * 31 + buf.nodename[i]; - } } }; /*}}}*/ -void MirrorMethod::RedirectItem(MirrorInfo const &info, FetchItem *const Itm) /*{{{*/ +void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/ { + std::unordered_map matchers; + matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture")); + matchers.emplace("Codename", LookupTag(Message, "Target-Codename")); + matchers.emplace("Component", LookupTag(Message, "Target-Component")); + matchers.emplace("Language", LookupTag(Message, "Target-Language")); + matchers.emplace("Release", LookupTag(Message, "Target-Release")); + matchers.emplace("Suite", LookupTag(Message, "Target-Suite")); + matchers.emplace("Type", LookupTag(Message, "Target-Type")); + decltype(info.list) possMirrors; + for (auto const &mirror : info.list) + { + bool failedMatch = false; + for (auto const &m : matchers) + { + if (m.second.empty()) + continue; + auto const tagsetiter = mirror.tags.find(m.first); + if (tagsetiter == mirror.tags.end()) + continue; + auto const tagset = tagsetiter->second; + if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end()) + { + failedMatch = true; + break; + } + } + if (failedMatch) + continue; + possMirrors.push_back(mirror); + } + for (auto &&mirror : possMirrors) + mirror.seed = genrng(); + std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) { + if (a.priority != b.priority) + return a.priority < b.priority; + return a.seed < b.seed; + }); std::string const path = Itm->Uri.substr(info.baseuri.length()); std::string altMirrors; std::unordered_map fields; fields.emplace("URI", Queue->Uri); - for (auto curMirror = info.list.cbegin(); curMirror != info.list.cend(); ++curMirror) + for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror) { - std::string mirror = *curMirror; + std::string mirror = curMirror->uri; if (APT::String::Endswith(mirror, "/") == false) mirror.append("/"); mirror.append(path); - if (curMirror == info.list.cbegin()) + if (curMirror == possMirrors.cbegin()) fields.emplace("New-URI", mirror); else if (altMirrors.empty()) altMirrors.append(mirror); @@ -106,7 +164,7 @@ void MirrorMethod::RedirectItem(MirrorInfo const &info, FetchItem *const Itm) /* } /*}}}*/ void MirrorMethod::DealWithPendingItems(std::vector const &baseuris, /*{{{*/ - MirrorInfo const &info, FetchItem *const Itm, + MirrorListInfo const &info, FetchItem *const Itm, std::function handler) { FetchItem **LastItm = &Itm->Next; @@ -133,7 +191,7 @@ void MirrorMethod::DealWithPendingItems(std::vector const &baseuris delete Itm; } /*}}}*/ -bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm) /*{{{*/ +bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/ { std::vector baseuris; for (auto const &i : mirrorfilestate) @@ -146,17 +204,25 @@ bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm FileFd mirrorlist; if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) { - std::string mirror; - while (mirrorlist.ReadLine(mirror)) + std::string line; + while (mirrorlist.ReadLine(line)) { - if (mirror.empty() || mirror[0] == '#') + if (line.empty() || line[0] == '#') continue; - info.list.push_back(mirror); + auto const tab = line.find('\t'); + if (tab == std::string::npos) + info.list.emplace_back(std::move(line)); + else + { + auto uri = line.substr(0, tab); + auto tagline = line.substr(tab + 1); + std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' '); + auto tags = VectorizeString(tagline, ' '); + tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end()); + info.list.emplace_back(std::move(uri), std::move(tags)); + } } mirrorlist.Close(); - // we reseed each time to avoid "races" with multiple mirror://s - std::mt19937 g(seedvalue); - std::shuffle(info.list.begin(), info.list.end(), g); if (info.list.empty()) { @@ -171,8 +237,9 @@ bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm { info.state = AVAILABLE; DealWithPendingItems(baseuris, info, Itm, [&]() { - RedirectItem(info, Queue); + RedirectItem(info, Queue, msgCache[Queue->Uri]); }); + msgCache.clear(); } } else @@ -253,7 +320,8 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{* auto const state = mirrorfilestate.find(mirrorfileuri); if (state == mirrorfilestate.end()) { - MirrorInfo info; + msgCache[Itm->Uri] = Message; + MirrorListInfo info; info.state = REQUESTED; info.baseuri = mirrorfileuri + '/'; auto const colon = info.baseuri.find(':'); @@ -275,12 +343,13 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{* { case REQUESTED: // lets wait for the requested mirror file + msgCache[Itm->Uri] = Message; return true; case FAILED: Fail("Downloading mirror file failed", false); return true; case AVAILABLE: - RedirectItem(state->second, Itm); + RedirectItem(state->second, Itm, Message); return true; } return false; -- cgit v1.2.3