summaryrefslogtreecommitdiff
path: root/methods/mirror.cc
diff options
context:
space:
mode:
authorJulian Klode <jak@debian.org>2018-01-03 21:05:16 +0000
committerJulian Klode <jak@debian.org>2018-01-03 21:05:16 +0000
commit6ee1b762322e725d50ea53e2cf16f8450e23c578 (patch)
tree92e1f41ec370d6064c9236fb28644cfc488a59a7 /methods/mirror.cc
parent5b197e9de5376e191018562309e2d42123c27a1d (diff)
parente4ed47f10844cf7ad933f7a9b64583869592f139 (diff)
Merge branch 'feature/amtshilfe' into 'master'
reimplement mirror method See merge request apt-team/apt!1
Diffstat (limited to 'methods/mirror.cc')
-rw-r--r--methods/mirror.cc728
1 files changed, 334 insertions, 394 deletions
diff --git a/methods/mirror.cc b/methods/mirror.cc
index b551802e4..add9f0875 100644
--- a/methods/mirror.cc
+++ b/methods/mirror.cc
@@ -1,18 +1,17 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
-// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
/* ######################################################################
- Mirror Acquire Method - This is the Mirror acquire method for APT.
-
+ Mirror URI – This method helps avoiding hardcoding of mirrors in the
+ sources.lists by looking up a list of mirrors first to which the
+ following requests are redirected.
+
##################################################################### */
/*}}}*/
// Include Files /*{{{*/
#include <config.h>
-#include <apt-pkg/acquire-item.h>
-#include <apt-pkg/acquire.h>
-#include <apt-pkg/aptconfiguration.h>
+#include "aptmethod.h"
#include <apt-pkg/configuration.h>
#include <apt-pkg/error.h>
#include <apt-pkg/fileutl.h>
@@ -20,447 +19,388 @@
#include <apt-pkg/sourcelist.h>
#include <apt-pkg/strutl.h>
-#include <algorithm>
-#include <fstream>
-#include <iostream>
+#include <functional>
+#include <random>
+#include <string>
+#include <unordered_map>
-#include <dirent.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
#include <sys/utsname.h>
-#include <unistd.h>
-
-using namespace std;
-
-#include <sstream>
-#include "http.h"
-#include "mirror.h"
#include <apti18n.h>
/*}}}*/
+constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
-/* Done:
- * - works with http (only!)
- * - always picks the first mirror from the list
- * - call out to problem reporting script
- * - supports "deb mirror://host/path/to/mirror-list/// dist component"
- * - uses pkgAcqMethod::FailReason() to have a string representation
- * of the failure that is also send to LP
- *
- * TODO:
- * - deal with running as non-root because we can't write to the lists
- dir then -> use the cached mirror file
- * - better method to download than having a pkgAcquire interface here
- * and better error handling there!
- * - support more than http
- * - testing :)
- */
-
-MirrorMethod::MirrorMethod()
- : HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false)
+static void sortByLength(std::vector<std::string> &vec) /*{{{*/
{
-}
-
-// HttpMethod::Configuration - Handle a configuration message /*{{{*/
-// ---------------------------------------------------------------------
-/* We stash the desired pipeline depth */
-bool MirrorMethod::Configuration(string Message)
-{
- if (HttpMethod::Configuration(Message) == false)
- return false;
- Debug = DebugEnabled();
-
- return true;
+ // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
+ // by checking for the longest matches first
+ std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
+ return a.length() > b.length();
+ });
}
/*}}}*/
-
-// clean the mirrors dir based on ttl information
-bool MirrorMethod::Clean(string Dir)
+class MirrorMethod : public aptMethod /*{{{*/
{
- vector<metaIndex *>::const_iterator I;
-
- if(Debug)
- clog << "MirrorMethod::Clean(): " << Dir << endl;
-
- if(Dir == "/")
- return _error->Error("will not clean: '/'");
-
- // read sources.list
- pkgSourceList list;
- list.ReadMainList();
-
- int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
- if (dirfd == -1)
- return _error->Errno("open",_("Unable to read %s"), Dir.c_str());
- DIR * const D = fdopendir(dirfd);
- if (D == nullptr)
- return _error->Errno("fdopendir",_("Unable to read %s"),Dir.c_str());
-
- for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
+ std::mt19937 genrng;
+ std::vector<std::string> sourceslist;
+ std::unordered_map<std::string, std::string> msgCache;
+ enum MirrorFileState
{
- // Skip some files..
- if (strcmp(Dir->d_name,"lock") == 0 ||
- strcmp(Dir->d_name,"partial") == 0 ||
- strcmp(Dir->d_name,"lost+found") == 0 ||
- strcmp(Dir->d_name,".") == 0 ||
- strcmp(Dir->d_name,"..") == 0)
- continue;
-
- // see if we have that uri
- for(I=list.begin(); I != list.end(); ++I)
+ REQUESTED,
+ FAILED,
+ AVAILABLE
+ };
+ struct MirrorInfo
+ {
+ std::string uri;
+ unsigned long priority = std::numeric_limits<decltype(priority)>::max();
+ decltype(genrng)::result_type seed = 0;
+ std::unordered_map<std::string, std::vector<std::string>> tags;
+ MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
{
- string uri = (*I)->GetURI();
- if(uri.compare(0, strlen("mirror://"), "mirror://") != 0)
- continue;
- string BaseUri = uri.substr(0,uri.size()-1);
- if (URItoFileName(BaseUri) == Dir->d_name)
- break;
+ for (auto &&tag : ptags)
+ {
+ auto const colonfound = tag.find(':');
+ if (unlikely(colonfound == std::string::npos))
+ continue;
+ auto name = tag.substr(0, colonfound);
+ auto value = tag.substr(colonfound + 1);
+ if (name == "arch")
+ tags["Architecture"].emplace_back(std::move(value));
+ else if (name == "lang")
+ tags["Language"].emplace_back(std::move(value));
+ else if (name == "priority")
+ priority = std::strtoul(value.c_str(), nullptr, 10);
+ else if (likely(name.empty() == false))
+ {
+ if (name == "codename" || name == "suite")
+ tags["Release"].push_back(value);
+ name[0] = std::toupper(name[0]);
+ tags[std::move(name)].emplace_back(std::move(value));
+ }
+ }
}
- // nothing found, nuke it
- if (I == list.end())
- RemoveFileAt("mirror", dirfd, Dir->d_name);
- }
- closedir(D);
- return true;
-}
-
-
-bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/)
-{
- // not that great to use pkgAcquire here, but we do not have
- // any other way right now
- string fetch = BaseUri;
- fetch.replace(0,strlen("mirror://"),"http://");
-
-#if 0 // no need for this, the getArchitectures() will also include the main
- // arch
- // append main architecture
- fetch += "?arch=" + _config->Find("Apt::Architecture");
-#endif
-
- // append all architectures
- std::vector<std::string> vec = APT::Configuration::getArchitectures();
- for (std::vector<std::string>::const_iterator I = vec.begin();
- I != vec.end(); ++I)
- if (I == vec.begin())
- fetch += "?arch=" + (*I);
- else
- fetch += "&arch=" + (*I);
+ };
+ struct MirrorListInfo
+ {
+ MirrorFileState state;
+ std::string baseuri;
+ std::vector<MirrorInfo> list;
+ };
+ std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
- // append the dist as a query string
- if (Dist != "")
- fetch += "&dist=" + Dist;
+ virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
- if(Debug)
- clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'"
- << " to " << MirrorFile << endl;
+ void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message);
+ bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm);
+ std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
+ void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler);
- pkgAcquire Fetcher;
- new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile);
- bool res = (Fetcher.Run() == pkgAcquire::Continue);
- if(res) {
- DownloadedMirrorFile = true;
- chmod(MirrorFile.c_str(), 0644);
+ public:
+ MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests), genrng(clock())
+ {
+ SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
}
- Fetcher.Shutdown();
-
- if(Debug)
- clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl;
-
- return res;
-}
-
-// Randomizes the lines in the mirror file, this is used so that
-// we spread the load on the mirrors evenly
-bool MirrorMethod::RandomizeMirrorFile(string mirror_file)
+};
+ /*}}}*/
+void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
{
- vector<string> content;
- string line;
-
- if (!FileExists(mirror_file))
- return false;
-
- // read
- ifstream in(mirror_file.c_str());
- while ( !in.eof() ) {
- getline(in, line);
- content.push_back(line);
- }
-
- // we want the file to be random for each different machine, but also
- // "stable" on the same machine. this is to avoid running into out-of-sync
- // issues (i.e. Release/Release.gpg different on each mirror)
- struct utsname buf;
- int seed=1;
- if(uname(&buf) == 0) {
- for(int i=0,seed=1; buf.nodename[i] != 0; ++i) {
- seed = seed * 31 + buf.nodename[i];
+ std::unordered_map<std::string, std::string> matchers;
+ matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
+ matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
+ matchers.emplace("Component", LookupTag(Message, "Target-Component"));
+ matchers.emplace("Language", LookupTag(Message, "Target-Language"));
+ matchers.emplace("Release", LookupTag(Message, "Target-Release"));
+ matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
+ matchers.emplace("Type", LookupTag(Message, "Target-Type"));
+ decltype(info.list) possMirrors;
+ for (auto const &mirror : info.list)
+ {
+ bool failedMatch = false;
+ for (auto const &m : matchers)
+ {
+ if (m.second.empty())
+ continue;
+ auto const tagsetiter = mirror.tags.find(m.first);
+ if (tagsetiter == mirror.tags.end())
+ continue;
+ auto const tagset = tagsetiter->second;
+ if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
+ {
+ failedMatch = true;
+ break;
+ }
}
+ if (failedMatch)
+ continue;
+ possMirrors.push_back(mirror);
}
- srand( seed );
- random_shuffle(content.begin(), content.end());
-
- // write
- ofstream out(mirror_file.c_str());
- while ( !content.empty()) {
- line = content.back();
- content.pop_back();
- out << line << "\n";
+ for (auto &&mirror : possMirrors)
+ mirror.seed = genrng();
+ std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
+ if (a.priority != b.priority)
+ return a.priority < b.priority;
+ return a.seed < b.seed;
+ });
+ std::string const path = Itm->Uri.substr(info.baseuri.length());
+ std::string altMirrors;
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Queue->Uri);
+ for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
+ {
+ std::string mirror = curMirror->uri;
+ if (APT::String::Endswith(mirror, "/") == false)
+ mirror.append("/");
+ mirror.append(path);
+ if (curMirror == possMirrors.cbegin())
+ fields.emplace("New-URI", mirror);
+ else if (altMirrors.empty())
+ altMirrors.append(mirror);
+ else
+ altMirrors.append("\n").append(mirror);
}
-
- return true;
+ fields.emplace("Alternate-URIs", altMirrors);
+ SendMessage("103 Redirect", std::move(fields));
+ Dequeue();
}
-
-/* convert a the Queue->Uri back to the mirror base uri and look
- * at all mirrors we have for this, this is needed as queue->uri
- * may point to different mirrors (if TryNextMirror() was run)
- */
-void MirrorMethod::CurrentQueueUriToMirror()
+ /*}}}*/
+void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
+ MirrorListInfo const &info, FetchItem *const Itm,
+ std::function<void()> handler)
{
- // already in mirror:// style so nothing to do
- if(Queue->Uri.find("mirror://") == 0)
- return;
-
- // find current mirror and select next one
- for (vector<string>::const_iterator mirror = AllMirrors.begin();
- mirror != AllMirrors.end(); ++mirror)
+ FetchItem **LastItm = &Itm->Next;
+ while (*LastItm != nullptr)
+ LastItm = &((*LastItm)->Next);
+ while (Queue != Itm)
{
- if (Queue->Uri.find(*mirror) == 0)
+ if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
+ std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
{
- Queue->Uri.replace(0, mirror->length(), BaseUri);
- return;
+ // move the item behind the aux file not related to it
+ *LastItm = Queue;
+ Queue = QueueBack = Queue->Next;
+ (*LastItm)->Next = nullptr;
+ LastItm = &((*LastItm)->Next);
+ }
+ else
+ {
+ handler();
}
}
- _error->Error("Internal error: Failed to convert %s back to %s",
- Queue->Uri.c_str(), BaseUri.c_str());
+ // now remove out trigger
+ QueueBack = Queue = Queue->Next;
+ delete Itm;
}
-
-bool MirrorMethod::TryNextMirror()
+ /*}}}*/
+bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/
{
- // find current mirror and select next one
- for (vector<string>::const_iterator mirror = AllMirrors.begin();
- mirror != AllMirrors.end(); ++mirror)
+ std::vector<std::string> baseuris;
+ for (auto const &i : mirrorfilestate)
+ if (info.baseuri.length() < i.second.baseuri.length() &&
+ i.second.state == REQUESTED &&
+ APT::String::Startswith(i.second.baseuri, info.baseuri))
+ baseuris.push_back(i.second.baseuri);
+ sortByLength(baseuris);
+
+ FileFd mirrorlist;
+ if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
{
- if (Queue->Uri.find(*mirror) != 0)
- continue;
-
- vector<string>::const_iterator nextmirror = mirror + 1;
- if (nextmirror == AllMirrors.end())
- break;
- Queue->Uri.replace(0, mirror->length(), *nextmirror);
- if (Debug)
- clog << "TryNextMirror: " << Queue->Uri << endl;
-
- // inform parent
- UsedMirror = *nextmirror;
- Log("Switching mirror");
- return true;
- }
-
- if (Debug)
- clog << "TryNextMirror could not find another mirror to try" << endl;
-
- return false;
-}
+ auto const accessColon = info.baseuri.find(':');
+ auto access = info.baseuri.substr(0, accessColon);
+ std::string prefixAccess;
+ if (APT::String::Startswith(access, "mirror") == false)
+ {
+ auto const plus = info.baseuri.find('+');
+ prefixAccess = info.baseuri.substr(0, plus);
+ access.erase(0, plus + 1);
+ }
+ std::vector<std::string> limitAccess;
+ // If the mirror file comes from an online source, allow only other online
+ // sources, not e.g. file:///. If the mirrorlist comes from there we can assume
+ // the admin knows what (s)he is doing through and not limit the options.
+ if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
+ [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
+ access == "mirror")
+ {
+ for (auto const &a : disallowLocal)
+ limitAccess.emplace_back(a);
+ }
+ std::string line;
+ while (mirrorlist.ReadLine(line))
+ {
+ if (line.empty() || line[0] == '#')
+ continue;
+ auto const access = line.substr(0, line.find(':'));
+ if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
+ continue;
+ auto const tab = line.find('\t');
+ if (tab == std::string::npos)
+ {
+ if (prefixAccess.empty())
+ info.list.emplace_back(std::move(line));
+ else
+ info.list.emplace_back(prefixAccess + '+' + line);
+ }
+ else
+ {
+ auto uri = line.substr(0, tab);
+ if (prefixAccess.empty() == false)
+ uri = prefixAccess + '+' + uri;
+ auto tagline = line.substr(tab + 1);
+ std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
+ auto tags = VectorizeString(tagline, ' ');
+ tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
+ info.list.emplace_back(std::move(uri), std::move(tags));
+ }
+ }
+ mirrorlist.Close();
-bool MirrorMethod::InitMirrors()
-{
- // if we do not have a MirrorFile, fallback
- if(!FileExists(MirrorFile))
- {
- // FIXME: fallback to a default mirror here instead
- // and provide a config option to define that default
- return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str());
+ if (info.list.empty())
+ {
+ info.state = FAILED;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ std::string msg;
+ strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+ Fail(msg, false);
+ });
+ }
+ else
+ {
+ info.state = AVAILABLE;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ RedirectItem(info, Queue, msgCache[Queue->Uri]);
+ });
+ msgCache.clear();
+ }
}
-
- if (access(MirrorFile.c_str(), R_OK) != 0)
- {
- // FIXME: fallback to a default mirror here instead
- // and provide a config option to define that default
- return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str());
- }
-
- // FIXME: make the mirror selection more clever, do not
- // just use the first one!
- // BUT: we can not make this random, the mirror has to be
- // stable across session, because otherwise we can
- // get into sync issues (got indexfiles from mirror A,
- // but packages from mirror B - one might be out of date etc)
- ifstream in(MirrorFile.c_str());
- string s;
- while (!in.eof())
+ else
{
- getline(in, s);
-
- // ignore lines that start with #
- if (s.find("#") == 0)
- continue;
- // ignore empty lines
- if (s.size() == 0)
- continue;
- // ignore non http lines
- if (s.compare(0, strlen("http://"), "http://") != 0)
- continue;
-
- AllMirrors.push_back(s);
+ info.state = FAILED;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ std::string msg;
+ strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+ Fail(msg, false);
+ });
}
- if (AllMirrors.empty()) {
- return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str());
- }
- Mirror = AllMirrors[0];
- UsedMirror = Mirror;
return true;
}
-
-string MirrorMethod::GetMirrorFileName(string mirror_uri_str)
+ /*}}}*/
+std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
{
- /*
- - a mirror_uri_str looks like this:
- mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg
-
- - the matching source.list entry
- deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main
-
- - we actually want to go after:
- http://people.ubuntu.com/~mvo/apt/mirror/mirrors
-
- And we need to save the BaseUri for later:
- - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors
-
- FIXME: what if we have two similar prefixes?
- mirror://people.ubuntu.com/~mvo/mirror
- mirror://people.ubuntu.com/~mvo/mirror2
- then mirror_uri_str looks like:
- mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg
- mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg
- we search sources.list and find:
- mirror://people.ubuntu.com/~mvo/apt/mirror
- in both cases! So we need to apply some domain knowledge here :( and
- check for /dists/ or /Release.gpg as suffixes
- */
- string name;
- if(Debug)
- std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl;
-
- // read sources.list and find match
- vector<metaIndex *>::const_iterator I;
- pkgSourceList list;
- list.ReadMainList();
- for(I=list.begin(); I != list.end(); ++I)
+ if (APT::String::Startswith(Itm->Uri, Binary))
{
- string uristr = (*I)->GetURI();
- if(Debug)
- std::cerr << "Checking: " << uristr << std::endl;
- if(uristr.substr(0,strlen("mirror://")) != string("mirror://"))
- continue;
- // find matching uri in sources.list
- if(mirror_uri_str.substr(0,uristr.size()) == uristr)
+ std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
+ if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
+ sourceslist.push_back(repouri);
+ }
+ if (sourceslist.empty())
+ {
+ // read sources.list and find the matching base uri
+ pkgSourceList sl;
+ if (sl.ReadMainList() == false)
+ {
+ _error->Error(_("The list of sources could not be read."));
+ return "";
+ }
+ std::string const needle = Binary + ":";
+ for (auto const &SL : sl)
{
- if(Debug)
- std::cerr << "found BaseURI: " << uristr << std::endl;
- BaseUri = uristr.substr(0,uristr.size()-1);
- Dist = (*I)->GetDist();
+ std::string uristr = SL->GetURI();
+ if (APT::String::Startswith(uristr, needle))
+ sourceslist.push_back(uristr);
}
+ sortByLength(sourceslist);
}
- // get new file
- name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri);
-
- if(Debug)
+ for (auto uristr : sourceslist)
{
- cerr << "base-uri: " << BaseUri << endl;
- cerr << "mirror-file: " << name << endl;
+ if (APT::String::Startswith(Itm->Uri, uristr))
+ {
+ uristr.erase(uristr.length() - 1); // remove the ending '/'
+ auto const colon = uristr.find(':');
+ if (unlikely(colon == std::string::npos))
+ continue;
+ auto const plus = uristr.find("+");
+ if (plus < colon)
+ {
+ // started as tor+mirror+http we want to get the file via tor+http
+ auto access = uristr.substr(0, colon);
+ std::string prefixAccess;
+ if (APT::String::Startswith(access, "mirror") == false)
+ {
+ prefixAccess = uristr.substr(0, plus);
+ access.erase(0, plus + 1);
+ uristr.erase(plus, strlen("mirror") + 1);
+ return uristr;
+ }
+ else
+ return uristr.substr(plus + 1);
+ }
+ else
+ {
+ uristr.replace(0, strlen("mirror"), "http");
+ return uristr;
+ }
+ }
}
- return name;
+ return "";
}
-
-// MirrorMethod::Fetch - Fetch an item /*{{{*/
-// ---------------------------------------------------------------------
-/* This adds an item to the pipeline. We keep the pipeline at a fixed
- depth. */
-bool MirrorMethod::Fetch(FetchItem *Itm)
+ /*}}}*/
+bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
{
- if(Debug)
- clog << "MirrorMethod::Fetch()" << endl;
-
- // the http method uses Fetch(0) as a way to update the pipeline,
- // just let it do its work in this case - Fetch() with a valid
- // Itm will always run before the first Fetch(0)
- if(Itm == NULL)
- return HttpMethod::Fetch(Itm);
-
- // if we don't have the name of the mirror file on disk yet,
- // calculate it now (can be derived from the uri)
- if(MirrorFile.empty())
- MirrorFile = GetMirrorFileName(Itm->Uri);
+ auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
+ if (mirrorinfo != mirrorfilestate.end())
+ return MirrorListFileRecieved(mirrorinfo->second, Itm);
- // download mirror file once (if we are after index files)
- if(Itm->IndexFile && !DownloadedMirrorFile)
+ std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
+ if (mirrorfileuri.empty())
{
- Clean(_config->FindDir("Dir::State::mirrors"));
- if (DownloadMirrorFile(Itm->Uri))
- RandomizeMirrorFile(MirrorFile);
+ _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
+ return false;
}
+ if (DebugEnabled())
+ std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
- if(AllMirrors.empty()) {
- if(!InitMirrors()) {
- // no valid mirror selected, something went wrong downloading
- // from the master mirror site most likely and there is
- // no old mirror file availalbe
+ // have we requested this mirror file already?
+ auto const state = mirrorfilestate.find(mirrorfileuri);
+ if (state == mirrorfilestate.end())
+ {
+ msgCache[Itm->Uri] = Message;
+ MirrorListInfo info;
+ info.state = REQUESTED;
+ info.baseuri = mirrorfileuri + '/';
+ auto const colon = info.baseuri.find(':');
+ if (unlikely(colon == std::string::npos))
return false;
- }
+ info.baseuri.replace(0, colon, Binary);
+ mirrorfilestate[mirrorfileuri] = info;
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Itm->Uri);
+ fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
+ fields.emplace("Aux-ShortDesc", "Mirrorlist");
+ fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
+ fields.emplace("Aux-Uri", mirrorfileuri);
+ SendMessage("351 Aux Request", std::move(fields));
+ return true;
}
- if(Itm->Uri.find("mirror://") != string::npos)
- Itm->Uri.replace(0,BaseUri.size(), Mirror);
-
- if(Debug)
- clog << "Fetch: " << Itm->Uri << endl << endl;
-
- // now run the real fetcher
- return HttpMethod::Fetch(Itm);
-}
-
-void MirrorMethod::Fail(string Err,bool Transient)
-{
- // FIXME: TryNextMirror is not ideal for indexfile as we may
- // run into auth issues
-
- if (Debug)
- clog << "Failure to get " << Queue->Uri << endl;
-
- // try the next mirror on fail (if its not a expected failure,
- // e.g. translations are ok to ignore)
- if (!Queue->FailIgnore && TryNextMirror())
- return;
-
- // all mirrors failed, so bail out
- string s;
- strprintf(s, _("[Mirror: %s]"), Mirror.c_str());
- SetIP(s);
-
- CurrentQueueUriToMirror();
- pkgAcqMethod::Fail(Err, Transient);
-}
-
-void MirrorMethod::URIStart(FetchResult &Res)
-{
- CurrentQueueUriToMirror();
- pkgAcqMethod::URIStart(Res);
-}
-
-void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt)
-{
- CurrentQueueUriToMirror();
- pkgAcqMethod::URIDone(Res, Alt);
+ switch (state->second.state)
+ {
+ case REQUESTED:
+ // lets wait for the requested mirror file
+ msgCache[Itm->Uri] = Message;
+ return true;
+ case FAILED:
+ Fail("Downloading mirror file failed", false);
+ return true;
+ case AVAILABLE:
+ RedirectItem(state->second, Itm, Message);
+ return true;
+ }
+ return false;
}
+ /*}}}*/
-
-int main()
+int main(int, const char *argv[])
{
- return MirrorMethod().Loop();
+ return MirrorMethod(flNotDir(argv[0])).Run();
}
-
-