From 96db74ce38e9451609fe33f9e25f3f9d42b1fe22 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:40:40 +0200 Subject: * apt-pkg/deb/dpkgpm.cc: - add missing include * methods/mirror.{cc,h}: - add SelectNextMirror() and InitMirrors() functions - read all mirrors into the AllMirrors vector --- methods/mirror.cc | 30 +++++++++++++++++++++++------- methods/mirror.h | 4 +++- 2 files changed, 26 insertions(+), 8 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b3a956b95..0a0d07e6b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,7 +162,21 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectMirror() +bool MirrorMethod::SelectNextMirror() +{ + if (AllMirrors.size() < 1) + return false; + + Mirror = AllMirrors[0]; + AllMirrors.erase(AllMirrors.begin()); + if(Debug) + cerr << "using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +bool MirrorMethod::InitMirrors() { // if we do not have a MirrorFile, fallback if(!FileExists(MirrorFile)) @@ -179,11 +193,13 @@ bool MirrorMethod::SelectMirror() // get into sync issues (got indexfiles from mirror A, // but packages from mirror B - one might be out of date etc) ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; + string s; + while (!in.eof()) + { + getline(in, s); + AllMirrors.push_back(s); + } + SelectNextMirror(); return true; } @@ -275,7 +291,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) } if(Mirror.empty()) { - if(!SelectMirror()) { + if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is // no old mirror file availalbe diff --git a/methods/mirror.h b/methods/mirror.h index ed817806b..1b506dc87 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,6 +26,7 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) + vector AllMirrors; // all available mirrors string MirrorFile; // the file that contains the list of mirrors bool DownloadedMirrorFile; // already downloaded this session @@ -34,7 +35,8 @@ class MirrorMethod : public HttpMethod protected: bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); - bool SelectMirror(); + bool InitMirrors(); + bool SelectNextMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3 From 483dfdd8aced593e966d221073c056c2e332584f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:51:25 +0200 Subject: methods/mirror.cc: on fail try the next mirror --- methods/mirror.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 0a0d07e6b..b8bd6db73 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -197,7 +197,8 @@ bool MirrorMethod::InitMirrors() while (!in.eof()) { getline(in, s); - AllMirrors.push_back(s); + if (s.size() > 0) + AllMirrors.push_back(s); } SelectNextMirror(); return true; @@ -314,6 +315,15 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // try the next mirror on fail + string old_mirror = Mirror; + if (SelectNextMirror()) + { + Queue->Uri.replace(0, old_mirror.size(), Mirror); + return; + } + + // all mirrors failed, so bail out if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); -- cgit v1.2.3 From 0ded3ad3438666f833773895ef6318f84d2f849d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 22:44:49 +0200 Subject: improve error message if mirror method fails --- methods/mirror.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b8bd6db73..b9fa55d87 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -25,6 +25,8 @@ using namespace std; +#include + #include "mirror.h" #include "http.h" #include "apti18n.h" @@ -164,16 +166,12 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) bool MirrorMethod::SelectNextMirror() { - if (AllMirrors.size() < 1) - return false; - - Mirror = AllMirrors[0]; - AllMirrors.erase(AllMirrors.begin()); if(Debug) cerr << "using mirror: " << Mirror << endl; + Mirror = AllMirrors[0]; UsedMirror = Mirror; - return true; + return false; } bool MirrorMethod::InitMirrors() @@ -324,6 +322,10 @@ void MirrorMethod::Fail(string Err,bool Transient) } // all mirrors failed, so bail out + string s; + strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); + SetIP(s); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); -- cgit v1.2.3 From 661f7b1c727aada703ffbb350d44759189a441fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 22:47:49 +0200 Subject: methods/mirror.cc: remove Acquire::Mirror::RefreshInterval (not really useful) --- methods/mirror.cc | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b9fa55d87..4d767c9f3 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -128,28 +128,6 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) if(Debug) clog << "MirrorMethod::DownloadMirrorFile(): " << endl; - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - DownloadedMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; - } - // not that great to use pkgAcquire here, but we do not have // any other way right now string fetch = BaseUri; -- cgit v1.2.3 From 0391542729e5e8a2ba9ae9b1d470f252e24eb296 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:13:42 +0200 Subject: methods/mirrors.cc: make cycle through the mirrors work properly --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++--------------- methods/mirror.h | 3 ++- 2 files changed, 42 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4d767c9f3..567522478 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -142,13 +142,44 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectNextMirror() +/* convert a the Queue->Uri back to the mirror base uri and look + * at all mirrors we have for this, this is needed as queue->uri + * may point to different mirrors (if TryNextMirror() was run) + */ +void MirrorMethod::CurrentQueueUriToMirror() +{ + // already in mirror:// style so nothing to do + if(Queue->Uri.find("mirror://") == 0) + return; + + // find current mirror and select next one + for (int i=0; i < AllMirrors.size(); i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + return; + } + } + _error->Error("Internal error: Failed to convert %s back to %s", + Queue->Uri, BaseUri); +} + +bool MirrorMethod::TryNextMirror() { if(Debug) cerr << "using mirror: " << Mirror << endl; - Mirror = AllMirrors[0]; - UsedMirror = Mirror; + // find current mirror and select next one + for (int i=0; i < AllMirrors.size()-1; i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + return true; + } + } + return false; } @@ -176,7 +207,8 @@ bool MirrorMethod::InitMirrors() if (s.size() > 0) AllMirrors.push_back(s); } - SelectNextMirror(); + Mirror = AllMirrors[0]; + UsedMirror = Mirror; return true; } @@ -292,34 +324,27 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - string old_mirror = Mirror; - if (SelectNextMirror()) - { - Queue->Uri.replace(0, old_mirror.size(), Mirror); + if (TryNextMirror()) return; - } // all mirrors failed, so bail out string s; strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); SetIP(s); - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::Fail(Err, Transient); } void MirrorMethod::URIStart(FetchResult &Res) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIStart(Res); } void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIDone(Res, Alt); } diff --git a/methods/mirror.h b/methods/mirror.h index 1b506dc87..0a3ea6e92 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -36,7 +36,8 @@ class MirrorMethod : public HttpMethod bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); bool InitMirrors(); - bool SelectNextMirror(); + bool TryNextMirror(); + void CurrentQueueUriToMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3 From b86f642111954754dd9932ed2f28a9ea85035e87 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:15:55 +0200 Subject: methods/mirror.cc: simplify uri.startswith() --- methods/mirror.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 567522478..cfc155f58 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -106,7 +106,7 @@ bool MirrorMethod::Clean(string Dir) for(I=list.begin(); I != list.end(); I++) { string uri = (*I)->GetURI(); - if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + if(uri.find("mirror://") != 0) continue; string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) -- cgit v1.2.3 From 963b16dcebba149ae2c35bd255b34242923fbea0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:30:01 +0200 Subject: implement Fail-Ignore bool in FetchItem that tells the method that a failure of this item is ok and does not need to be tried on all mirrors --- methods/mirror.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index cfc155f58..ea0fba438 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,24 +162,26 @@ void MirrorMethod::CurrentQueueUriToMirror() } } _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri, BaseUri); + Queue->Uri.c_str(), BaseUri.c_str()); } bool MirrorMethod::TryNextMirror() { - if(Debug) - cerr << "using mirror: " << Mirror << endl; - // find current mirror and select next one for (int i=0; i < AllMirrors.size()-1; i++) { if (Queue->Uri.find(AllMirrors[i]) == 0) { Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; return true; } } + if (Debug) + clog << "TryNextMirror could not find another mirror to try" << endl; + return false; } @@ -307,15 +309,12 @@ bool MirrorMethod::Fetch(FetchItem *Itm) return false; } } - if(Debug) - clog << "selected mirror: " << Mirror << endl; + if(Itm->Uri.find("mirror://") != string::npos) + Itm->Uri.replace(0,BaseUri.size(), Mirror); - for (FetchItem *I = Queue; I != 0; I = I->Next) - { - if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(), Mirror); - } + if(Debug) + clog << "Fetch: " << Itm->Uri << endl << endl; // now run the real fetcher return HttpMethod::Fetch(Itm); @@ -324,7 +323,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - if (TryNextMirror()) + if (!Queue->FailIgnore && TryNextMirror()) return; // all mirrors failed, so bail out -- cgit v1.2.3 From 2ac9b90b7b3ebeea9a2580b6f317f1dfefc8c8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:57:00 +0200 Subject: methods/mirror.cc: debug improvements --- methods/mirror.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ea0fba438..b2b6b2ecf 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -301,7 +301,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) { + if(AllMirrors.empty()) { if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is @@ -322,7 +322,14 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // try the next mirror on fail + // FIXME: TryNextMirror is not ideal for indexfile as we may + // run into auth issues + + if (Debug) + clog << "Failure to get " << Queue->Uri << endl; + + // try the next mirror on fail (if its not a expected failure, + // e.g. translations are ok to ignore) if (!Queue->FailIgnore && TryNextMirror()) return; -- cgit v1.2.3