From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/mirror.cc | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 methods/mirror.cc (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 000000000..fad076e91 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,97 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + Mirror Aquire Method - This is the Mirror aquire method for APT. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include +#include + +#include +#include +using namespace std; + +#include "mirror.h" +#include "http.h" + + /*}}}*/ + +MirrorMethod::MirrorMethod() + : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) +{ +#if 0 + HasMirrorFile=true; + BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + Mirror="http://de.archive.ubuntu.com/ubuntu/"; +#endif +}; + +bool MirrorMethod::GetMirrorFile(string uri) +{ + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + BaseUri = uri.substr(0,uri.find(Marker)); + BaseUri.replace(0,strlen("mirror://"),"http://"); + + MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + + // FIXME: fetch it with curl + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + cerr << "fetch-result: " << res << endl; + + if(res) + HasMirrorFile = true; + Fetcher.Shutdown(); + return true; +} + +bool MirrorMethod::SelectMirror() +{ + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + cerr << "Mirror: " << Mirror << endl; +} + +// MirrorMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool MirrorMethod::Fetch(FetchItem *Itm) +{ + // get mirror information + if(!HasMirrorFile) + { + GetMirrorFile(Itm->Uri); + SelectMirror(); + } + + // change the items in the queue + Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); + cerr << "new Fetch-uri: " << Itm->Uri << endl; + + // FIXME: fetch it with! + +}; + +int main() +{ + setlocale(LC_ALL, ""); + + MirrorMethod Mth; + + return Mth.Run(); +} + + -- cgit v1.2.3 From 14e097c105c49a102a642c8108979356c5cc3152 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 22:59:35 +0100 Subject: * working mirror implementation based on http method --- methods/mirror.cc | 72 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 14 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index fad076e91..c4d911a3c 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace std; #include "mirror.h" @@ -25,31 +27,50 @@ using namespace std; /*}}}*/ MirrorMethod::MirrorMethod() - : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) + : HttpMethod(), HasMirrorFile(false) { #if 0 HasMirrorFile=true; - BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; + MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; Mirror="http://de.archive.ubuntu.com/ubuntu/"; #endif }; +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + + bool MirrorMethod::GetMirrorFile(string uri) { string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); BaseUri = uri.substr(0,uri.find(Marker)); - BaseUri.replace(0,strlen("mirror://"),"http://"); + + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + } // FIXME: fetch it with curl pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - cerr << "fetch-result: " << res << endl; if(res) HasMirrorFile = true; @@ -61,7 +82,9 @@ bool MirrorMethod::SelectMirror() { ifstream in(MirrorFile.c_str()); getline(in, Mirror); - cerr << "Mirror: " << Mirror << endl; + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + return true; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -77,21 +100,42 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - // change the items in the queue - Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); - cerr << "new Fetch-uri: " << Itm->Uri << endl; + if(Queue->Uri.find("mirror://") != string::npos) + Queue->Uri.replace(0,BaseUri.size(),Mirror); - // FIXME: fetch it with! - + // now run the real fetcher + return HttpMethod::Fetch(Itm); }; +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + int main() { setlocale(LC_ALL, ""); MirrorMethod Mth; - return Mth.Run(); + return Mth.Loop(); } -- cgit v1.2.3 From 86c17f0a4bd3c0e3dde5bb58b3df9c41e5a88763 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:09:26 +0100 Subject: * todo added --- methods/mirror.cc | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index c4d911a3c..c49764044 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -26,6 +26,14 @@ using namespace std; /*}}}*/ +/* + * TODO: + * - support keeping the mirror file around (evil listclearer strikes again) + * - better method to download than having a pkgAcquire interface here + * - testing :) + * + */ + MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { -- cgit v1.2.3 From 3ed91a17d21d0d456a12accc507b82e1c0f1a697 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:21:47 +0100 Subject: * more todo items --- methods/mirror.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index c49764044..1911d6115 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -28,10 +28,11 @@ using namespace std; /* * TODO: - * - support keeping the mirror file around (evil listclearer strikes again) * - better method to download than having a pkgAcquire interface here + * - support keeping the mirror file around (evil listclearer strikes again) + * -> /var/lib/apt/mirrors dir? how to cleanup? by time? + * - provide some TTL time until the mirror file is get again (1h? 6h?) * - testing :) - * */ MirrorMethod::MirrorMethod() -- cgit v1.2.3 From ef1e6d8822bb34527bba5d2318b5768013efd010 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Nov 2006 11:06:09 +0100 Subject: * methods/mirror.cc: - check the complette queue on Fetch() for urls to transform --- methods/mirror.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 1911d6115..4de981522 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,6 +32,7 @@ using namespace std; * - support keeping the mirror file around (evil listclearer strikes again) * -> /var/lib/apt/mirrors dir? how to cleanup? by time? * - provide some TTL time until the mirror file is get again (1h? 6h?) + * - deal with runing as non-root (we can't write to the lists dir then) * - testing :) */ @@ -109,8 +110,11 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - if(Queue->Uri.find("mirror://") != string::npos) - Queue->Uri.replace(0,BaseUri.size(),Mirror); + for (FetchItem *I = Queue; I != 0; I = I->Next) + { + if(I->Uri.find("mirror://") != string::npos) + I->Uri.replace(0,BaseUri.size(),Mirror); + } // now run the real fetcher return HttpMethod::Fetch(Itm); -- cgit v1.2.3 From d731f9c574138a9cb31c586a8f1e7d7181a44456 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:19:32 +0100 Subject: * apt-pkg/init.cc: - added Dir::State::Mirrors * doc/examples/configure-index: - added Acquire::mirror::RefreshInterval * methods/mirror.{cc,h}: - download the mirror file into Dir::State::Mirrors - added RefreshInterval option to not ask for the mirror file too often --- methods/mirror.cc | 56 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 10 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4de981522..e70a40f55 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -18,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -28,11 +29,16 @@ using namespace std; /* * TODO: + * - send expected checksum to the mirror method so that + some checking/falling back can be done here already + * - keep the mirror file around in /var/lib/apt/mirrors + * can't be put into lists/ because of the listclearer + * cleanup by time (mtime relative to the other mtimes) + * - use a TTL time the mirror file is fetched again (6h?) + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - support keeping the mirror file around (evil listclearer strikes again) - * -> /var/lib/apt/mirrors dir? how to cleanup? by time? - * - provide some TTL time until the mirror file is get again (1h? 6h?) - * - deal with runing as non-root (we can't write to the lists dir then) + * - magicmarker is (a bit) evil * - testing :) */ @@ -60,6 +66,12 @@ bool MirrorMethod::Configuration(string Message) } /*}}}*/ +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string dir) +{ + +} + bool MirrorMethod::GetMirrorFile(string uri) { @@ -69,7 +81,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { @@ -77,19 +89,43 @@ bool MirrorMethod::GetMirrorFile(string uri) cerr << "mirror-file: " << MirrorFile << endl; } - // FIXME: fetch it with curl + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + HasMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - - if(res) + if(res) HasMirrorFile = true; Fetcher.Shutdown(); - return true; + return res; } bool MirrorMethod::SelectMirror() { + // FIXME: make the mirror selection more clever, do not + // just use the first one! ifstream in(MirrorFile.c_str()); getline(in, Mirror); if(Debug) @@ -103,7 +139,7 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // get mirror information + // select mirror only once per session if(!HasMirrorFile) { GetMirrorFile(Itm->Uri); -- cgit v1.2.3 From 70288656715d2fe4c2c33598124ae48f7bca9cdf Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:38:55 +0100 Subject: * methods/mirror.cc: - implemented simple Clean() implementation based on the time of the last access for a mirror file --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index e70a40f55..425a2f7f5 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -19,12 +19,14 @@ #include #include #include +#include +#include using namespace std; #include "mirror.h" #include "http.h" - +#include "apti18n.h" /*}}}*/ /* @@ -67,9 +69,54 @@ bool MirrorMethod::Configuration(string Message) /*}}}*/ // clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string dir) +bool MirrorMethod::Clean(string Dir) { + // FIXME: it would better to have a global idea of the mirrors + // in the sources.list and use this instead of this time + // based approach. currently apt does not support this :/ + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string StartDir = SafeGetCWD(); + if (chdir(Dir.c_str()) != 0) + { + closedir(D); + return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); + } + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,"lock") == 0 || + strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + // Del everything not touched for MaxAge days + time_t t,now,max; + struct stat buf; + if(stat(Dir->d_name, &buf) != 0) + { + cerr << "Can't stat '" << Dir->d_name << "'" << endl; + continue; + } + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); + if(t + max < now) + { + if(Debug) + clog << "Mirror file is older than MaxAge days, deleting" << endl; + unlink(Dir->d_name); + } + }; + chdir(StartDir.c_str()); + closedir(D); + return true; } @@ -81,6 +128,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); + // get new file MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) @@ -108,7 +156,7 @@ bool MirrorMethod::GetMirrorFile(string uri) return true; } if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; } // not that great to use pkgAcquire here, but we do not have @@ -142,6 +190,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) // select mirror only once per session if(!HasMirrorFile) { + Clean(_config->FindDir("Dir::State::mirrors")); GetMirrorFile(Itm->Uri); SelectMirror(); } -- cgit v1.2.3 From 0c312e0ed24fed69412c181634dd4515e9b1d46f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 12:10:05 +0100 Subject: * implemented proper mirror list cleanup --- methods/mirror.cc | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 425a2f7f5..00f7b7807 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -71,9 +72,14 @@ bool MirrorMethod::Configuration(string Message) // clean the mirrors dir based on ttl information bool MirrorMethod::Clean(string Dir) { - // FIXME: it would better to have a global idea of the mirrors - // in the sources.list and use this instead of this time - // based approach. currently apt does not support this :/ + vector::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + // read sources.list + pkgSourceList list; + list.ReadMainList(); DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -94,24 +100,21 @@ bool MirrorMethod::Clean(string Dir) strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; - - // Del everything not touched for MaxAge days - time_t t,now,max; - struct stat buf; - if(stat(Dir->d_name, &buf) != 0) + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) { - cerr << "Can't stat '" << Dir->d_name << "'" << endl; - continue; + string uri = (*I)->GetURI(); + if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + string BaseUri = uri.substr(0,uri.find(Marker)); + if (URItoFileName(BaseUri) == Dir->d_name) + break; } - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); - if(t + max < now) - { - if(Debug) - clog << "Mirror file is older than MaxAge days, deleting" << endl; + // nothing found, nuke it + if (I == list.end()) unlink(Dir->d_name); - } }; chdir(StartDir.c_str()); -- cgit v1.2.3 From 933833c51fbfe3dca3f3a3073d441e5856462605 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 13:07:22 +0100 Subject: * methods/mirror.cc: - updated the TODO --- methods/mirror.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 00f7b7807..f08b324ec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -34,14 +34,18 @@ using namespace std; * TODO: * - send expected checksum to the mirror method so that some checking/falling back can be done here already - * - keep the mirror file around in /var/lib/apt/mirrors - * can't be put into lists/ because of the listclearer - * cleanup by time (mtime relative to the other mtimes) - * - use a TTL time the mirror file is fetched again (6h?) + use pkgAcquire::Custom600Header() for this? what about gpgv + failures? + #OR# + * - implement it at the pkgAcquire::Item::Failed() level but then + we need to send back what uri exactly was failing + * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil + * - magicmarker is (a bit) evil, maybe just use a similar approach as in + clean and read the sources.list and use the GetURI() method to find + the prefix? * - testing :) */ -- cgit v1.2.3 From cae9cdcefc34eba6d023cb30cbbdb9bbf909b8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 16:35:56 +0100 Subject: * basic error reporting from apt in place now (ReportMirrorFailures()) --- methods/mirror.cc | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index f08b324ec..428726a3d 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,14 +32,14 @@ using namespace std; /* * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - use pkgAcquire::Custom600Header() for this? what about gpgv - failures? + * - what about gpgv failures? better standard format for errors + to send back to LP #OR# * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing - + we need to send back what uri exactly was failing + [mvo: the problem with this approach is ::Failed() is not really + called for all failures :/ e.g. md5sum mismatch in a archive + is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here @@ -214,6 +214,9 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // FIXME: queue next mirror? + ReportMirrorFailure(Err); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -228,11 +231,23 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { + // FIXME: queue next mirror? + if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) + ReportMirrorFailure("499 Hash mismatch"); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } +void MirrorMethod::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed + std::cerr << "\nReportMirrorFailure: " + << Queue->Uri + << " FailCode: " + << FailCode << std::endl; +} int main() { -- cgit v1.2.3 From 2769f0bcf6c1ac73e883f47857fa227de92f4525 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 17:16:42 +0100 Subject: * mirror-failure.py: example mirror failure cgi * methods/mirror.cc: prepare for the failure submit --- methods/mirror.cc | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 428726a3d..6621d47e2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -247,6 +247,12 @@ void MirrorMethod::ReportMirrorFailure(string FailCode) << Queue->Uri << " FailCode: " << FailCode << std::endl; +#if 0 // FIXME: do not use system, make sure to properly encode + // URI/FailCode, do not hardcode the submit url + system("curl -d url=" + Queue->Uri + + " -d FailureCode=" + FailCode + + " http://localhost:8000/ &"); +#endif } int main() -- cgit v1.2.3 From 362d29343e5d25248bcd84300aa1b18effe891e7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Jan 2007 14:42:17 +0100 Subject: make the mirror failures actually produce a error message --- methods/mirror.cc | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 6621d47e2..8f9b8ed34 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -30,13 +30,18 @@ using namespace std; #include "apti18n.h" /*}}}*/ -/* +/* Done: + * - works with http only + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * * TODO: - * - what about gpgv failures? better standard format for errors - to send back to LP - #OR# - * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing + * what about gpgv failures? this should call-out to the problem reporting + script, but we need to know what mirror was used + * better standard format for errors to send back + * - implement failure reporting at the pkgAcquire::Item::Failed() level + but then we need to send back what uri exactly was failing [mvo: the problem with this approach is ::Failed() is not really called for all failures :/ e.g. md5sum mismatch in a archive is not] @@ -46,6 +51,7 @@ using namespace std; * - magicmarker is (a bit) evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? + * support more than http * - testing :) */ @@ -243,16 +249,34 @@ void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) void MirrorMethod::ReportMirrorFailure(string FailCode) { // report that Queue->Uri failed +#if 0 std::cerr << "\nReportMirrorFailure: " << Queue->Uri << " FailCode: " << FailCode << std::endl; -#if 0 // FIXME: do not use system, make sure to properly encode - // URI/FailCode, do not hardcode the submit url - system("curl -d url=" + Queue->Uri + - " -d FailureCode=" + FailCode + - " http://localhost:8000/ &"); #endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/lib/apt/report-mirror-failure"); + Args[i++] = report.c_str(); + Args[i++] = Queue->Uri.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } } int main() -- cgit v1.2.3 From 36280399db0ae203d3f1ae4d44b946f31e9a38ce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 1 Feb 2007 12:32:50 +0100 Subject: * commited the latest mirror failure detection code --- methods/mirror.cc | 45 ++++----------------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 8f9b8ed34..8ccfb8559 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -38,7 +38,7 @@ using namespace std; * * TODO: * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used + script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() * better standard format for errors to send back * - implement failure reporting at the pkgAcquire::Item::Failed() level but then we need to send back what uri exactly was failing @@ -48,7 +48,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil, maybe just use a similar approach as in + * - magicmarker is evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? * support more than http @@ -191,6 +191,8 @@ bool MirrorMethod::SelectMirror() getline(in, Mirror); if(Debug) cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; return true; } @@ -220,9 +222,6 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // FIXME: queue next mirror? - ReportMirrorFailure(Err); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -237,47 +236,11 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - // FIXME: queue next mirror? - if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) - ReportMirrorFailure("499 Hash mismatch"); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } -void MirrorMethod::ReportMirrorFailure(string FailCode) -{ - // report that Queue->Uri failed -#if 0 - std::cerr << "\nReportMirrorFailure: " - << Queue->Uri - << " FailCode: " - << FailCode << std::endl; -#endif - const char *Args[40]; - unsigned int i = 0; - string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/lib/apt/report-mirror-failure"); - Args[i++] = report.c_str(); - Args[i++] = Queue->Uri.c_str(); - Args[i++] = FailCode.c_str(); - pid_t pid = ExecFork(); - if(pid < 0) - { - _error->Error("ReportMirrorFailure Fork failed"); - return; - } - else if(pid == 0) - { - execvp(report.c_str(), (char**)Args); - } - if(!ExecWait(pid, "report-mirror-failure")) - { - _error->Warning("Couldn't report problem to '%s'", - _config->Find("Acquire::Mirror::ReportFailures").c_str()); - } -} int main() { -- cgit v1.2.3 From 59271f62e4a291c8d96e1f6073203c395734b6ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 17:52:28 +0100 Subject: * use pkgAcqMethod::FailReason() for consistent error reporting --- methods/mirror.cc | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index 8ccfb8559..b64879bec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -31,20 +31,14 @@ using namespace std; /*}}}*/ /* Done: - * - works with http only + * - works with http (only!) * - always picks the first mirror from the list * - call out to problem reporting script * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - use pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP * * TODO: - * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() - * better standard format for errors to send back - * - implement failure reporting at the pkgAcquire::Item::Failed() level - but then we need to send back what uri exactly was failing - [mvo: the problem with this approach is ::Failed() is not really - called for all failures :/ e.g. md5sum mismatch in a archive - is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here -- cgit v1.2.3 From 066b53e95ba224f81d078e7d8df3002275f8a092 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 18:46:45 +0100 Subject: * get rid of magic mirror --- methods/mirror.cc | 62 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 16 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index b64879bec..ff91130b8 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,22 +42,13 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is evil, maybe just use a similar approach as in - clean and read the sources.list and use the GetURI() method to find - the prefix? - * support more than http + * - support more than http * - testing :) */ MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { -#if 0 - HasMirrorFile=true; - BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; - MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; - Mirror="http://de.archive.ubuntu.com/ubuntu/"; -#endif }; // HttpMethod::Configuration - Handle a configuration message /*{{{*/ @@ -111,8 +102,7 @@ bool MirrorMethod::Clean(string Dir) string uri = (*I)->GetURI(); if(uri.substr(0,strlen("mirror://")) != string("mirror://")) continue; - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - string BaseUri = uri.substr(0,uri.find(Marker)); + string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) break; } @@ -127,11 +117,51 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string uri) +bool MirrorMethod::GetMirrorFile(string mirror_uri_str) { - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - BaseUri = uri.substr(0,uri.find(Marker)); - + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); -- cgit v1.2.3 From f0b509cdb44cb5e79e9c5ddd7ebec46965138534 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 19:06:42 +0100 Subject: * apt-pkg/acquire-item.cc: - use FailReason in pkgAcquire::Item::Failed when available * methods/mirror.cc: - move some debug output into if(Debug) --- methods/mirror.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index ff91130b8..c5c0c7461 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -143,7 +143,8 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + if(Debug) + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -152,13 +153,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) for(I=list.begin(); I != list.end(); I++) { string uristr = (*I)->GetURI(); - std::cerr << "Checking: " << uristr << std::endl; + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) continue; // find matching uri in sources.list if(mirror_uri_str.substr(0,uristr.size()) == uristr) { - std::cerr << "found BaseURI: " << uristr << std::endl; + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; BaseUri = uristr.substr(0,uristr.size()-1); } } -- cgit v1.2.3 From 3f599bb721c4ac58d8ff18991c9704b5f30eaa2b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 10:49:25 +0100 Subject: * apt-pkg/acquire-item.cc: - default to "/usr/lib/apt/apt-report-mirror-failure" * cmdline/apt-report-mirror-failure: - no default comit url for now * debian/rules: - move apt-report-mirror-failure into /usr/lib/apt * doc/examples/configure-index: - more documentation * methods/mirror.cc: - updated TODO --- methods/mirror.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'methods/mirror.cc') diff --git a/methods/mirror.cc b/methods/mirror.cc index c5c0c7461..9a86a10c2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,6 +42,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here + * and better error handling there! * - support more than http * - testing :) */ -- cgit v1.2.3