From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/http.cc | 13 ++----- methods/http.h | 7 ++-- methods/http_main.cc | 15 ++++++++ methods/makefile | 9 ++++- methods/mirror.cc | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ methods/mirror.h | 40 ++++++++++++++++++++++ 6 files changed, 166 insertions(+), 15 deletions(-) create mode 100644 methods/http_main.cc create mode 100644 methods/mirror.cc create mode 100644 methods/mirror.h (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c6623c46f..deaa8d0c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,7 +38,6 @@ #include #include #include -#include #include // Internet stuff @@ -57,7 +56,7 @@ time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; bool Debug = false; - +URI Proxy; unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; @@ -990,7 +989,7 @@ void HttpMethod::SigTerm(int) depth. */ bool HttpMethod::Fetch(FetchItem *) { - if (Server == 0) + if (Server == 0) return true; // Queue the requests @@ -1223,13 +1222,5 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - - HttpMethod Mth; - - return Mth.Loop(); -} diff --git a/methods/http.h b/methods/http.h index 541e2952c..5eac11401 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 -#include + using std::cout; using std::endl; @@ -134,7 +134,6 @@ class HttpMethod : public pkgAcqMethod bool ServerDie(ServerState *Srv); int DealWithHeaders(FetchResult &Res,ServerState *Srv); - virtual bool Fetch(FetchItem *); virtual bool Configuration(string Message); // In the event of a fatal signal this file will be closed and timestamped. @@ -142,6 +141,9 @@ class HttpMethod : public pkgAcqMethod static int FailFd; static time_t FailTime; static void SigTerm(int); + + protected: + virtual bool Fetch(FetchItem *); public: friend class ServerState; @@ -158,6 +160,5 @@ class HttpMethod : public pkgAcqMethod }; }; -URI Proxy; #endif diff --git a/methods/http_main.cc b/methods/http_main.cc new file mode 100644 index 000000000..2c46ab19d --- /dev/null +++ b/methods/http_main.cc @@ -0,0 +1,15 @@ +#include +#include + +#include "connect.h" +#include "rfc2553emu.h" +#include "http.h" + + +int main() +{ + setlocale(LC_ALL, ""); + + HttpMethod Mth; + return Mth.Loop(); +} diff --git a/methods/makefile b/methods/makefile index 1e3b1ef85..12e446a55 100644 --- a/methods/makefile +++ b/methods/makefile @@ -49,7 +49,7 @@ include $(PROGRAM_H) PROGRAM=http SLIBS = -lapt-pkg $(SOCKETLIBS) LIB_MAKES = apt-pkg/makefile -SOURCE = http.cc rfc2553emu.cc connect.cc +SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) # The ftp method @@ -66,6 +66,13 @@ LIB_MAKES = apt-pkg/makefile SOURCE = rsh.cc include $(PROGRAM_H) +# The mirror method +PROGRAM=mirror +SLIBS = -lapt-pkg $(SOCKETLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc +include $(PROGRAM_H) + # SSH and vzip2 method symlink binary: $(BIN)/ssh $(BIN)/bzip2 veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 000000000..fad076e91 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,97 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + Mirror Aquire Method - This is the Mirror aquire method for APT. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include +#include + +#include +#include +using namespace std; + +#include "mirror.h" +#include "http.h" + + /*}}}*/ + +MirrorMethod::MirrorMethod() + : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) +{ +#if 0 + HasMirrorFile=true; + BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + Mirror="http://de.archive.ubuntu.com/ubuntu/"; +#endif +}; + +bool MirrorMethod::GetMirrorFile(string uri) +{ + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + BaseUri = uri.substr(0,uri.find(Marker)); + BaseUri.replace(0,strlen("mirror://"),"http://"); + + MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + + // FIXME: fetch it with curl + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + cerr << "fetch-result: " << res << endl; + + if(res) + HasMirrorFile = true; + Fetcher.Shutdown(); + return true; +} + +bool MirrorMethod::SelectMirror() +{ + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + cerr << "Mirror: " << Mirror << endl; +} + +// MirrorMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool MirrorMethod::Fetch(FetchItem *Itm) +{ + // get mirror information + if(!HasMirrorFile) + { + GetMirrorFile(Itm->Uri); + SelectMirror(); + } + + // change the items in the queue + Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); + cerr << "new Fetch-uri: " << Itm->Uri << endl; + + // FIXME: fetch it with! + +}; + +int main() +{ + setlocale(LC_ALL, ""); + + MirrorMethod Mth; + + return Mth.Run(); +} + + diff --git a/methods/mirror.h b/methods/mirror.h new file mode 100644 index 000000000..ca3428830 --- /dev/null +++ b/methods/mirror.h @@ -0,0 +1,40 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + MIRROR Aquire Method - This is the MIRROR aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_MIRROR_H +#define APT_MIRROR_H + + +#include + +using std::cout; +using std::cerr; +using std::endl; + +#include "http.h" + +class MirrorMethod : public pkgAcqMethod +{ + FetchResult Res; + string Mirror; + string BaseUri; + string MirrorFile; + bool HasMirrorFile; + + protected: + bool GetMirrorFile(string uri); + bool SelectMirror(); + public: + MirrorMethod(); + virtual bool Fetch(FetchItem *Itm); +}; + + +#endif -- cgit v1.2.3 From 14e097c105c49a102a642c8108979356c5cc3152 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 22:59:35 +0100 Subject: * working mirror implementation based on http method --- methods/mirror.cc | 72 ++++++++++++++++++++++++++++++++++++++++++++----------- methods/mirror.h | 15 ++++++++---- 2 files changed, 69 insertions(+), 18 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index fad076e91..c4d911a3c 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace std; #include "mirror.h" @@ -25,31 +27,50 @@ using namespace std; /*}}}*/ MirrorMethod::MirrorMethod() - : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) + : HttpMethod(), HasMirrorFile(false) { #if 0 HasMirrorFile=true; - BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; + MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; Mirror="http://de.archive.ubuntu.com/ubuntu/"; #endif }; +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + + bool MirrorMethod::GetMirrorFile(string uri) { string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); BaseUri = uri.substr(0,uri.find(Marker)); - BaseUri.replace(0,strlen("mirror://"),"http://"); + + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + } // FIXME: fetch it with curl pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - cerr << "fetch-result: " << res << endl; if(res) HasMirrorFile = true; @@ -61,7 +82,9 @@ bool MirrorMethod::SelectMirror() { ifstream in(MirrorFile.c_str()); getline(in, Mirror); - cerr << "Mirror: " << Mirror << endl; + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + return true; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -77,21 +100,42 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - // change the items in the queue - Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); - cerr << "new Fetch-uri: " << Itm->Uri << endl; + if(Queue->Uri.find("mirror://") != string::npos) + Queue->Uri.replace(0,BaseUri.size(),Mirror); - // FIXME: fetch it with! - + // now run the real fetcher + return HttpMethod::Fetch(Itm); }; +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + int main() { setlocale(LC_ALL, ""); MirrorMethod Mth; - return Mth.Run(); + return Mth.Loop(); } diff --git a/methods/mirror.h b/methods/mirror.h index ca3428830..91cf8c64f 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -20,17 +20,24 @@ using std::endl; #include "http.h" -class MirrorMethod : public pkgAcqMethod +class MirrorMethod : public HttpMethod { FetchResult Res; - string Mirror; - string BaseUri; - string MirrorFile; + string BaseUri; // the original mirror://... url + string Mirror; // the selected mirror uri (http://...) + string MirrorFile; // bool HasMirrorFile; + bool Debug; + protected: bool GetMirrorFile(string uri); bool SelectMirror(); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual bool Configuration(string Message); + public: MirrorMethod(); virtual bool Fetch(FetchItem *Itm); -- cgit v1.2.3 From 86c17f0a4bd3c0e3dde5bb58b3df9c41e5a88763 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:09:26 +0100 Subject: * todo added --- methods/mirror.cc | 8 ++++++++ methods/mirror.h | 3 +++ 2 files changed, 11 insertions(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c4d911a3c..c49764044 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -26,6 +26,14 @@ using namespace std; /*}}}*/ +/* + * TODO: + * - support keeping the mirror file around (evil listclearer strikes again) + * - better method to download than having a pkgAcquire interface here + * - testing :) + * + */ + MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { diff --git a/methods/mirror.h b/methods/mirror.h index 91cf8c64f..a47b8fbf1 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -23,6 +23,7 @@ using std::endl; class MirrorMethod : public HttpMethod { FetchResult Res; + // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) string MirrorFile; // @@ -33,6 +34,8 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + + // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); -- cgit v1.2.3 From 3ed91a17d21d0d456a12accc507b82e1c0f1a697 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:21:47 +0100 Subject: * more todo items --- methods/mirror.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c49764044..1911d6115 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -28,10 +28,11 @@ using namespace std; /* * TODO: - * - support keeping the mirror file around (evil listclearer strikes again) * - better method to download than having a pkgAcquire interface here + * - support keeping the mirror file around (evil listclearer strikes again) + * -> /var/lib/apt/mirrors dir? how to cleanup? by time? + * - provide some TTL time until the mirror file is get again (1h? 6h?) * - testing :) - * */ MirrorMethod::MirrorMethod() -- cgit v1.2.3 From ef1e6d8822bb34527bba5d2318b5768013efd010 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Nov 2006 11:06:09 +0100 Subject: * methods/mirror.cc: - check the complette queue on Fetch() for urls to transform --- methods/mirror.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 1911d6115..4de981522 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,6 +32,7 @@ using namespace std; * - support keeping the mirror file around (evil listclearer strikes again) * -> /var/lib/apt/mirrors dir? how to cleanup? by time? * - provide some TTL time until the mirror file is get again (1h? 6h?) + * - deal with runing as non-root (we can't write to the lists dir then) * - testing :) */ @@ -109,8 +110,11 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - if(Queue->Uri.find("mirror://") != string::npos) - Queue->Uri.replace(0,BaseUri.size(),Mirror); + for (FetchItem *I = Queue; I != 0; I = I->Next) + { + if(I->Uri.find("mirror://") != string::npos) + I->Uri.replace(0,BaseUri.size(),Mirror); + } // now run the real fetcher return HttpMethod::Fetch(Itm); -- cgit v1.2.3 From d731f9c574138a9cb31c586a8f1e7d7181a44456 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:19:32 +0100 Subject: * apt-pkg/init.cc: - added Dir::State::Mirrors * doc/examples/configure-index: - added Acquire::mirror::RefreshInterval * methods/mirror.{cc,h}: - download the mirror file into Dir::State::Mirrors - added RefreshInterval option to not ask for the mirror file too often --- methods/mirror.cc | 56 +++++++++++++++++++++++++++++++++++++++++++++---------- methods/mirror.h | 1 + 2 files changed, 47 insertions(+), 10 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4de981522..e70a40f55 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -18,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -28,11 +29,16 @@ using namespace std; /* * TODO: + * - send expected checksum to the mirror method so that + some checking/falling back can be done here already + * - keep the mirror file around in /var/lib/apt/mirrors + * can't be put into lists/ because of the listclearer + * cleanup by time (mtime relative to the other mtimes) + * - use a TTL time the mirror file is fetched again (6h?) + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - support keeping the mirror file around (evil listclearer strikes again) - * -> /var/lib/apt/mirrors dir? how to cleanup? by time? - * - provide some TTL time until the mirror file is get again (1h? 6h?) - * - deal with runing as non-root (we can't write to the lists dir then) + * - magicmarker is (a bit) evil * - testing :) */ @@ -60,6 +66,12 @@ bool MirrorMethod::Configuration(string Message) } /*}}}*/ +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string dir) +{ + +} + bool MirrorMethod::GetMirrorFile(string uri) { @@ -69,7 +81,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { @@ -77,19 +89,43 @@ bool MirrorMethod::GetMirrorFile(string uri) cerr << "mirror-file: " << MirrorFile << endl; } - // FIXME: fetch it with curl + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + HasMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - - if(res) + if(res) HasMirrorFile = true; Fetcher.Shutdown(); - return true; + return res; } bool MirrorMethod::SelectMirror() { + // FIXME: make the mirror selection more clever, do not + // just use the first one! ifstream in(MirrorFile.c_str()); getline(in, Mirror); if(Debug) @@ -103,7 +139,7 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // get mirror information + // select mirror only once per session if(!HasMirrorFile) { GetMirrorFile(Itm->Uri); diff --git a/methods/mirror.h b/methods/mirror.h index a47b8fbf1..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -34,6 +34,7 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + bool Clean(string dir); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); -- cgit v1.2.3 From 70288656715d2fe4c2c33598124ae48f7bca9cdf Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:38:55 +0100 Subject: * methods/mirror.cc: - implemented simple Clean() implementation based on the time of the last access for a mirror file --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index e70a40f55..425a2f7f5 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -19,12 +19,14 @@ #include #include #include +#include +#include using namespace std; #include "mirror.h" #include "http.h" - +#include "apti18n.h" /*}}}*/ /* @@ -67,9 +69,54 @@ bool MirrorMethod::Configuration(string Message) /*}}}*/ // clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string dir) +bool MirrorMethod::Clean(string Dir) { + // FIXME: it would better to have a global idea of the mirrors + // in the sources.list and use this instead of this time + // based approach. currently apt does not support this :/ + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string StartDir = SafeGetCWD(); + if (chdir(Dir.c_str()) != 0) + { + closedir(D); + return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); + } + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,"lock") == 0 || + strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + // Del everything not touched for MaxAge days + time_t t,now,max; + struct stat buf; + if(stat(Dir->d_name, &buf) != 0) + { + cerr << "Can't stat '" << Dir->d_name << "'" << endl; + continue; + } + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); + if(t + max < now) + { + if(Debug) + clog << "Mirror file is older than MaxAge days, deleting" << endl; + unlink(Dir->d_name); + } + }; + chdir(StartDir.c_str()); + closedir(D); + return true; } @@ -81,6 +128,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); + // get new file MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) @@ -108,7 +156,7 @@ bool MirrorMethod::GetMirrorFile(string uri) return true; } if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; } // not that great to use pkgAcquire here, but we do not have @@ -142,6 +190,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) // select mirror only once per session if(!HasMirrorFile) { + Clean(_config->FindDir("Dir::State::mirrors")); GetMirrorFile(Itm->Uri); SelectMirror(); } -- cgit v1.2.3 From 0c312e0ed24fed69412c181634dd4515e9b1d46f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 12:10:05 +0100 Subject: * implemented proper mirror list cleanup --- methods/mirror.cc | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 425a2f7f5..00f7b7807 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -71,9 +72,14 @@ bool MirrorMethod::Configuration(string Message) // clean the mirrors dir based on ttl information bool MirrorMethod::Clean(string Dir) { - // FIXME: it would better to have a global idea of the mirrors - // in the sources.list and use this instead of this time - // based approach. currently apt does not support this :/ + vector::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + // read sources.list + pkgSourceList list; + list.ReadMainList(); DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -94,24 +100,21 @@ bool MirrorMethod::Clean(string Dir) strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; - - // Del everything not touched for MaxAge days - time_t t,now,max; - struct stat buf; - if(stat(Dir->d_name, &buf) != 0) + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) { - cerr << "Can't stat '" << Dir->d_name << "'" << endl; - continue; + string uri = (*I)->GetURI(); + if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + string BaseUri = uri.substr(0,uri.find(Marker)); + if (URItoFileName(BaseUri) == Dir->d_name) + break; } - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); - if(t + max < now) - { - if(Debug) - clog << "Mirror file is older than MaxAge days, deleting" << endl; + // nothing found, nuke it + if (I == list.end()) unlink(Dir->d_name); - } }; chdir(StartDir.c_str()); -- cgit v1.2.3 From 933833c51fbfe3dca3f3a3073d441e5856462605 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 13:07:22 +0100 Subject: * methods/mirror.cc: - updated the TODO --- methods/mirror.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 00f7b7807..f08b324ec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -34,14 +34,18 @@ using namespace std; * TODO: * - send expected checksum to the mirror method so that some checking/falling back can be done here already - * - keep the mirror file around in /var/lib/apt/mirrors - * can't be put into lists/ because of the listclearer - * cleanup by time (mtime relative to the other mtimes) - * - use a TTL time the mirror file is fetched again (6h?) + use pkgAcquire::Custom600Header() for this? what about gpgv + failures? + #OR# + * - implement it at the pkgAcquire::Item::Failed() level but then + we need to send back what uri exactly was failing + * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil + * - magicmarker is (a bit) evil, maybe just use a similar approach as in + clean and read the sources.list and use the GetURI() method to find + the prefix? * - testing :) */ -- cgit v1.2.3 From cae9cdcefc34eba6d023cb30cbbdb9bbf909b8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 16:35:56 +0100 Subject: * basic error reporting from apt in place now (ReportMirrorFailures()) --- methods/mirror.cc | 27 +++++++++++++++++++++------ methods/mirror.h | 1 + 2 files changed, 22 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index f08b324ec..428726a3d 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,14 +32,14 @@ using namespace std; /* * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - use pkgAcquire::Custom600Header() for this? what about gpgv - failures? + * - what about gpgv failures? better standard format for errors + to send back to LP #OR# * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing - + we need to send back what uri exactly was failing + [mvo: the problem with this approach is ::Failed() is not really + called for all failures :/ e.g. md5sum mismatch in a archive + is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here @@ -214,6 +214,9 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // FIXME: queue next mirror? + ReportMirrorFailure(Err); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -228,11 +231,23 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { + // FIXME: queue next mirror? + if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) + ReportMirrorFailure("499 Hash mismatch"); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } +void MirrorMethod::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed + std::cerr << "\nReportMirrorFailure: " + << Queue->Uri + << " FailCode: " + << FailCode << std::endl; +} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..3ff9e1a96 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,6 +35,7 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); + void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); -- cgit v1.2.3 From 2769f0bcf6c1ac73e883f47857fa227de92f4525 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 17:16:42 +0100 Subject: * mirror-failure.py: example mirror failure cgi * methods/mirror.cc: prepare for the failure submit --- methods/mirror.cc | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 428726a3d..6621d47e2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -247,6 +247,12 @@ void MirrorMethod::ReportMirrorFailure(string FailCode) << Queue->Uri << " FailCode: " << FailCode << std::endl; +#if 0 // FIXME: do not use system, make sure to properly encode + // URI/FailCode, do not hardcode the submit url + system("curl -d url=" + Queue->Uri + + " -d FailureCode=" + FailCode + + " http://localhost:8000/ &"); +#endif } int main() -- cgit v1.2.3 From 362d29343e5d25248bcd84300aa1b18effe891e7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Jan 2007 14:42:17 +0100 Subject: make the mirror failures actually produce a error message --- methods/mirror.cc | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 6621d47e2..8f9b8ed34 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -30,13 +30,18 @@ using namespace std; #include "apti18n.h" /*}}}*/ -/* +/* Done: + * - works with http only + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * * TODO: - * - what about gpgv failures? better standard format for errors - to send back to LP - #OR# - * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing + * what about gpgv failures? this should call-out to the problem reporting + script, but we need to know what mirror was used + * better standard format for errors to send back + * - implement failure reporting at the pkgAcquire::Item::Failed() level + but then we need to send back what uri exactly was failing [mvo: the problem with this approach is ::Failed() is not really called for all failures :/ e.g. md5sum mismatch in a archive is not] @@ -46,6 +51,7 @@ using namespace std; * - magicmarker is (a bit) evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? + * support more than http * - testing :) */ @@ -243,16 +249,34 @@ void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) void MirrorMethod::ReportMirrorFailure(string FailCode) { // report that Queue->Uri failed +#if 0 std::cerr << "\nReportMirrorFailure: " << Queue->Uri << " FailCode: " << FailCode << std::endl; -#if 0 // FIXME: do not use system, make sure to properly encode - // URI/FailCode, do not hardcode the submit url - system("curl -d url=" + Queue->Uri + - " -d FailureCode=" + FailCode + - " http://localhost:8000/ &"); #endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/lib/apt/report-mirror-failure"); + Args[i++] = report.c_str(); + Args[i++] = Queue->Uri.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } } int main() -- cgit v1.2.3 From 36280399db0ae203d3f1ae4d44b946f31e9a38ce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 1 Feb 2007 12:32:50 +0100 Subject: * commited the latest mirror failure detection code --- methods/connect.cc | 17 ++++++++--------- methods/mirror.cc | 45 ++++----------------------------------------- methods/mirror.h | 1 - 3 files changed, 12 insertions(+), 51 deletions(-) (limited to 'methods') diff --git a/methods/connect.cc b/methods/connect.cc index 8c2ac6d56..4e227c3fd 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -18,6 +18,7 @@ #include #include #include +#include // Internet stuff #include @@ -67,12 +68,10 @@ static bool DoConnect(struct addrinfo *Addr,string Host, wrong this will get tacked onto the end of the error message */ if (LastHostAddr->ai_next != 0) { - char Name2[NI_MAXHOST + NI_MAXSERV + 10]; - snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service); - Owner->SetFailExtraMsg(string(Name2)); - } - else - Owner->SetFailExtraMsg(""); + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"),Name,Service); + Owner->SetIP(ss.str()); + } // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, @@ -89,7 +88,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, /* This implements a timeout for connect by opening the connection nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) { - Owner->SetFailExtraMsg("\nFailReason: Timeout"); + Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " "connection timed out"),Host.c_str(),Service,Name); } @@ -104,7 +103,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, { errno = Err; if(errno == ECONNREFUSED) - Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused"); + Owner->SetFailReason("ConnectionRefused"); return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); } @@ -169,7 +168,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, if (Res == EAI_AGAIN) { - Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure"); + Owner->SetFailReason("TmpResolveFailure"); return _error->Error(_("Temporary failure resolving '%s'"), Host.c_str()); } diff --git a/methods/mirror.cc b/methods/mirror.cc index 8f9b8ed34..8ccfb8559 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -38,7 +38,7 @@ using namespace std; * * TODO: * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used + script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() * better standard format for errors to send back * - implement failure reporting at the pkgAcquire::Item::Failed() level but then we need to send back what uri exactly was failing @@ -48,7 +48,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil, maybe just use a similar approach as in + * - magicmarker is evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? * support more than http @@ -191,6 +191,8 @@ bool MirrorMethod::SelectMirror() getline(in, Mirror); if(Debug) cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; return true; } @@ -220,9 +222,6 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // FIXME: queue next mirror? - ReportMirrorFailure(Err); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -237,47 +236,11 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - // FIXME: queue next mirror? - if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) - ReportMirrorFailure("499 Hash mismatch"); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } -void MirrorMethod::ReportMirrorFailure(string FailCode) -{ - // report that Queue->Uri failed -#if 0 - std::cerr << "\nReportMirrorFailure: " - << Queue->Uri - << " FailCode: " - << FailCode << std::endl; -#endif - const char *Args[40]; - unsigned int i = 0; - string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/lib/apt/report-mirror-failure"); - Args[i++] = report.c_str(); - Args[i++] = Queue->Uri.c_str(); - Args[i++] = FailCode.c_str(); - pid_t pid = ExecFork(); - if(pid < 0) - { - _error->Error("ReportMirrorFailure Fork failed"); - return; - } - else if(pid == 0) - { - execvp(report.c_str(), (char**)Args); - } - if(!ExecWait(pid, "report-mirror-failure")) - { - _error->Warning("Couldn't report problem to '%s'", - _config->Find("Acquire::Mirror::ReportFailures").c_str()); - } -} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 3ff9e1a96..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,7 +35,6 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); - void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); -- cgit v1.2.3 From 59271f62e4a291c8d96e1f6073203c395734b6ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 17:52:28 +0100 Subject: * use pkgAcqMethod::FailReason() for consistent error reporting --- methods/connect.cc | 1 + methods/http.cc | 3 +++ methods/mirror.cc | 12 +++--------- 3 files changed, 7 insertions(+), 9 deletions(-) (limited to 'methods') diff --git a/methods/connect.cc b/methods/connect.cc index 4e227c3fd..145001fb3 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -163,6 +163,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, DefPort = 0; continue; } + Owner->SetFailReason("ResolveFailure"); return _error->Error(_("Could not resolve '%s'"),Host.c_str()); } diff --git a/methods/http.cc b/methods/http.cc index deaa8d0c8..01ad14655 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,6 +914,9 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) failure */ if (Srv->Result < 200 || Srv->Result >= 300) { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result); + SetFailReason(err); _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) return 4; diff --git a/methods/mirror.cc b/methods/mirror.cc index 8ccfb8559..b64879bec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -31,20 +31,14 @@ using namespace std; /*}}}*/ /* Done: - * - works with http only + * - works with http (only!) * - always picks the first mirror from the list * - call out to problem reporting script * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - use pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP * * TODO: - * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() - * better standard format for errors to send back - * - implement failure reporting at the pkgAcquire::Item::Failed() level - but then we need to send back what uri exactly was failing - [mvo: the problem with this approach is ::Failed() is not really - called for all failures :/ e.g. md5sum mismatch in a archive - is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here -- cgit v1.2.3 From 066b53e95ba224f81d078e7d8df3002275f8a092 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 18:46:45 +0100 Subject: * get rid of magic mirror --- methods/mirror.cc | 62 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b64879bec..ff91130b8 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,22 +42,13 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is evil, maybe just use a similar approach as in - clean and read the sources.list and use the GetURI() method to find - the prefix? - * support more than http + * - support more than http * - testing :) */ MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { -#if 0 - HasMirrorFile=true; - BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; - MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; - Mirror="http://de.archive.ubuntu.com/ubuntu/"; -#endif }; // HttpMethod::Configuration - Handle a configuration message /*{{{*/ @@ -111,8 +102,7 @@ bool MirrorMethod::Clean(string Dir) string uri = (*I)->GetURI(); if(uri.substr(0,strlen("mirror://")) != string("mirror://")) continue; - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - string BaseUri = uri.substr(0,uri.find(Marker)); + string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) break; } @@ -127,11 +117,51 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string uri) +bool MirrorMethod::GetMirrorFile(string mirror_uri_str) { - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - BaseUri = uri.substr(0,uri.find(Marker)); - + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); -- cgit v1.2.3 From f0b509cdb44cb5e79e9c5ddd7ebec46965138534 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 19:06:42 +0100 Subject: * apt-pkg/acquire-item.cc: - use FailReason in pkgAcquire::Item::Failed when available * methods/mirror.cc: - move some debug output into if(Debug) --- methods/mirror.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ff91130b8..c5c0c7461 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -143,7 +143,8 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + if(Debug) + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -152,13 +153,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) for(I=list.begin(); I != list.end(); I++) { string uristr = (*I)->GetURI(); - std::cerr << "Checking: " << uristr << std::endl; + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) continue; // find matching uri in sources.list if(mirror_uri_str.substr(0,uristr.size()) == uristr) { - std::cerr << "found BaseURI: " << uristr << std::endl; + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; BaseUri = uristr.substr(0,uristr.size()-1); } } -- cgit v1.2.3 From 3f599bb721c4ac58d8ff18991c9704b5f30eaa2b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 10:49:25 +0100 Subject: * apt-pkg/acquire-item.cc: - default to "/usr/lib/apt/apt-report-mirror-failure" * cmdline/apt-report-mirror-failure: - no default comit url for now * debian/rules: - move apt-report-mirror-failure into /usr/lib/apt * doc/examples/configure-index: - more documentation * methods/mirror.cc: - updated TODO --- methods/mirror.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c5c0c7461..9a86a10c2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,6 +42,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here + * and better error handling there! * - support more than http * - testing :) */ -- cgit v1.2.3 From 68acfb437f0c6c2e026be6fd3b7da3fc8d64c523 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 16:37:20 +0100 Subject: * ABI version bumped --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index f971e56e8..d66ba11e3 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.51 +LIB_APT_PKG_MAJOR = 3.52 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From d715b9c969264eec96913b577f63a3e9e498b2e6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 21 Feb 2007 09:59:53 +0100 Subject: * fix in the pkgRecords destructor * Bump ABI version --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index d66ba11e3..610bd5b49 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.52 +LIB_APT_PKG_MAJOR = 3.53 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From d530b4c24fc452efc82be4b7e1ebc536a00c09b7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 3 Oct 2007 02:21:52 +0200 Subject: * Reset curl options and timestamp between downloaded files. Thanks to Ryan Murray for the patch --- methods/https.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index e6717e63a..3b2b0bb19 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -115,6 +115,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // - error checking/reporting // - more debug options? (CURLOPT_DEBUGFUNCTION?) + curl_easy_reset(curl); SetupProxy(); // callbacks @@ -125,6 +126,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FILETIME, true); // FIXME: https: offer various options of verification bool peer_verify = _config->FindB("Acquire::https::Verify-Peer", false); @@ -202,6 +204,9 @@ bool HttpsMethod::Fetch(FetchItem *Itm) CURLcode success = curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_responsecode); + long curl_servdate; + curl_easy_getinfo(curl, CURLINFO_FILETIME, &curl_servdate); + // cleanup if(success != 0) { @@ -215,6 +220,14 @@ bool HttpsMethod::Fetch(FetchItem *Itm) if (Res.Size == 0) Res.Size = File->Size(); + // Timestamp + struct utimbuf UBuf; + if (curl_servdate != -1) { + UBuf.actime = curl_servdate; + UBuf.modtime = curl_servdate; + utime(File->Name().c_str(),&UBuf); + } + // check the downloaded result struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) -- cgit v1.2.3 From 3de6d4585be0b8a9bd98b5fb1d62ff922537c435 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Oct 2007 07:50:24 +0200 Subject: * methods/https.cc: - fix off-by-one error I-M-S handling - cleanup after I-M-S hit - only send LastModified if we actually have a file - delete zero size I-M-S hits --- methods/https.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3b2b0bb19..b2bbbddb1 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -161,13 +161,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - // set time values - if(Itm->LastModified > 0) - { - curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); - } - // speed limit int dlLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024; if (dlLimit > 0) @@ -183,7 +176,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // error handling curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - // In this case we send an if-range query with a range header + // if we have the file send an if-range query with a range header if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { char Buf[1000]; @@ -191,11 +184,17 @@ bool HttpsMethod::Fetch(FetchItem *Itm) (long)SBuf.st_size - 1, TimeRFC1123(SBuf.st_mtime).c_str()); headers = curl_slist_append(headers, Buf); + } + else if(Itm->LastModified > 0) + { + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); } // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); - File->Seek(File->Size()); + if (File->Size() > 0) + File->Seek(File->Size() - 1); // keep apt updated Res.Filename = Itm->DestFile; @@ -217,9 +216,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } File->Close(); - if (Res.Size == 0) - Res.Size = File->Size(); - // Timestamp struct utimbuf UBuf; if (curl_servdate != -1) { @@ -232,15 +228,19 @@ bool HttpsMethod::Fetch(FetchItem *Itm) struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) { - Res.Size = Buf.st_size; Res.Filename = File->Name(); Res.LastModified = Buf.st_mtime; Res.IMSHit = false; if (curl_responsecode == 304) { + unlink(File->Name().c_str()); Res.IMSHit = true; Res.LastModified = Itm->LastModified; + Res.Size = 0; + URIDone(Res); + return true; } + Res.Size = Buf.st_size; } // take hashes -- cgit v1.2.3 From 38eedeb77790c106eb35940e3d7835a7e0f0297e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Nov 2007 15:54:17 +0100 Subject: methods/mirror.{cc,h}: - improve the mirror method so that it only updates the mirror list from the server if we get new indexfiles - code cleanup --- methods/mirror.cc | 152 +++++++++++++++++++++++++++++++++--------------------- methods/mirror.h | 7 +-- 2 files changed, 97 insertions(+), 62 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 9a86a10c2..43e56c71a 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -48,7 +48,7 @@ using namespace std; */ MirrorMethod::MirrorMethod() - : HttpMethod(), HasMirrorFile(false) + : HttpMethod(), DownloadedMirrorFile(false) { }; @@ -73,6 +73,9 @@ bool MirrorMethod::Clean(string Dir) if(Debug) clog << "MirrorMethod::Clean(): " << Dir << endl; + if(Dir == "/") + return _error->Error("will not clean: '/'"); + // read sources.list pkgSourceList list; list.ReadMainList(); @@ -118,7 +121,71 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string mirror_uri_str) +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) +{ + + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + DownloadedMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); + + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} + +bool MirrorMethod::SelectMirror() +{ + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + + // FIXME: make the mirror selection more clever, do not + // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) { /* - a mirror_uri_str looks like this: @@ -144,8 +211,9 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ + string name; if(Debug) - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -166,62 +234,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) BaseUri = uristr.substr(0,uristr.size()-1); } } - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - // get new file - MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; - } - - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - HasMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + cerr << "mirror-file: " << name << endl; } - - // not that great to use pkgAcquire here, but we do not have - // any other way right now - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) - HasMirrorFile = true; - Fetcher.Shutdown(); - return res; -} - -bool MirrorMethod::SelectMirror() -{ - // FIXME: make the mirror selection more clever, do not - // just use the first one! - ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; - return true; + return name; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -230,20 +251,33 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // select mirror only once per session - if(!HasMirrorFile) + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - GetMirrorFile(Itm->Uri); - SelectMirror(); + DownloadMirrorFile(Itm->Uri); } + if(Mirror.empty()) + SelectMirror(); + for (FetchItem *I = Queue; I != 0; I = I->Next) { if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(),Mirror); + I->Uri.replace(0,BaseUri.size(), Mirror); } - + // now run the real fetcher return HttpMethod::Fetch(Itm); }; diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..ed817806b 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,13 +26,14 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) - string MirrorFile; // - bool HasMirrorFile; + string MirrorFile; // the file that contains the list of mirrors + bool DownloadedMirrorFile; // already downloaded this session bool Debug; protected: - bool GetMirrorFile(string uri); + bool DownloadMirrorFile(string uri); + string GetMirrorFileName(string uri); bool SelectMirror(); bool Clean(string dir); -- cgit v1.2.3 From 0df1d3ac4312a2051b10478cf1c8b242864cb3b1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 6 Dec 2007 19:47:36 +0100 Subject: we do not break the abi --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 085b357b8..3c12bb974 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.5 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From 5148def81fb20b22fc9ec54bc212e34260fbfebb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 7 Dec 2007 22:54:41 +0100 Subject: update ABI, the homepage change broke it --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 3c12bb974..085b357b8 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.5 +LIB_APT_PKG_MAJOR = 4.6 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From d7bab8f1142e474ac4bafd139e93439aeddfd546 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 3 Jan 2008 15:22:44 +0100 Subject: methods/makefile: remove duplicated entry for https --- methods/makefile | 7 ------- 1 file changed, 7 deletions(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 085b357b8..f765a4d37 100644 --- a/methods/makefile +++ b/methods/makefile @@ -59,13 +59,6 @@ LIB_MAKES = apt-pkg/makefile SOURCE = https.cc include $(PROGRAM_H) -# The https method -PROGRAM=https -SLIBS = -lapt-pkg -lcurl -LIB_MAKES = apt-pkg/makefile -SOURCE = https.cc -include $(PROGRAM_H) - # The ftp method PROGRAM=ftp SLIBS = -lapt-pkg $(SOCKETLIBS) -- cgit v1.2.3 From 484befd1518245e795ed92e6c97ad50786f3b4a2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 11 Mar 2008 09:59:32 +0100 Subject: * typo fixes (LP: #107960) --- methods/http.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 6aa4261ff..61321c850 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -367,8 +367,8 @@ bool ServerState::Close() /*}}}*/ // ServerState::RunHeaders - Get the headers before the data /*{{{*/ // --------------------------------------------------------------------- -/* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header - parse error occured */ +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ int ServerState::RunHeaders() { State = Header; -- cgit v1.2.3 From 236527f0fffe30fc8988e17d926778c23ac1f902 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Jan 2009 17:39:37 +0100 Subject: LP: #263089 --- methods/gpgv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 9f4683e6e..f3277b300 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -264,7 +264,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // least one bad signature. good signatures and NoPubKey signatures // happen easily when a file is signed with multiple signatures if(GoodSigners.empty() or !BadSigners.empty()) - return _error->Error(errmsg.c_str()); + return _error->Error("%s", errmsg.c_str()); } // Just pass the raw output up, because passing it as a real data -- cgit v1.2.3 From ebb461fdf02ee3e038d4b3a4ab1a0a60188edf9a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Feb 2009 12:14:36 +0100 Subject: * [ABI break] merge support for http redirects, thanks to Jeff Licquia and Anthony Towns * [ABI break] use int for the package IDs (thanks to Steve Cotton) - Galician updated. Closes: #509151 - Catalan updated. Closes: #509375 - Vietnamese updated. Closes: #509422 - Traditional Chinese added. Closes: #510664 * COPYING: - Actualized. Removed obsolete Qt section, added GPLv2 clause. (Closes: #440049, #509337) * Clarify the --help for 'purge' (LP: #243948) --- methods/http.cc | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- methods/http.h | 3 +++ methods/makefile | 2 +- 3 files changed, 81 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index b0fb89fda..44274bd78 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include // Internet stuff @@ -56,6 +57,7 @@ int HttpMethod::FailFd = -1; time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; +bool AllowRedirect = false; bool Debug = false; URI Proxy; @@ -627,6 +629,12 @@ bool ServerState::HeaderLine(string Line) return true; } + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + return true; } /*}}}*/ @@ -899,7 +907,9 @@ bool HttpMethod::ServerDie(ServerState *Srv) 1 - IMS hit 3 - Unrecoverable error 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) */ + 5 - Unrecoverable non-server error (close the connection) + 6 - Try again with a new or changed URI + */ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified @@ -911,6 +921,27 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) return 1; } + /* Redirect + * + * Note that it is only OK for us to treat all redirection the same + * because we *always* use GET, not other HTTP methods. There are + * three redirection codes for which it is not appropriate that we + * redirect. Pass on those codes so the error handling kicks in. + */ + if (AllowRedirect + && (Srv->Result > 300 && Srv->Result < 400) + && (Srv->Result != 300 // Multiple Choices + && Srv->Result != 304 // Not Modified + && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved) + { + if (!Srv->Location.empty()) + { + NextURI = Srv->Location; + return 6; + } + /* else pass through for error message */ + } + /* We have a reply we dont handle. This should indicate a perm server failure */ if (Srv->Result < 200 || Srv->Result >= 300) @@ -1028,6 +1059,7 @@ bool HttpMethod::Configuration(string Message) if (pkgAcqMethod::Configuration(Message) == false) return false; + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); @@ -1041,6 +1073,10 @@ bool HttpMethod::Configuration(string Message) /* */ int HttpMethod::Loop() { + typedef vector StringVector; + typedef vector::iterator StringVectorIterator; + map Redirected; + signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -1227,6 +1263,46 @@ int HttpMethod::Loop() break; } + // Try again with a new URL + case 6: + { + // Clear rest of response if there is content + if (Server->HaveContent) + { + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(); + delete File; + File = 0; + } + + /* Detect redirect loops. No more redirects are allowed + after the same URI is seen twice in a queue item. */ + StringVector &R = Redirected[Queue->DestFile]; + bool StopRedirects = false; + if (R.size() == 0) + R.push_back(Queue->Uri); + else if (R[0] == "STOP" || R.size() > 10) + StopRedirects = true; + else + { + for (StringVectorIterator I = R.begin(); I != R.end(); I++) + if (Queue->Uri == *I) + { + R[0] = "STOP"; + break; + } + + R.push_back(Queue->Uri); + } + + if (StopRedirects == false) + Redirect(NextURI); + else + Fail(); + + break; + } + default: Fail(_("Internal error")); break; diff --git a/methods/http.h b/methods/http.h index dec5cd80f..bc076e1f8 100644 --- a/methods/http.h +++ b/methods/http.h @@ -99,6 +99,7 @@ struct ServerState enum {Chunked,Stream,Closes} Encoding; enum {Header, Data} State; bool Persistent; + string Location; // This is a Persistent attribute of the server itself. bool Pipeline; @@ -145,6 +146,8 @@ class HttpMethod : public pkgAcqMethod protected: virtual bool Fetch(FetchItem *); + string NextURI; + public: friend class ServerState; diff --git a/methods/makefile b/methods/makefile index 5c4fa82bf..1d022be90 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.7 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From 8872c430df84660fa994db0ebfa67861239fe739 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:36:36 +0100 Subject: * methods/https.cc: - add Acquire::https::AllowRedirect support --- methods/https.cc | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 98dfeefa1..87de54589 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -208,6 +208,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); + // set redirect options and default to 10 redirects + bool AllowRedirect = _config->FindI("Acquire::https::AllowRedirect", true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); + // debug if(_config->FindB("Debug::Acquire::https", false)) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); -- cgit v1.2.3 From 7e522e9e0e4808202061e0a73345bd8b18eccde3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:38:11 +0100 Subject: * apt-pkg/pkgcache.cc: - do not run "dpkg --configure pkg" if pkg is in trigger-awaited state (LP: #322955) - do not unlink files in partial/ (thanks to robbiew) --- methods/https.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 87de54589..8bf44b52a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -253,7 +253,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // cleanup if(success != 0) { - unlink(File->Name().c_str()); _error->Error("%s", curl_errorstr); Fail(); return true; -- cgit v1.2.3 From 5dad4134da37576de12721f34165c1fe4d138b3c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 19 Mar 2009 15:42:25 +0100 Subject: * methods/mirror.cc: - when download the mirror file and the server is down, return a propper error message (LP: #278635) --- methods/mirror.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index bdd783cc7..b3a956b95 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -123,6 +123,8 @@ bool MirrorMethod::Clean(string Dir) bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) { + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): " << endl; // check the file, if it is not older than RefreshInterval just use it // otherwise try to get a new one @@ -251,6 +253,9 @@ string MirrorMethod::GetMirrorFileName(string mirror_uri_str) depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { + if(Debug) + clog << "MirrorMethod::Fetch()" << endl; + // the http method uses Fetch(0) as a way to update the pipeline, // just let it do its work in this case - Fetch() with a valid // Itm will always run before the first Fetch(0) @@ -269,8 +274,17 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) - SelectMirror(); + if(Mirror.empty()) { + if(!SelectMirror()) { + // no valid mirror selected, something went wrong downloading + // from the master mirror site most likely and there is + // no old mirror file availalbe + return false; + } + } + if(Debug) + clog << "selected mirror: " << Mirror << endl; + for (FetchItem *I = Queue; I != 0; I = I->Next) { -- cgit v1.2.3 From 0b77f4775db7bc45964e0337b8978a170b3f0483 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Apr 2009 22:52:11 +0200 Subject: * methods/gpgv.cc: - properly check for expired and revoked keys (closes: #433091) LP: #356012 --- methods/gpgv.cc | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index f3277b300..150c1d315 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -17,13 +17,18 @@ #define GNUPGBADSIG "[GNUPG:] BADSIG" #define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" #define GNUPGVALIDSIG "[GNUPG:] VALIDSIG" +#define GNUPGGOODSIG "[GNUPG:] GOODSIG" +#define GNUPGKEYEXPIRED "[GNUPG:] KEYEXPIRED" +#define GNUPGREVKEYSIG "[GNUPG:] REVKEYSIG" #define GNUPGNODATA "[GNUPG:] NODATA" class GPGVMethod : public pkgAcqMethod { private: string VerifyGetSigners(const char *file, const char *outfile, - vector &GoodSigners, vector &BadSigners, + vector &GoodSigners, + vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners); protected: @@ -37,6 +42,7 @@ class GPGVMethod : public pkgAcqMethod string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector &GoodSigners, vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners) { // setup a (empty) stringstream for formating the return value @@ -179,15 +185,27 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, std::cerr << "Got NODATA! " << std::endl; BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); } - if (strncmp(buffer, GNUPGVALIDSIG, sizeof(GNUPGVALIDSIG)-1) == 0) + if (strncmp(buffer, GNUPGKEYEXPIRED, sizeof(GNUPGKEYEXPIRED)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got KEYEXPIRED! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGREVKEYSIG, sizeof(GNUPGREVKEYSIG)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got REVKEYSIG! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGGOODSIG, sizeof(GNUPGGOODSIG)-1) == 0) { char *sig = buffer + sizeof(GNUPGPREFIX); - char *p = sig + sizeof("VALIDSIG"); + char *p = sig + sizeof("GOODSIG"); while (*p && isxdigit(*p)) p++; *p = 0; if (_config->FindB("Debug::Acquire::gpgv", false)) - std::cerr << "Got VALIDSIG, key ID:" << sig << std::endl; + std::cerr << "Got GOODSIG, key ID:" << sig << std::endl; GoodSigners.push_back(string(sig)); } } @@ -227,6 +245,8 @@ bool GPGVMethod::Fetch(FetchItem *Itm) string keyID; vector GoodSigners; vector BadSigners; + // a worthless signature is a expired or revoked one + vector WorthlessSigners; vector NoPubKeySigners; FetchResult Res; @@ -235,13 +255,14 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // Run gpgv on file, extract contents and get the key ID of the signer string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), - GoodSigners, BadSigners, NoPubKeySigners); + GoodSigners, BadSigners, WorthlessSigners, + NoPubKeySigners); if (GoodSigners.empty() || !BadSigners.empty() || !NoPubKeySigners.empty()) { string errmsg; // In this case, something bad probably happened, so we just go // with what the other method gave us for an error message. - if (BadSigners.empty() && NoPubKeySigners.empty()) + if (BadSigners.empty() && WorthlessSigners.empty() && NoPubKeySigners.empty()) errmsg = msg; else { @@ -252,6 +273,13 @@ bool GPGVMethod::Fetch(FetchItem *Itm) I != BadSigners.end(); I++) errmsg += (*I + "\n"); } + if (!WorthlessSigners.empty()) + { + errmsg += _("The following signatures were invalid:\n"); + for (vector::iterator I = WorthlessSigners.begin(); + I != WorthlessSigners.end(); I++) + errmsg += (*I + "\n"); + } if (!NoPubKeySigners.empty()) { errmsg += _("The following signatures couldn't be verified because the public key is not available:\n"); -- cgit v1.2.3 From 217d575b5713f4d9275177b58a36c581a7880c03 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 14 Jul 2009 11:22:41 +0200 Subject: * apt-pkg/acquire-worker.cc: - show error details of failed methods * apt-pkg/contrib/fileutl.cc: - if a process aborts with signal, show signal number * methods/http.cc: - ignore SIGPIPE, we deal with EPIPE from write in HttpMethod::ServerDie() (LP: #385144) --- methods/http_main.cc | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'methods') diff --git a/methods/http_main.cc b/methods/http_main.cc index 2c46ab19d..7815c2fc1 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,6 @@ #include #include +#include #include "connect.h" #include "rfc2553emu.h" @@ -10,6 +11,10 @@ int main() { setlocale(LC_ALL, ""); + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + HttpMethod Mth; return Mth.Loop(); } -- cgit v1.2.3 From 056539eeb8eb84cea65c0c15466111b978ee156d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Jul 2009 18:25:16 +0200 Subject: methods/http.cc: remove merge artifact --- methods/http.cc | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 476fb751f..859dbd39c 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -1316,15 +1316,4 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - - HttpMethod Mth; - return Mth.Loop(); -} - -- cgit v1.2.3