From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/http.cc | 13 ++----- methods/http.h | 7 ++-- methods/http_main.cc | 15 ++++++++ methods/makefile | 9 ++++- methods/mirror.cc | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ methods/mirror.h | 40 ++++++++++++++++++++++ 6 files changed, 166 insertions(+), 15 deletions(-) create mode 100644 methods/http_main.cc create mode 100644 methods/mirror.cc create mode 100644 methods/mirror.h (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c6623c46f..deaa8d0c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,7 +38,6 @@ #include #include #include -#include #include // Internet stuff @@ -57,7 +56,7 @@ time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; bool Debug = false; - +URI Proxy; unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; @@ -990,7 +989,7 @@ void HttpMethod::SigTerm(int) depth. */ bool HttpMethod::Fetch(FetchItem *) { - if (Server == 0) + if (Server == 0) return true; // Queue the requests @@ -1223,13 +1222,5 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - - HttpMethod Mth; - - return Mth.Loop(); -} diff --git a/methods/http.h b/methods/http.h index 541e2952c..5eac11401 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 -#include + using std::cout; using std::endl; @@ -134,7 +134,6 @@ class HttpMethod : public pkgAcqMethod bool ServerDie(ServerState *Srv); int DealWithHeaders(FetchResult &Res,ServerState *Srv); - virtual bool Fetch(FetchItem *); virtual bool Configuration(string Message); // In the event of a fatal signal this file will be closed and timestamped. @@ -142,6 +141,9 @@ class HttpMethod : public pkgAcqMethod static int FailFd; static time_t FailTime; static void SigTerm(int); + + protected: + virtual bool Fetch(FetchItem *); public: friend class ServerState; @@ -158,6 +160,5 @@ class HttpMethod : public pkgAcqMethod }; }; -URI Proxy; #endif diff --git a/methods/http_main.cc b/methods/http_main.cc new file mode 100644 index 000000000..2c46ab19d --- /dev/null +++ b/methods/http_main.cc @@ -0,0 +1,15 @@ +#include +#include + +#include "connect.h" +#include "rfc2553emu.h" +#include "http.h" + + +int main() +{ + setlocale(LC_ALL, ""); + + HttpMethod Mth; + return Mth.Loop(); +} diff --git a/methods/makefile b/methods/makefile index 1e3b1ef85..12e446a55 100644 --- a/methods/makefile +++ b/methods/makefile @@ -49,7 +49,7 @@ include $(PROGRAM_H) PROGRAM=http SLIBS = -lapt-pkg $(SOCKETLIBS) LIB_MAKES = apt-pkg/makefile -SOURCE = http.cc rfc2553emu.cc connect.cc +SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) # The ftp method @@ -66,6 +66,13 @@ LIB_MAKES = apt-pkg/makefile SOURCE = rsh.cc include $(PROGRAM_H) +# The mirror method +PROGRAM=mirror +SLIBS = -lapt-pkg $(SOCKETLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc +include $(PROGRAM_H) + # SSH and vzip2 method symlink binary: $(BIN)/ssh $(BIN)/bzip2 veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 000000000..fad076e91 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,97 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + Mirror Aquire Method - This is the Mirror aquire method for APT. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include +#include + +#include +#include +using namespace std; + +#include "mirror.h" +#include "http.h" + + /*}}}*/ + +MirrorMethod::MirrorMethod() + : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) +{ +#if 0 + HasMirrorFile=true; + BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + Mirror="http://de.archive.ubuntu.com/ubuntu/"; +#endif +}; + +bool MirrorMethod::GetMirrorFile(string uri) +{ + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + BaseUri = uri.substr(0,uri.find(Marker)); + BaseUri.replace(0,strlen("mirror://"),"http://"); + + MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + + // FIXME: fetch it with curl + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + cerr << "fetch-result: " << res << endl; + + if(res) + HasMirrorFile = true; + Fetcher.Shutdown(); + return true; +} + +bool MirrorMethod::SelectMirror() +{ + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + cerr << "Mirror: " << Mirror << endl; +} + +// MirrorMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool MirrorMethod::Fetch(FetchItem *Itm) +{ + // get mirror information + if(!HasMirrorFile) + { + GetMirrorFile(Itm->Uri); + SelectMirror(); + } + + // change the items in the queue + Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); + cerr << "new Fetch-uri: " << Itm->Uri << endl; + + // FIXME: fetch it with! + +}; + +int main() +{ + setlocale(LC_ALL, ""); + + MirrorMethod Mth; + + return Mth.Run(); +} + + diff --git a/methods/mirror.h b/methods/mirror.h new file mode 100644 index 000000000..ca3428830 --- /dev/null +++ b/methods/mirror.h @@ -0,0 +1,40 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + MIRROR Aquire Method - This is the MIRROR aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_MIRROR_H +#define APT_MIRROR_H + + +#include + +using std::cout; +using std::cerr; +using std::endl; + +#include "http.h" + +class MirrorMethod : public pkgAcqMethod +{ + FetchResult Res; + string Mirror; + string BaseUri; + string MirrorFile; + bool HasMirrorFile; + + protected: + bool GetMirrorFile(string uri); + bool SelectMirror(); + public: + MirrorMethod(); + virtual bool Fetch(FetchItem *Itm); +}; + + +#endif -- cgit v1.2.3 From 14e097c105c49a102a642c8108979356c5cc3152 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 22:59:35 +0100 Subject: * working mirror implementation based on http method --- methods/mirror.cc | 72 ++++++++++++++++++++++++++++++++++++++++++++----------- methods/mirror.h | 15 ++++++++---- 2 files changed, 69 insertions(+), 18 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index fad076e91..c4d911a3c 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace std; #include "mirror.h" @@ -25,31 +27,50 @@ using namespace std; /*}}}*/ MirrorMethod::MirrorMethod() - : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) + : HttpMethod(), HasMirrorFile(false) { #if 0 HasMirrorFile=true; - BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; + MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; Mirror="http://de.archive.ubuntu.com/ubuntu/"; #endif }; +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + + bool MirrorMethod::GetMirrorFile(string uri) { string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); BaseUri = uri.substr(0,uri.find(Marker)); - BaseUri.replace(0,strlen("mirror://"),"http://"); + + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + } // FIXME: fetch it with curl pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - cerr << "fetch-result: " << res << endl; if(res) HasMirrorFile = true; @@ -61,7 +82,9 @@ bool MirrorMethod::SelectMirror() { ifstream in(MirrorFile.c_str()); getline(in, Mirror); - cerr << "Mirror: " << Mirror << endl; + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + return true; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -77,21 +100,42 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - // change the items in the queue - Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); - cerr << "new Fetch-uri: " << Itm->Uri << endl; + if(Queue->Uri.find("mirror://") != string::npos) + Queue->Uri.replace(0,BaseUri.size(),Mirror); - // FIXME: fetch it with! - + // now run the real fetcher + return HttpMethod::Fetch(Itm); }; +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + int main() { setlocale(LC_ALL, ""); MirrorMethod Mth; - return Mth.Run(); + return Mth.Loop(); } diff --git a/methods/mirror.h b/methods/mirror.h index ca3428830..91cf8c64f 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -20,17 +20,24 @@ using std::endl; #include "http.h" -class MirrorMethod : public pkgAcqMethod +class MirrorMethod : public HttpMethod { FetchResult Res; - string Mirror; - string BaseUri; - string MirrorFile; + string BaseUri; // the original mirror://... url + string Mirror; // the selected mirror uri (http://...) + string MirrorFile; // bool HasMirrorFile; + bool Debug; + protected: bool GetMirrorFile(string uri); bool SelectMirror(); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual bool Configuration(string Message); + public: MirrorMethod(); virtual bool Fetch(FetchItem *Itm); -- cgit v1.2.3 From 86c17f0a4bd3c0e3dde5bb58b3df9c41e5a88763 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:09:26 +0100 Subject: * todo added --- methods/mirror.cc | 8 ++++++++ methods/mirror.h | 3 +++ 2 files changed, 11 insertions(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c4d911a3c..c49764044 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -26,6 +26,14 @@ using namespace std; /*}}}*/ +/* + * TODO: + * - support keeping the mirror file around (evil listclearer strikes again) + * - better method to download than having a pkgAcquire interface here + * - testing :) + * + */ + MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { diff --git a/methods/mirror.h b/methods/mirror.h index 91cf8c64f..a47b8fbf1 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -23,6 +23,7 @@ using std::endl; class MirrorMethod : public HttpMethod { FetchResult Res; + // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) string MirrorFile; // @@ -33,6 +34,8 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + + // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); -- cgit v1.2.3 From 3ed91a17d21d0d456a12accc507b82e1c0f1a697 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:21:47 +0100 Subject: * more todo items --- methods/mirror.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c49764044..1911d6115 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -28,10 +28,11 @@ using namespace std; /* * TODO: - * - support keeping the mirror file around (evil listclearer strikes again) * - better method to download than having a pkgAcquire interface here + * - support keeping the mirror file around (evil listclearer strikes again) + * -> /var/lib/apt/mirrors dir? how to cleanup? by time? + * - provide some TTL time until the mirror file is get again (1h? 6h?) * - testing :) - * */ MirrorMethod::MirrorMethod() -- cgit v1.2.3 From ef1e6d8822bb34527bba5d2318b5768013efd010 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Nov 2006 11:06:09 +0100 Subject: * methods/mirror.cc: - check the complette queue on Fetch() for urls to transform --- methods/mirror.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 1911d6115..4de981522 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,6 +32,7 @@ using namespace std; * - support keeping the mirror file around (evil listclearer strikes again) * -> /var/lib/apt/mirrors dir? how to cleanup? by time? * - provide some TTL time until the mirror file is get again (1h? 6h?) + * - deal with runing as non-root (we can't write to the lists dir then) * - testing :) */ @@ -109,8 +110,11 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - if(Queue->Uri.find("mirror://") != string::npos) - Queue->Uri.replace(0,BaseUri.size(),Mirror); + for (FetchItem *I = Queue; I != 0; I = I->Next) + { + if(I->Uri.find("mirror://") != string::npos) + I->Uri.replace(0,BaseUri.size(),Mirror); + } // now run the real fetcher return HttpMethod::Fetch(Itm); -- cgit v1.2.3 From d731f9c574138a9cb31c586a8f1e7d7181a44456 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:19:32 +0100 Subject: * apt-pkg/init.cc: - added Dir::State::Mirrors * doc/examples/configure-index: - added Acquire::mirror::RefreshInterval * methods/mirror.{cc,h}: - download the mirror file into Dir::State::Mirrors - added RefreshInterval option to not ask for the mirror file too often --- methods/mirror.cc | 56 +++++++++++++++++++++++++++++++++++++++++++++---------- methods/mirror.h | 1 + 2 files changed, 47 insertions(+), 10 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4de981522..e70a40f55 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -18,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -28,11 +29,16 @@ using namespace std; /* * TODO: + * - send expected checksum to the mirror method so that + some checking/falling back can be done here already + * - keep the mirror file around in /var/lib/apt/mirrors + * can't be put into lists/ because of the listclearer + * cleanup by time (mtime relative to the other mtimes) + * - use a TTL time the mirror file is fetched again (6h?) + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - support keeping the mirror file around (evil listclearer strikes again) - * -> /var/lib/apt/mirrors dir? how to cleanup? by time? - * - provide some TTL time until the mirror file is get again (1h? 6h?) - * - deal with runing as non-root (we can't write to the lists dir then) + * - magicmarker is (a bit) evil * - testing :) */ @@ -60,6 +66,12 @@ bool MirrorMethod::Configuration(string Message) } /*}}}*/ +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string dir) +{ + +} + bool MirrorMethod::GetMirrorFile(string uri) { @@ -69,7 +81,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { @@ -77,19 +89,43 @@ bool MirrorMethod::GetMirrorFile(string uri) cerr << "mirror-file: " << MirrorFile << endl; } - // FIXME: fetch it with curl + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + HasMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - - if(res) + if(res) HasMirrorFile = true; Fetcher.Shutdown(); - return true; + return res; } bool MirrorMethod::SelectMirror() { + // FIXME: make the mirror selection more clever, do not + // just use the first one! ifstream in(MirrorFile.c_str()); getline(in, Mirror); if(Debug) @@ -103,7 +139,7 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // get mirror information + // select mirror only once per session if(!HasMirrorFile) { GetMirrorFile(Itm->Uri); diff --git a/methods/mirror.h b/methods/mirror.h index a47b8fbf1..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -34,6 +34,7 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + bool Clean(string dir); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); -- cgit v1.2.3 From 70288656715d2fe4c2c33598124ae48f7bca9cdf Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:38:55 +0100 Subject: * methods/mirror.cc: - implemented simple Clean() implementation based on the time of the last access for a mirror file --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index e70a40f55..425a2f7f5 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -19,12 +19,14 @@ #include #include #include +#include +#include using namespace std; #include "mirror.h" #include "http.h" - +#include "apti18n.h" /*}}}*/ /* @@ -67,9 +69,54 @@ bool MirrorMethod::Configuration(string Message) /*}}}*/ // clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string dir) +bool MirrorMethod::Clean(string Dir) { + // FIXME: it would better to have a global idea of the mirrors + // in the sources.list and use this instead of this time + // based approach. currently apt does not support this :/ + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string StartDir = SafeGetCWD(); + if (chdir(Dir.c_str()) != 0) + { + closedir(D); + return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); + } + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,"lock") == 0 || + strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + // Del everything not touched for MaxAge days + time_t t,now,max; + struct stat buf; + if(stat(Dir->d_name, &buf) != 0) + { + cerr << "Can't stat '" << Dir->d_name << "'" << endl; + continue; + } + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); + if(t + max < now) + { + if(Debug) + clog << "Mirror file is older than MaxAge days, deleting" << endl; + unlink(Dir->d_name); + } + }; + chdir(StartDir.c_str()); + closedir(D); + return true; } @@ -81,6 +128,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); + // get new file MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) @@ -108,7 +156,7 @@ bool MirrorMethod::GetMirrorFile(string uri) return true; } if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; } // not that great to use pkgAcquire here, but we do not have @@ -142,6 +190,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) // select mirror only once per session if(!HasMirrorFile) { + Clean(_config->FindDir("Dir::State::mirrors")); GetMirrorFile(Itm->Uri); SelectMirror(); } -- cgit v1.2.3 From 0c312e0ed24fed69412c181634dd4515e9b1d46f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 12:10:05 +0100 Subject: * implemented proper mirror list cleanup --- methods/mirror.cc | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 425a2f7f5..00f7b7807 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -71,9 +72,14 @@ bool MirrorMethod::Configuration(string Message) // clean the mirrors dir based on ttl information bool MirrorMethod::Clean(string Dir) { - // FIXME: it would better to have a global idea of the mirrors - // in the sources.list and use this instead of this time - // based approach. currently apt does not support this :/ + vector::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + // read sources.list + pkgSourceList list; + list.ReadMainList(); DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -94,24 +100,21 @@ bool MirrorMethod::Clean(string Dir) strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; - - // Del everything not touched for MaxAge days - time_t t,now,max; - struct stat buf; - if(stat(Dir->d_name, &buf) != 0) + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) { - cerr << "Can't stat '" << Dir->d_name << "'" << endl; - continue; + string uri = (*I)->GetURI(); + if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + string BaseUri = uri.substr(0,uri.find(Marker)); + if (URItoFileName(BaseUri) == Dir->d_name) + break; } - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); - if(t + max < now) - { - if(Debug) - clog << "Mirror file is older than MaxAge days, deleting" << endl; + // nothing found, nuke it + if (I == list.end()) unlink(Dir->d_name); - } }; chdir(StartDir.c_str()); -- cgit v1.2.3 From 933833c51fbfe3dca3f3a3073d441e5856462605 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 13:07:22 +0100 Subject: * methods/mirror.cc: - updated the TODO --- methods/mirror.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 00f7b7807..f08b324ec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -34,14 +34,18 @@ using namespace std; * TODO: * - send expected checksum to the mirror method so that some checking/falling back can be done here already - * - keep the mirror file around in /var/lib/apt/mirrors - * can't be put into lists/ because of the listclearer - * cleanup by time (mtime relative to the other mtimes) - * - use a TTL time the mirror file is fetched again (6h?) + use pkgAcquire::Custom600Header() for this? what about gpgv + failures? + #OR# + * - implement it at the pkgAcquire::Item::Failed() level but then + we need to send back what uri exactly was failing + * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil + * - magicmarker is (a bit) evil, maybe just use a similar approach as in + clean and read the sources.list and use the GetURI() method to find + the prefix? * - testing :) */ -- cgit v1.2.3 From cae9cdcefc34eba6d023cb30cbbdb9bbf909b8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 16:35:56 +0100 Subject: * basic error reporting from apt in place now (ReportMirrorFailures()) --- methods/mirror.cc | 27 +++++++++++++++++++++------ methods/mirror.h | 1 + 2 files changed, 22 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index f08b324ec..428726a3d 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,14 +32,14 @@ using namespace std; /* * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - use pkgAcquire::Custom600Header() for this? what about gpgv - failures? + * - what about gpgv failures? better standard format for errors + to send back to LP #OR# * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing - + we need to send back what uri exactly was failing + [mvo: the problem with this approach is ::Failed() is not really + called for all failures :/ e.g. md5sum mismatch in a archive + is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here @@ -214,6 +214,9 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // FIXME: queue next mirror? + ReportMirrorFailure(Err); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -228,11 +231,23 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { + // FIXME: queue next mirror? + if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) + ReportMirrorFailure("499 Hash mismatch"); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } +void MirrorMethod::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed + std::cerr << "\nReportMirrorFailure: " + << Queue->Uri + << " FailCode: " + << FailCode << std::endl; +} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..3ff9e1a96 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,6 +35,7 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); + void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); -- cgit v1.2.3 From 2769f0bcf6c1ac73e883f47857fa227de92f4525 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 17:16:42 +0100 Subject: * mirror-failure.py: example mirror failure cgi * methods/mirror.cc: prepare for the failure submit --- methods/mirror.cc | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 428726a3d..6621d47e2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -247,6 +247,12 @@ void MirrorMethod::ReportMirrorFailure(string FailCode) << Queue->Uri << " FailCode: " << FailCode << std::endl; +#if 0 // FIXME: do not use system, make sure to properly encode + // URI/FailCode, do not hardcode the submit url + system("curl -d url=" + Queue->Uri + + " -d FailureCode=" + FailCode + + " http://localhost:8000/ &"); +#endif } int main() -- cgit v1.2.3 From 362d29343e5d25248bcd84300aa1b18effe891e7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Jan 2007 14:42:17 +0100 Subject: make the mirror failures actually produce a error message --- methods/mirror.cc | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 6621d47e2..8f9b8ed34 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -30,13 +30,18 @@ using namespace std; #include "apti18n.h" /*}}}*/ -/* +/* Done: + * - works with http only + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * * TODO: - * - what about gpgv failures? better standard format for errors - to send back to LP - #OR# - * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing + * what about gpgv failures? this should call-out to the problem reporting + script, but we need to know what mirror was used + * better standard format for errors to send back + * - implement failure reporting at the pkgAcquire::Item::Failed() level + but then we need to send back what uri exactly was failing [mvo: the problem with this approach is ::Failed() is not really called for all failures :/ e.g. md5sum mismatch in a archive is not] @@ -46,6 +51,7 @@ using namespace std; * - magicmarker is (a bit) evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? + * support more than http * - testing :) */ @@ -243,16 +249,34 @@ void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) void MirrorMethod::ReportMirrorFailure(string FailCode) { // report that Queue->Uri failed +#if 0 std::cerr << "\nReportMirrorFailure: " << Queue->Uri << " FailCode: " << FailCode << std::endl; -#if 0 // FIXME: do not use system, make sure to properly encode - // URI/FailCode, do not hardcode the submit url - system("curl -d url=" + Queue->Uri + - " -d FailureCode=" + FailCode + - " http://localhost:8000/ &"); #endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/lib/apt/report-mirror-failure"); + Args[i++] = report.c_str(); + Args[i++] = Queue->Uri.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } } int main() -- cgit v1.2.3 From 36280399db0ae203d3f1ae4d44b946f31e9a38ce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 1 Feb 2007 12:32:50 +0100 Subject: * commited the latest mirror failure detection code --- methods/connect.cc | 17 ++++++++--------- methods/mirror.cc | 45 ++++----------------------------------------- methods/mirror.h | 1 - 3 files changed, 12 insertions(+), 51 deletions(-) (limited to 'methods') diff --git a/methods/connect.cc b/methods/connect.cc index 8c2ac6d56..4e227c3fd 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -18,6 +18,7 @@ #include #include #include +#include // Internet stuff #include @@ -67,12 +68,10 @@ static bool DoConnect(struct addrinfo *Addr,string Host, wrong this will get tacked onto the end of the error message */ if (LastHostAddr->ai_next != 0) { - char Name2[NI_MAXHOST + NI_MAXSERV + 10]; - snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service); - Owner->SetFailExtraMsg(string(Name2)); - } - else - Owner->SetFailExtraMsg(""); + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"),Name,Service); + Owner->SetIP(ss.str()); + } // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, @@ -89,7 +88,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, /* This implements a timeout for connect by opening the connection nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) { - Owner->SetFailExtraMsg("\nFailReason: Timeout"); + Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " "connection timed out"),Host.c_str(),Service,Name); } @@ -104,7 +103,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, { errno = Err; if(errno == ECONNREFUSED) - Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused"); + Owner->SetFailReason("ConnectionRefused"); return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); } @@ -169,7 +168,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, if (Res == EAI_AGAIN) { - Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure"); + Owner->SetFailReason("TmpResolveFailure"); return _error->Error(_("Temporary failure resolving '%s'"), Host.c_str()); } diff --git a/methods/mirror.cc b/methods/mirror.cc index 8f9b8ed34..8ccfb8559 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -38,7 +38,7 @@ using namespace std; * * TODO: * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used + script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() * better standard format for errors to send back * - implement failure reporting at the pkgAcquire::Item::Failed() level but then we need to send back what uri exactly was failing @@ -48,7 +48,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil, maybe just use a similar approach as in + * - magicmarker is evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? * support more than http @@ -191,6 +191,8 @@ bool MirrorMethod::SelectMirror() getline(in, Mirror); if(Debug) cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; return true; } @@ -220,9 +222,6 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // FIXME: queue next mirror? - ReportMirrorFailure(Err); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -237,47 +236,11 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - // FIXME: queue next mirror? - if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) - ReportMirrorFailure("499 Hash mismatch"); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } -void MirrorMethod::ReportMirrorFailure(string FailCode) -{ - // report that Queue->Uri failed -#if 0 - std::cerr << "\nReportMirrorFailure: " - << Queue->Uri - << " FailCode: " - << FailCode << std::endl; -#endif - const char *Args[40]; - unsigned int i = 0; - string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/lib/apt/report-mirror-failure"); - Args[i++] = report.c_str(); - Args[i++] = Queue->Uri.c_str(); - Args[i++] = FailCode.c_str(); - pid_t pid = ExecFork(); - if(pid < 0) - { - _error->Error("ReportMirrorFailure Fork failed"); - return; - } - else if(pid == 0) - { - execvp(report.c_str(), (char**)Args); - } - if(!ExecWait(pid, "report-mirror-failure")) - { - _error->Warning("Couldn't report problem to '%s'", - _config->Find("Acquire::Mirror::ReportFailures").c_str()); - } -} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 3ff9e1a96..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,7 +35,6 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); - void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); -- cgit v1.2.3 From 59271f62e4a291c8d96e1f6073203c395734b6ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 17:52:28 +0100 Subject: * use pkgAcqMethod::FailReason() for consistent error reporting --- methods/connect.cc | 1 + methods/http.cc | 3 +++ methods/mirror.cc | 12 +++--------- 3 files changed, 7 insertions(+), 9 deletions(-) (limited to 'methods') diff --git a/methods/connect.cc b/methods/connect.cc index 4e227c3fd..145001fb3 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -163,6 +163,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, DefPort = 0; continue; } + Owner->SetFailReason("ResolveFailure"); return _error->Error(_("Could not resolve '%s'"),Host.c_str()); } diff --git a/methods/http.cc b/methods/http.cc index deaa8d0c8..01ad14655 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,6 +914,9 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) failure */ if (Srv->Result < 200 || Srv->Result >= 300) { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result); + SetFailReason(err); _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) return 4; diff --git a/methods/mirror.cc b/methods/mirror.cc index 8ccfb8559..b64879bec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -31,20 +31,14 @@ using namespace std; /*}}}*/ /* Done: - * - works with http only + * - works with http (only!) * - always picks the first mirror from the list * - call out to problem reporting script * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - use pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP * * TODO: - * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() - * better standard format for errors to send back - * - implement failure reporting at the pkgAcquire::Item::Failed() level - but then we need to send back what uri exactly was failing - [mvo: the problem with this approach is ::Failed() is not really - called for all failures :/ e.g. md5sum mismatch in a archive - is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here -- cgit v1.2.3 From 066b53e95ba224f81d078e7d8df3002275f8a092 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 18:46:45 +0100 Subject: * get rid of magic mirror --- methods/mirror.cc | 62 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b64879bec..ff91130b8 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,22 +42,13 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is evil, maybe just use a similar approach as in - clean and read the sources.list and use the GetURI() method to find - the prefix? - * support more than http + * - support more than http * - testing :) */ MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { -#if 0 - HasMirrorFile=true; - BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; - MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; - Mirror="http://de.archive.ubuntu.com/ubuntu/"; -#endif }; // HttpMethod::Configuration - Handle a configuration message /*{{{*/ @@ -111,8 +102,7 @@ bool MirrorMethod::Clean(string Dir) string uri = (*I)->GetURI(); if(uri.substr(0,strlen("mirror://")) != string("mirror://")) continue; - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - string BaseUri = uri.substr(0,uri.find(Marker)); + string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) break; } @@ -127,11 +117,51 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string uri) +bool MirrorMethod::GetMirrorFile(string mirror_uri_str) { - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - BaseUri = uri.substr(0,uri.find(Marker)); - + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); -- cgit v1.2.3 From f0b509cdb44cb5e79e9c5ddd7ebec46965138534 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 19:06:42 +0100 Subject: * apt-pkg/acquire-item.cc: - use FailReason in pkgAcquire::Item::Failed when available * methods/mirror.cc: - move some debug output into if(Debug) --- methods/mirror.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ff91130b8..c5c0c7461 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -143,7 +143,8 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + if(Debug) + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -152,13 +153,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) for(I=list.begin(); I != list.end(); I++) { string uristr = (*I)->GetURI(); - std::cerr << "Checking: " << uristr << std::endl; + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) continue; // find matching uri in sources.list if(mirror_uri_str.substr(0,uristr.size()) == uristr) { - std::cerr << "found BaseURI: " << uristr << std::endl; + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; BaseUri = uristr.substr(0,uristr.size()-1); } } -- cgit v1.2.3 From 3f599bb721c4ac58d8ff18991c9704b5f30eaa2b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 10:49:25 +0100 Subject: * apt-pkg/acquire-item.cc: - default to "/usr/lib/apt/apt-report-mirror-failure" * cmdline/apt-report-mirror-failure: - no default comit url for now * debian/rules: - move apt-report-mirror-failure into /usr/lib/apt * doc/examples/configure-index: - more documentation * methods/mirror.cc: - updated TODO --- methods/mirror.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c5c0c7461..9a86a10c2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,6 +42,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here + * and better error handling there! * - support more than http * - testing :) */ -- cgit v1.2.3 From 68acfb437f0c6c2e026be6fd3b7da3fc8d64c523 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 16:37:20 +0100 Subject: * ABI version bumped --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index f971e56e8..d66ba11e3 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.51 +LIB_APT_PKG_MAJOR = 3.52 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From d715b9c969264eec96913b577f63a3e9e498b2e6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 21 Feb 2007 09:59:53 +0100 Subject: * fix in the pkgRecords destructor * Bump ABI version --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index d66ba11e3..610bd5b49 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.52 +LIB_APT_PKG_MAJOR = 3.53 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From d530b4c24fc452efc82be4b7e1ebc536a00c09b7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 3 Oct 2007 02:21:52 +0200 Subject: * Reset curl options and timestamp between downloaded files. Thanks to Ryan Murray for the patch --- methods/https.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index e6717e63a..3b2b0bb19 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -115,6 +115,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // - error checking/reporting // - more debug options? (CURLOPT_DEBUGFUNCTION?) + curl_easy_reset(curl); SetupProxy(); // callbacks @@ -125,6 +126,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FILETIME, true); // FIXME: https: offer various options of verification bool peer_verify = _config->FindB("Acquire::https::Verify-Peer", false); @@ -202,6 +204,9 @@ bool HttpsMethod::Fetch(FetchItem *Itm) CURLcode success = curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_responsecode); + long curl_servdate; + curl_easy_getinfo(curl, CURLINFO_FILETIME, &curl_servdate); + // cleanup if(success != 0) { @@ -215,6 +220,14 @@ bool HttpsMethod::Fetch(FetchItem *Itm) if (Res.Size == 0) Res.Size = File->Size(); + // Timestamp + struct utimbuf UBuf; + if (curl_servdate != -1) { + UBuf.actime = curl_servdate; + UBuf.modtime = curl_servdate; + utime(File->Name().c_str(),&UBuf); + } + // check the downloaded result struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) -- cgit v1.2.3 From 3de6d4585be0b8a9bd98b5fb1d62ff922537c435 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Oct 2007 07:50:24 +0200 Subject: * methods/https.cc: - fix off-by-one error I-M-S handling - cleanup after I-M-S hit - only send LastModified if we actually have a file - delete zero size I-M-S hits --- methods/https.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3b2b0bb19..b2bbbddb1 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -161,13 +161,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - // set time values - if(Itm->LastModified > 0) - { - curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); - } - // speed limit int dlLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024; if (dlLimit > 0) @@ -183,7 +176,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // error handling curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - // In this case we send an if-range query with a range header + // if we have the file send an if-range query with a range header if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { char Buf[1000]; @@ -191,11 +184,17 @@ bool HttpsMethod::Fetch(FetchItem *Itm) (long)SBuf.st_size - 1, TimeRFC1123(SBuf.st_mtime).c_str()); headers = curl_slist_append(headers, Buf); + } + else if(Itm->LastModified > 0) + { + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); } // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); - File->Seek(File->Size()); + if (File->Size() > 0) + File->Seek(File->Size() - 1); // keep apt updated Res.Filename = Itm->DestFile; @@ -217,9 +216,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } File->Close(); - if (Res.Size == 0) - Res.Size = File->Size(); - // Timestamp struct utimbuf UBuf; if (curl_servdate != -1) { @@ -232,15 +228,19 @@ bool HttpsMethod::Fetch(FetchItem *Itm) struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) { - Res.Size = Buf.st_size; Res.Filename = File->Name(); Res.LastModified = Buf.st_mtime; Res.IMSHit = false; if (curl_responsecode == 304) { + unlink(File->Name().c_str()); Res.IMSHit = true; Res.LastModified = Itm->LastModified; + Res.Size = 0; + URIDone(Res); + return true; } + Res.Size = Buf.st_size; } // take hashes -- cgit v1.2.3 From 38eedeb77790c106eb35940e3d7835a7e0f0297e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Nov 2007 15:54:17 +0100 Subject: methods/mirror.{cc,h}: - improve the mirror method so that it only updates the mirror list from the server if we get new indexfiles - code cleanup --- methods/mirror.cc | 152 +++++++++++++++++++++++++++++++++--------------------- methods/mirror.h | 7 +-- 2 files changed, 97 insertions(+), 62 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 9a86a10c2..43e56c71a 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -48,7 +48,7 @@ using namespace std; */ MirrorMethod::MirrorMethod() - : HttpMethod(), HasMirrorFile(false) + : HttpMethod(), DownloadedMirrorFile(false) { }; @@ -73,6 +73,9 @@ bool MirrorMethod::Clean(string Dir) if(Debug) clog << "MirrorMethod::Clean(): " << Dir << endl; + if(Dir == "/") + return _error->Error("will not clean: '/'"); + // read sources.list pkgSourceList list; list.ReadMainList(); @@ -118,7 +121,71 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string mirror_uri_str) +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) +{ + + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + DownloadedMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); + + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} + +bool MirrorMethod::SelectMirror() +{ + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + + // FIXME: make the mirror selection more clever, do not + // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) { /* - a mirror_uri_str looks like this: @@ -144,8 +211,9 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ + string name; if(Debug) - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -166,62 +234,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) BaseUri = uristr.substr(0,uristr.size()-1); } } - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - // get new file - MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; - } - - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - HasMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + cerr << "mirror-file: " << name << endl; } - - // not that great to use pkgAcquire here, but we do not have - // any other way right now - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) - HasMirrorFile = true; - Fetcher.Shutdown(); - return res; -} - -bool MirrorMethod::SelectMirror() -{ - // FIXME: make the mirror selection more clever, do not - // just use the first one! - ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; - return true; + return name; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -230,20 +251,33 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // select mirror only once per session - if(!HasMirrorFile) + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - GetMirrorFile(Itm->Uri); - SelectMirror(); + DownloadMirrorFile(Itm->Uri); } + if(Mirror.empty()) + SelectMirror(); + for (FetchItem *I = Queue; I != 0; I = I->Next) { if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(),Mirror); + I->Uri.replace(0,BaseUri.size(), Mirror); } - + // now run the real fetcher return HttpMethod::Fetch(Itm); }; diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..ed817806b 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,13 +26,14 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) - string MirrorFile; // - bool HasMirrorFile; + string MirrorFile; // the file that contains the list of mirrors + bool DownloadedMirrorFile; // already downloaded this session bool Debug; protected: - bool GetMirrorFile(string uri); + bool DownloadMirrorFile(string uri); + string GetMirrorFileName(string uri); bool SelectMirror(); bool Clean(string dir); -- cgit v1.2.3 From 0df1d3ac4312a2051b10478cf1c8b242864cb3b1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 6 Dec 2007 19:47:36 +0100 Subject: we do not break the abi --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 085b357b8..3c12bb974 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.5 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From 5148def81fb20b22fc9ec54bc212e34260fbfebb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 7 Dec 2007 22:54:41 +0100 Subject: update ABI, the homepage change broke it --- methods/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 3c12bb974..085b357b8 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.5 +LIB_APT_PKG_MAJOR = 4.6 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From d7bab8f1142e474ac4bafd139e93439aeddfd546 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 3 Jan 2008 15:22:44 +0100 Subject: methods/makefile: remove duplicated entry for https --- methods/makefile | 7 ------- 1 file changed, 7 deletions(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 085b357b8..f765a4d37 100644 --- a/methods/makefile +++ b/methods/makefile @@ -59,13 +59,6 @@ LIB_MAKES = apt-pkg/makefile SOURCE = https.cc include $(PROGRAM_H) -# The https method -PROGRAM=https -SLIBS = -lapt-pkg -lcurl -LIB_MAKES = apt-pkg/makefile -SOURCE = https.cc -include $(PROGRAM_H) - # The ftp method PROGRAM=ftp SLIBS = -lapt-pkg $(SOCKETLIBS) -- cgit v1.2.3 From 484befd1518245e795ed92e6c97ad50786f3b4a2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 11 Mar 2008 09:59:32 +0100 Subject: * typo fixes (LP: #107960) --- methods/http.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 6aa4261ff..61321c850 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -367,8 +367,8 @@ bool ServerState::Close() /*}}}*/ // ServerState::RunHeaders - Get the headers before the data /*{{{*/ // --------------------------------------------------------------------- -/* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header - parse error occured */ +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ int ServerState::RunHeaders() { State = Header; -- cgit v1.2.3 From 236527f0fffe30fc8988e17d926778c23ac1f902 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Jan 2009 17:39:37 +0100 Subject: LP: #263089 --- methods/gpgv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 9f4683e6e..f3277b300 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -264,7 +264,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // least one bad signature. good signatures and NoPubKey signatures // happen easily when a file is signed with multiple signatures if(GoodSigners.empty() or !BadSigners.empty()) - return _error->Error(errmsg.c_str()); + return _error->Error("%s", errmsg.c_str()); } // Just pass the raw output up, because passing it as a real data -- cgit v1.2.3 From ebb461fdf02ee3e038d4b3a4ab1a0a60188edf9a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Feb 2009 12:14:36 +0100 Subject: * [ABI break] merge support for http redirects, thanks to Jeff Licquia and Anthony Towns * [ABI break] use int for the package IDs (thanks to Steve Cotton) - Galician updated. Closes: #509151 - Catalan updated. Closes: #509375 - Vietnamese updated. Closes: #509422 - Traditional Chinese added. Closes: #510664 * COPYING: - Actualized. Removed obsolete Qt section, added GPLv2 clause. (Closes: #440049, #509337) * Clarify the --help for 'purge' (LP: #243948) --- methods/http.cc | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- methods/http.h | 3 +++ methods/makefile | 2 +- 3 files changed, 81 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index b0fb89fda..44274bd78 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include // Internet stuff @@ -56,6 +57,7 @@ int HttpMethod::FailFd = -1; time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; +bool AllowRedirect = false; bool Debug = false; URI Proxy; @@ -627,6 +629,12 @@ bool ServerState::HeaderLine(string Line) return true; } + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + return true; } /*}}}*/ @@ -899,7 +907,9 @@ bool HttpMethod::ServerDie(ServerState *Srv) 1 - IMS hit 3 - Unrecoverable error 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) */ + 5 - Unrecoverable non-server error (close the connection) + 6 - Try again with a new or changed URI + */ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified @@ -911,6 +921,27 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) return 1; } + /* Redirect + * + * Note that it is only OK for us to treat all redirection the same + * because we *always* use GET, not other HTTP methods. There are + * three redirection codes for which it is not appropriate that we + * redirect. Pass on those codes so the error handling kicks in. + */ + if (AllowRedirect + && (Srv->Result > 300 && Srv->Result < 400) + && (Srv->Result != 300 // Multiple Choices + && Srv->Result != 304 // Not Modified + && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved) + { + if (!Srv->Location.empty()) + { + NextURI = Srv->Location; + return 6; + } + /* else pass through for error message */ + } + /* We have a reply we dont handle. This should indicate a perm server failure */ if (Srv->Result < 200 || Srv->Result >= 300) @@ -1028,6 +1059,7 @@ bool HttpMethod::Configuration(string Message) if (pkgAcqMethod::Configuration(Message) == false) return false; + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); @@ -1041,6 +1073,10 @@ bool HttpMethod::Configuration(string Message) /* */ int HttpMethod::Loop() { + typedef vector StringVector; + typedef vector::iterator StringVectorIterator; + map Redirected; + signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -1227,6 +1263,46 @@ int HttpMethod::Loop() break; } + // Try again with a new URL + case 6: + { + // Clear rest of response if there is content + if (Server->HaveContent) + { + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(); + delete File; + File = 0; + } + + /* Detect redirect loops. No more redirects are allowed + after the same URI is seen twice in a queue item. */ + StringVector &R = Redirected[Queue->DestFile]; + bool StopRedirects = false; + if (R.size() == 0) + R.push_back(Queue->Uri); + else if (R[0] == "STOP" || R.size() > 10) + StopRedirects = true; + else + { + for (StringVectorIterator I = R.begin(); I != R.end(); I++) + if (Queue->Uri == *I) + { + R[0] = "STOP"; + break; + } + + R.push_back(Queue->Uri); + } + + if (StopRedirects == false) + Redirect(NextURI); + else + Fail(); + + break; + } + default: Fail(_("Internal error")); break; diff --git a/methods/http.h b/methods/http.h index dec5cd80f..bc076e1f8 100644 --- a/methods/http.h +++ b/methods/http.h @@ -99,6 +99,7 @@ struct ServerState enum {Chunked,Stream,Closes} Encoding; enum {Header, Data} State; bool Persistent; + string Location; // This is a Persistent attribute of the server itself. bool Pipeline; @@ -145,6 +146,8 @@ class HttpMethod : public pkgAcqMethod protected: virtual bool Fetch(FetchItem *); + string NextURI; + public: friend class ServerState; diff --git a/methods/makefile b/methods/makefile index 5c4fa82bf..1d022be90 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.7 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3 From 8872c430df84660fa994db0ebfa67861239fe739 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:36:36 +0100 Subject: * methods/https.cc: - add Acquire::https::AllowRedirect support --- methods/https.cc | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 98dfeefa1..87de54589 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -208,6 +208,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); + // set redirect options and default to 10 redirects + bool AllowRedirect = _config->FindI("Acquire::https::AllowRedirect", true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); + // debug if(_config->FindB("Debug::Acquire::https", false)) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); -- cgit v1.2.3 From 7e522e9e0e4808202061e0a73345bd8b18eccde3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:38:11 +0100 Subject: * apt-pkg/pkgcache.cc: - do not run "dpkg --configure pkg" if pkg is in trigger-awaited state (LP: #322955) - do not unlink files in partial/ (thanks to robbiew) --- methods/https.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 87de54589..8bf44b52a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -253,7 +253,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // cleanup if(success != 0) { - unlink(File->Name().c_str()); _error->Error("%s", curl_errorstr); Fail(); return true; -- cgit v1.2.3 From 5dad4134da37576de12721f34165c1fe4d138b3c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 19 Mar 2009 15:42:25 +0100 Subject: * methods/mirror.cc: - when download the mirror file and the server is down, return a propper error message (LP: #278635) --- methods/mirror.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index bdd783cc7..b3a956b95 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -123,6 +123,8 @@ bool MirrorMethod::Clean(string Dir) bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) { + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): " << endl; // check the file, if it is not older than RefreshInterval just use it // otherwise try to get a new one @@ -251,6 +253,9 @@ string MirrorMethod::GetMirrorFileName(string mirror_uri_str) depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { + if(Debug) + clog << "MirrorMethod::Fetch()" << endl; + // the http method uses Fetch(0) as a way to update the pipeline, // just let it do its work in this case - Fetch() with a valid // Itm will always run before the first Fetch(0) @@ -269,8 +274,17 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) - SelectMirror(); + if(Mirror.empty()) { + if(!SelectMirror()) { + // no valid mirror selected, something went wrong downloading + // from the master mirror site most likely and there is + // no old mirror file availalbe + return false; + } + } + if(Debug) + clog << "selected mirror: " << Mirror << endl; + for (FetchItem *I = Queue; I != 0; I = I->Next) { -- cgit v1.2.3 From 0b77f4775db7bc45964e0337b8978a170b3f0483 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Apr 2009 22:52:11 +0200 Subject: * methods/gpgv.cc: - properly check for expired and revoked keys (closes: #433091) LP: #356012 --- methods/gpgv.cc | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index f3277b300..150c1d315 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -17,13 +17,18 @@ #define GNUPGBADSIG "[GNUPG:] BADSIG" #define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" #define GNUPGVALIDSIG "[GNUPG:] VALIDSIG" +#define GNUPGGOODSIG "[GNUPG:] GOODSIG" +#define GNUPGKEYEXPIRED "[GNUPG:] KEYEXPIRED" +#define GNUPGREVKEYSIG "[GNUPG:] REVKEYSIG" #define GNUPGNODATA "[GNUPG:] NODATA" class GPGVMethod : public pkgAcqMethod { private: string VerifyGetSigners(const char *file, const char *outfile, - vector &GoodSigners, vector &BadSigners, + vector &GoodSigners, + vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners); protected: @@ -37,6 +42,7 @@ class GPGVMethod : public pkgAcqMethod string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector &GoodSigners, vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners) { // setup a (empty) stringstream for formating the return value @@ -179,15 +185,27 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, std::cerr << "Got NODATA! " << std::endl; BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); } - if (strncmp(buffer, GNUPGVALIDSIG, sizeof(GNUPGVALIDSIG)-1) == 0) + if (strncmp(buffer, GNUPGKEYEXPIRED, sizeof(GNUPGKEYEXPIRED)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got KEYEXPIRED! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGREVKEYSIG, sizeof(GNUPGREVKEYSIG)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got REVKEYSIG! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGGOODSIG, sizeof(GNUPGGOODSIG)-1) == 0) { char *sig = buffer + sizeof(GNUPGPREFIX); - char *p = sig + sizeof("VALIDSIG"); + char *p = sig + sizeof("GOODSIG"); while (*p && isxdigit(*p)) p++; *p = 0; if (_config->FindB("Debug::Acquire::gpgv", false)) - std::cerr << "Got VALIDSIG, key ID:" << sig << std::endl; + std::cerr << "Got GOODSIG, key ID:" << sig << std::endl; GoodSigners.push_back(string(sig)); } } @@ -227,6 +245,8 @@ bool GPGVMethod::Fetch(FetchItem *Itm) string keyID; vector GoodSigners; vector BadSigners; + // a worthless signature is a expired or revoked one + vector WorthlessSigners; vector NoPubKeySigners; FetchResult Res; @@ -235,13 +255,14 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // Run gpgv on file, extract contents and get the key ID of the signer string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), - GoodSigners, BadSigners, NoPubKeySigners); + GoodSigners, BadSigners, WorthlessSigners, + NoPubKeySigners); if (GoodSigners.empty() || !BadSigners.empty() || !NoPubKeySigners.empty()) { string errmsg; // In this case, something bad probably happened, so we just go // with what the other method gave us for an error message. - if (BadSigners.empty() && NoPubKeySigners.empty()) + if (BadSigners.empty() && WorthlessSigners.empty() && NoPubKeySigners.empty()) errmsg = msg; else { @@ -252,6 +273,13 @@ bool GPGVMethod::Fetch(FetchItem *Itm) I != BadSigners.end(); I++) errmsg += (*I + "\n"); } + if (!WorthlessSigners.empty()) + { + errmsg += _("The following signatures were invalid:\n"); + for (vector::iterator I = WorthlessSigners.begin(); + I != WorthlessSigners.end(); I++) + errmsg += (*I + "\n"); + } if (!NoPubKeySigners.empty()) { errmsg += _("The following signatures couldn't be verified because the public key is not available:\n"); -- cgit v1.2.3 From 217d575b5713f4d9275177b58a36c581a7880c03 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 14 Jul 2009 11:22:41 +0200 Subject: * apt-pkg/acquire-worker.cc: - show error details of failed methods * apt-pkg/contrib/fileutl.cc: - if a process aborts with signal, show signal number * methods/http.cc: - ignore SIGPIPE, we deal with EPIPE from write in HttpMethod::ServerDie() (LP: #385144) --- methods/http_main.cc | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'methods') diff --git a/methods/http_main.cc b/methods/http_main.cc index 2c46ab19d..7815c2fc1 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,6 @@ #include #include +#include #include "connect.h" #include "rfc2553emu.h" @@ -10,6 +11,10 @@ int main() { setlocale(LC_ALL, ""); + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + HttpMethod Mth; return Mth.Loop(); } -- cgit v1.2.3 From 056539eeb8eb84cea65c0c15466111b978ee156d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Jul 2009 18:25:16 +0200 Subject: methods/http.cc: remove merge artifact --- methods/http.cc | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 476fb751f..859dbd39c 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -1316,15 +1316,4 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - - HttpMethod Mth; - return Mth.Loop(); -} - -- cgit v1.2.3 From 1082d4c7ad98c091688bdeb427e48913d1327279 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 4 May 2010 16:10:47 +0200 Subject: * methods/rred.cc: - use the patchfile modification time instead of the one from the "old" file - thanks to Philipp Weis for noticing! (Closes: #571541) --- methods/rred.cc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'methods') diff --git a/methods/rred.cc b/methods/rred.cc index 262c78cab..f42c7a072 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -477,23 +477,26 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ Patch.Close(); To.Close(); - // Transfer the modification times - struct stat Buf; - if (stat(Path.c_str(),&Buf) != 0) + /* Transfer the modification times from the patch file + to be able to see in which state the file should be + and use the access time from the "old" file */ + struct stat BufBase, BufPatch; + if (stat(Path.c_str(),&BufBase) != 0 || + stat(string(Path+".ed").c_str(),&BufPatch) != 0) return _error->Errno("stat",_("Failed to stat")); struct utimbuf TimeBuf; - TimeBuf.actime = Buf.st_atime; - TimeBuf.modtime = Buf.st_mtime; + TimeBuf.actime = BufBase.st_atime; + TimeBuf.modtime = BufPatch.st_mtime; if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0) return _error->Errno("utime",_("Failed to set modification time")); - if (stat(Itm->DestFile.c_str(),&Buf) != 0) + if (stat(Itm->DestFile.c_str(),&BufBase) != 0) return _error->Errno("stat",_("Failed to stat")); // return done - Res.LastModified = Buf.st_mtime; - Res.Size = Buf.st_size; + Res.LastModified = BufBase.st_mtime; + Res.Size = BufBase.st_size; Res.TakeHashes(Hash); URIDone(Res); -- cgit v1.2.3 From f5a3d0095a5937c14c509a588eb6b7e946faf0c1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 12 May 2010 11:44:50 +0200 Subject: * methods/gpgv.cc: - remove the keyrings count limit by using vector magic --- methods/gpgv.cc | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index c58e6cc45..a149d67dd 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -12,6 +12,8 @@ #include #include +#include + #define GNUPGPREFIX "[GNUPG:]" #define GNUPGBADSIG "[GNUPG:] BADSIG" #define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" @@ -87,23 +89,18 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) { - const char *Args[400]; - unsigned int i = 0; + std::vector Args; + Args.reserve(30); - Args[i++] = gpgvpath.c_str(); - Args[i++] = "--status-fd"; - Args[i++] = "3"; - Args[i++] = "--ignore-time-conflict"; + Args.push_back(gpgvpath.c_str()); + Args.push_back("--status-fd"); + Args.push_back("3"); + Args.push_back("--ignore-time-conflict"); for (vector::const_iterator K = keyrings.begin(); K != keyrings.end(); ++K) { - Args[i++] = "--keyring"; - Args[i++] = K->c_str(); - // check overflow (minus a bit of extra space at the end) - if(i >= sizeof(Args)/sizeof(char*)-5) { - std::clog << _("E: Too many keyrings should be passed to gpgv. Exiting.") << std::endl; - exit(111); - } + Args.push_back("--keyring"); + Args.push_back(K->c_str()); } Configuration::Item const *Opts; @@ -115,23 +112,18 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, { if (Opts->Value.empty() == true) continue; - Args[i++] = Opts->Value.c_str(); - // check overflow (minus a bit of extra space at the end) - if(i >= sizeof(Args)/sizeof(char*)-5) { - std::clog << _("E: Argument list from Acquire::gpgv::Options too long. Exiting.") << std::endl; - exit(111); - } + Args.push_back(Opts->Value.c_str()); } } - Args[i++] = file; - Args[i++] = outfile; - Args[i++] = NULL; + Args.push_back(file); + Args.push_back(outfile); + Args.push_back(NULL); if (Debug == true) { std::clog << "Preparing to exec: " << gpgvpath; - for(unsigned int j=0;Args[j] != NULL; j++) - std::clog << " " << Args[j]; + for(std::vector::const_iterator a = Args.begin();*a != NULL; ++a) + std::clog << " " << *a; std::clog << std::endl; } int const nullfd = open("/dev/null", O_RDONLY); @@ -145,7 +137,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, putenv((char *)"LANG="); putenv((char *)"LC_ALL="); putenv((char *)"LC_MESSAGES="); - execvp(gpgvpath.c_str(), (char **)Args); + execvp(gpgvpath.c_str(), (char **) &Args[0]); exit(111); } -- cgit v1.2.3 From f64684a40c97fc52bcd7942bed81bf08b02fdd28 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 25 May 2010 17:45:12 +0200 Subject: methods/http.cc: shorten comment on return values of DealWithHeaders (because we use a enum now) --- methods/http.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c05abc862..d43dd14c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,13 +914,7 @@ bool HttpMethod::ServerDie(ServerState *Srv) // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ // --------------------------------------------------------------------- /* We look at the header data we got back from the server and decide what - to do. Returns - 0 - File is open, - 1 - IMS hit - 3 - Unrecoverable error - 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) - 6 - Try again with a new or changed URI + to do. Returns DealWithHeadersResult (see http.h for details). */ HttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) -- cgit v1.2.3 From 96db74ce38e9451609fe33f9e25f3f9d42b1fe22 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:40:40 +0200 Subject: * apt-pkg/deb/dpkgpm.cc: - add missing include * methods/mirror.{cc,h}: - add SelectNextMirror() and InitMirrors() functions - read all mirrors into the AllMirrors vector --- methods/mirror.cc | 30 +++++++++++++++++++++++------- methods/mirror.h | 4 +++- 2 files changed, 26 insertions(+), 8 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b3a956b95..0a0d07e6b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,7 +162,21 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectMirror() +bool MirrorMethod::SelectNextMirror() +{ + if (AllMirrors.size() < 1) + return false; + + Mirror = AllMirrors[0]; + AllMirrors.erase(AllMirrors.begin()); + if(Debug) + cerr << "using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +bool MirrorMethod::InitMirrors() { // if we do not have a MirrorFile, fallback if(!FileExists(MirrorFile)) @@ -179,11 +193,13 @@ bool MirrorMethod::SelectMirror() // get into sync issues (got indexfiles from mirror A, // but packages from mirror B - one might be out of date etc) ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; + string s; + while (!in.eof()) + { + getline(in, s); + AllMirrors.push_back(s); + } + SelectNextMirror(); return true; } @@ -275,7 +291,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) } if(Mirror.empty()) { - if(!SelectMirror()) { + if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is // no old mirror file availalbe diff --git a/methods/mirror.h b/methods/mirror.h index ed817806b..1b506dc87 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,6 +26,7 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) + vector AllMirrors; // all available mirrors string MirrorFile; // the file that contains the list of mirrors bool DownloadedMirrorFile; // already downloaded this session @@ -34,7 +35,8 @@ class MirrorMethod : public HttpMethod protected: bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); - bool SelectMirror(); + bool InitMirrors(); + bool SelectNextMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3 From 483dfdd8aced593e966d221073c056c2e332584f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:51:25 +0200 Subject: methods/mirror.cc: on fail try the next mirror --- methods/mirror.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 0a0d07e6b..b8bd6db73 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -197,7 +197,8 @@ bool MirrorMethod::InitMirrors() while (!in.eof()) { getline(in, s); - AllMirrors.push_back(s); + if (s.size() > 0) + AllMirrors.push_back(s); } SelectNextMirror(); return true; @@ -314,6 +315,15 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // try the next mirror on fail + string old_mirror = Mirror; + if (SelectNextMirror()) + { + Queue->Uri.replace(0, old_mirror.size(), Mirror); + return; + } + + // all mirrors failed, so bail out if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); -- cgit v1.2.3 From 0ded3ad3438666f833773895ef6318f84d2f849d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 22:44:49 +0200 Subject: improve error message if mirror method fails --- methods/mirror.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b8bd6db73..b9fa55d87 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -25,6 +25,8 @@ using namespace std; +#include + #include "mirror.h" #include "http.h" #include "apti18n.h" @@ -164,16 +166,12 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) bool MirrorMethod::SelectNextMirror() { - if (AllMirrors.size() < 1) - return false; - - Mirror = AllMirrors[0]; - AllMirrors.erase(AllMirrors.begin()); if(Debug) cerr << "using mirror: " << Mirror << endl; + Mirror = AllMirrors[0]; UsedMirror = Mirror; - return true; + return false; } bool MirrorMethod::InitMirrors() @@ -324,6 +322,10 @@ void MirrorMethod::Fail(string Err,bool Transient) } // all mirrors failed, so bail out + string s; + strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); + SetIP(s); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); -- cgit v1.2.3 From 661f7b1c727aada703ffbb350d44759189a441fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 22:47:49 +0200 Subject: methods/mirror.cc: remove Acquire::Mirror::RefreshInterval (not really useful) --- methods/mirror.cc | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b9fa55d87..4d767c9f3 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -128,28 +128,6 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) if(Debug) clog << "MirrorMethod::DownloadMirrorFile(): " << endl; - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - DownloadedMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; - } - // not that great to use pkgAcquire here, but we do not have // any other way right now string fetch = BaseUri; -- cgit v1.2.3 From 0391542729e5e8a2ba9ae9b1d470f252e24eb296 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:13:42 +0200 Subject: methods/mirrors.cc: make cycle through the mirrors work properly --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++--------------- methods/mirror.h | 3 ++- 2 files changed, 42 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4d767c9f3..567522478 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -142,13 +142,44 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectNextMirror() +/* convert a the Queue->Uri back to the mirror base uri and look + * at all mirrors we have for this, this is needed as queue->uri + * may point to different mirrors (if TryNextMirror() was run) + */ +void MirrorMethod::CurrentQueueUriToMirror() +{ + // already in mirror:// style so nothing to do + if(Queue->Uri.find("mirror://") == 0) + return; + + // find current mirror and select next one + for (int i=0; i < AllMirrors.size(); i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + return; + } + } + _error->Error("Internal error: Failed to convert %s back to %s", + Queue->Uri, BaseUri); +} + +bool MirrorMethod::TryNextMirror() { if(Debug) cerr << "using mirror: " << Mirror << endl; - Mirror = AllMirrors[0]; - UsedMirror = Mirror; + // find current mirror and select next one + for (int i=0; i < AllMirrors.size()-1; i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + return true; + } + } + return false; } @@ -176,7 +207,8 @@ bool MirrorMethod::InitMirrors() if (s.size() > 0) AllMirrors.push_back(s); } - SelectNextMirror(); + Mirror = AllMirrors[0]; + UsedMirror = Mirror; return true; } @@ -292,34 +324,27 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - string old_mirror = Mirror; - if (SelectNextMirror()) - { - Queue->Uri.replace(0, old_mirror.size(), Mirror); + if (TryNextMirror()) return; - } // all mirrors failed, so bail out string s; strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); SetIP(s); - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::Fail(Err, Transient); } void MirrorMethod::URIStart(FetchResult &Res) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIStart(Res); } void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIDone(Res, Alt); } diff --git a/methods/mirror.h b/methods/mirror.h index 1b506dc87..0a3ea6e92 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -36,7 +36,8 @@ class MirrorMethod : public HttpMethod bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); bool InitMirrors(); - bool SelectNextMirror(); + bool TryNextMirror(); + void CurrentQueueUriToMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3 From b86f642111954754dd9932ed2f28a9ea85035e87 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:15:55 +0200 Subject: methods/mirror.cc: simplify uri.startswith() --- methods/mirror.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 567522478..cfc155f58 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -106,7 +106,7 @@ bool MirrorMethod::Clean(string Dir) for(I=list.begin(); I != list.end(); I++) { string uri = (*I)->GetURI(); - if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + if(uri.find("mirror://") != 0) continue; string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) -- cgit v1.2.3 From 963b16dcebba149ae2c35bd255b34242923fbea0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:30:01 +0200 Subject: implement Fail-Ignore bool in FetchItem that tells the method that a failure of this item is ok and does not need to be tried on all mirrors --- methods/mirror.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index cfc155f58..ea0fba438 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,24 +162,26 @@ void MirrorMethod::CurrentQueueUriToMirror() } } _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri, BaseUri); + Queue->Uri.c_str(), BaseUri.c_str()); } bool MirrorMethod::TryNextMirror() { - if(Debug) - cerr << "using mirror: " << Mirror << endl; - // find current mirror and select next one for (int i=0; i < AllMirrors.size()-1; i++) { if (Queue->Uri.find(AllMirrors[i]) == 0) { Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; return true; } } + if (Debug) + clog << "TryNextMirror could not find another mirror to try" << endl; + return false; } @@ -307,15 +309,12 @@ bool MirrorMethod::Fetch(FetchItem *Itm) return false; } } - if(Debug) - clog << "selected mirror: " << Mirror << endl; + if(Itm->Uri.find("mirror://") != string::npos) + Itm->Uri.replace(0,BaseUri.size(), Mirror); - for (FetchItem *I = Queue; I != 0; I = I->Next) - { - if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(), Mirror); - } + if(Debug) + clog << "Fetch: " << Itm->Uri << endl << endl; // now run the real fetcher return HttpMethod::Fetch(Itm); @@ -324,7 +323,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - if (TryNextMirror()) + if (!Queue->FailIgnore && TryNextMirror()) return; // all mirrors failed, so bail out -- cgit v1.2.3 From 2ac9b90b7b3ebeea9a2580b6f317f1dfefc8c8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:57:00 +0200 Subject: methods/mirror.cc: debug improvements --- methods/mirror.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ea0fba438..b2b6b2ecf 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -301,7 +301,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) { + if(AllMirrors.empty()) { if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is @@ -322,7 +322,14 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // try the next mirror on fail + // FIXME: TryNextMirror is not ideal for indexfile as we may + // run into auth issues + + if (Debug) + clog << "Failure to get " << Queue->Uri << endl; + + // try the next mirror on fail (if its not a expected failure, + // e.g. translations are ok to ignore) if (!Queue->FailIgnore && TryNextMirror()) return; -- cgit v1.2.3 From 96cc64a521957d63704de72ed95f1c839698c53c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 00:53:44 +0200 Subject: move the users away from the deprecated StrToTime() method --- methods/ftp.cc | 3 +-- methods/http.cc | 2 +- methods/rsh.cc | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index 3e1725823..97248f900 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -661,8 +661,7 @@ bool FTPConn::ModTime(const char *Path, time_t &Time) return true; // Parse it - StrToTime(Msg,Time); - return true; + return FTPMDTMStrToTime(Msg.c_str(), Time); } /*}}}*/ // FTPConn::CreateDataFd - Get a data connection /*{{{*/ diff --git a/methods/http.cc b/methods/http.cc index d43dd14c8..5fdc62696 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -631,7 +631,7 @@ bool ServerState::HeaderLine(string Line) if (stringcasecmp(Tag,"Last-Modified:") == 0) { - if (StrToTime(Val,Date) == false) + if (RFC1123StrToTime(Val.c_str(), Date) == false) return _error->Error(_("Unknown date format")); return true; } diff --git a/methods/rsh.cc b/methods/rsh.cc index f0ccfc42d..97b4ef151 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -278,8 +278,7 @@ bool RSHConn::ModTime(const char *Path, time_t &Time) return false; // Parse it - StrToTime(Msg,Time); - return true; + return FTPMDTMStrToTime(Msg.c_str(), Time); } /*}}}*/ // RSHConn::Get - Get a file /*{{{*/ -- cgit v1.2.3 From 51561c4de75e84c2b2d037eb57387d3d3c2aa494 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 17:13:40 +0200 Subject: fix compiler warning in the new mirror code --- methods/http.h | 2 +- methods/mirror.cc | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) (limited to 'methods') diff --git a/methods/http.h b/methods/http.h index d0677bdaa..0bc019e77 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 - +#include using std::cout; using std::endl; diff --git a/methods/mirror.cc b/methods/mirror.cc index b2b6b2ecf..e8873d97b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -153,11 +153,12 @@ void MirrorMethod::CurrentQueueUriToMirror() return; // find current mirror and select next one - for (int i=0; i < AllMirrors.size(); i++) + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - if (Queue->Uri.find(AllMirrors[i]) == 0) + if (Queue->Uri.find(*mirror) == 0) { - Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + Queue->Uri.replace(0, mirror->length(), BaseUri); return; } } @@ -168,15 +169,19 @@ void MirrorMethod::CurrentQueueUriToMirror() bool MirrorMethod::TryNextMirror() { // find current mirror and select next one - for (int i=0; i < AllMirrors.size()-1; i++) + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - if (Queue->Uri.find(AllMirrors[i]) == 0) - { - Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); - if (Debug) - clog << "TryNextMirror: " << Queue->Uri << endl; - return true; - } + if (Queue->Uri.find(*mirror) != 0) + continue; + + vector::const_iterator nextmirror = mirror + 1; + if (nextmirror != AllMirrors.end()) + break; + Queue->Uri.replace(0, mirror->length(), *nextmirror); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; + return true; } if (Debug) -- cgit v1.2.3 From a319c4eeae62511d1cb58986742491d3e224bf20 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 17:19:40 +0200 Subject: * apt-pkg/indexcopy.cc: - move the gpg codecopy to a new method and use it also in methods/gpgv.cc --- methods/gpgv.cc | 47 ++++++----------------------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index a149d67dd..5f5f23f7d 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -58,26 +59,14 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, int fd[2]; FILE *pipein; int status; - string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - // FIXME: remove support for deprecated APT::GPGV setting - string const trustedFile = _config->FindFile("Dir::Etc::Trusted", - _config->Find("APT::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg").c_str()); - string const trustedPath = _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d"); - if (Debug == true) - { - std::clog << "gpgv path: " << gpgvpath << std::endl; - std::clog << "Keyring file: " << trustedFile << std::endl; - std::clog << "Keyring path: " << trustedPath << std::endl; - } - vector keyrings = GetListOfFilesInDir(trustedPath, "gpg", false); - if (FileExists(trustedFile) == true) - keyrings.push_back(trustedFile); - - if (keyrings.empty() == true) + string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); + std::vector Args = SigVerify::GetGPGVCommandLine(); + if (Args.empty() == true) { // TRANSLATOR: %s is the trusted keyring parts directory - ioprintf(ret, _("No keyring installed in %s."), trustedPath.c_str()); + ioprintf(ret, _("No keyring installed in %s."), + _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); return ret.str(); } @@ -89,32 +78,8 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) { - std::vector Args; - Args.reserve(30); - - Args.push_back(gpgvpath.c_str()); Args.push_back("--status-fd"); Args.push_back("3"); - Args.push_back("--ignore-time-conflict"); - for (vector::const_iterator K = keyrings.begin(); - K != keyrings.end(); ++K) - { - Args.push_back("--keyring"); - Args.push_back(K->c_str()); - } - - Configuration::Item const *Opts; - Opts = _config->Tree("Acquire::gpgv::Options"); - if (Opts != 0) - { - Opts = Opts->Child; - for (; Opts != 0; Opts = Opts->Next) - { - if (Opts->Value.empty() == true) - continue; - Args.push_back(Opts->Value.c_str()); - } - } Args.push_back(file); Args.push_back(outfile); Args.push_back(NULL); -- cgit v1.2.3 From cf440facb498fa0ec70148723b13d6d019758c0e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 23:20:26 +0200 Subject: enhance the split out of the gpgv commandline mangling by splitting out the call completely --- methods/gpgv.cc | 51 ++++++++++----------------------------------------- 1 file changed, 10 insertions(+), 41 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 5f5f23f7d..018e4f622 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -55,61 +55,29 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, if (Debug == true) std::clog << "inside VerifyGetSigners" << std::endl; - pid_t pid; int fd[2]; - FILE *pipein; - int status; - - string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - std::vector Args = SigVerify::GetGPGVCommandLine(); - if (Args.empty() == true) - { - // TRANSLATOR: %s is the trusted keyring parts directory - ioprintf(ret, _("No keyring installed in %s."), - _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); - return ret.str(); - } if (pipe(fd) < 0) return "Couldn't create pipe"; - pid = fork(); + pid_t pid = fork(); if (pid < 0) return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) { - Args.push_back("--status-fd"); - Args.push_back("3"); - Args.push_back(file); - Args.push_back(outfile); - Args.push_back(NULL); - - if (Debug == true) + if (SigVerify::RunGPGV(outfile, file, 3, fd) == false) { - std::clog << "Preparing to exec: " << gpgvpath; - for(std::vector::const_iterator a = Args.begin();*a != NULL; ++a) - std::clog << " " << *a; - std::clog << std::endl; + // TRANSLATOR: %s is the trusted keyring parts directory + ioprintf(ret, _("No keyring installed in %s."), + _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); + return ret.str(); } - int const nullfd = open("/dev/null", O_RDONLY); - close(fd[0]); - // Redirect output to /dev/null; we read from the status fd - dup2(nullfd, STDOUT_FILENO); - dup2(nullfd, STDERR_FILENO); - // Redirect the pipe to the status fd (3) - dup2(fd[1], 3); - - putenv((char *)"LANG="); - putenv((char *)"LC_ALL="); - putenv((char *)"LC_MESSAGES="); - execvp(gpgvpath.c_str(), (char **) &Args[0]); - exit(111); } close(fd[1]); - pipein = fdopen(fd[0], "r"); - + FILE *pipein = fdopen(fd[0], "r"); + // Loop over the output of gpgv, and check the signatures. size_t buffersize = 64; char *buffer = (char *) malloc(buffersize); @@ -182,6 +150,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, } fclose(pipein); + int status; waitpid(pid, &status, 0); if (Debug == true) { @@ -200,7 +169,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, } else if (WEXITSTATUS(status) == 111) { - ioprintf(ret, _("Could not execute '%s' to verify signature (is gpgv installed?)"), gpgvpath.c_str()); + ioprintf(ret, _("Could not execute 'gpgv' to verify signature (is gpgv installed?)")); return ret.str(); } else -- cgit v1.2.3 From 330463dd2374bd11757c6f2662f279fc31f035a0 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 24 Jun 2010 10:57:45 +0200 Subject: methods/ftp.h: Handle different logins are on the same server (Closes: #586904). --- methods/ftp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/ftp.h b/methods/ftp.h index 1bcea41b6..d7f1f7fbe 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -40,7 +40,7 @@ class FTPConn public: - bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; + bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port && Other.User == ServerName.User && Other.Password == ServerName.Password; }; // Raw connection IO bool ReadResp(unsigned int &Ret,string &Text); -- cgit v1.2.3 From 127e6df37213a1fda0dd5b44182acf678ccbbf02 Mon Sep 17 00:00:00 2001 From: "martin@piware.de" <> Date: Tue, 6 Jul 2010 13:14:57 +0200 Subject: methods/gzip.cc: With FileFd now being able to read gzipped files, there is no need for the gzip method any more to spawn an external gzip process. Rewrite it to use FileFd directly, which makes the code a lot simpler, and also using less memory and overhead. --- methods/gzip.cc | 63 +++++++-------------------------------------------------- 1 file changed, 7 insertions(+), 56 deletions(-) (limited to 'methods') diff --git a/methods/gzip.cc b/methods/gzip.cc index f732c0b86..72e3ac909 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -23,8 +23,6 @@ #include /*}}}*/ -const char *Prog; - class GzipMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); @@ -43,14 +41,12 @@ bool GzipMethod::Fetch(FetchItem *Itm) URI Get = Itm->Uri; string Path = Get.Host + Get.Path; // To account for relative paths - string GzPathOption = "Dir::bin::"+string(Prog); - FetchResult Res; Res.Filename = Itm->DestFile; URIStart(Res); // Open the source and destination files - FileFd From(Path,FileFd::ReadOnly); + FileFd From(Path,FileFd::ReadOnlyGzip); // if the file is empty, just rename it and return if(From.Size() == 0) @@ -59,40 +55,12 @@ bool GzipMethod::Fetch(FetchItem *Itm) return true; } - int GzOut[2]; - if (pipe(GzOut) < 0) - return _error->Errno("pipe",_("Couldn't open pipe for %s"),Prog); - - // Fork gzip - pid_t Process = ExecFork(); - if (Process == 0) - { - close(GzOut[0]); - dup2(From.Fd(),STDIN_FILENO); - dup2(GzOut[1],STDOUT_FILENO); - From.Close(); - close(GzOut[1]); - SetCloseExec(STDIN_FILENO,false); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[3]; - string Tmp = _config->Find(GzPathOption,Prog); - Args[0] = Tmp.c_str(); - Args[1] = "-d"; - Args[2] = 0; - execvp(Args[0],(char **)Args); - _exit(100); - } - From.Close(); - close(GzOut[1]); - - FileFd FromGz(GzOut[0]); // For autoclose FileFd To(Itm->DestFile,FileFd::WriteEmpty); To.EraseOnFailure(); if (_error->PendingError() == true) return false; - // Read data from gzip, generate checksums and write + // Read data from source, generate checksums and write Hashes Hash; bool Failed = false; while (1) @@ -100,36 +68,23 @@ bool GzipMethod::Fetch(FetchItem *Itm) unsigned char Buffer[4*1024]; unsigned long Count; - Count = read(GzOut[0],Buffer,sizeof(Buffer)); - if (Count < 0 && errno == EINTR) - continue; - - if (Count < 0) + if (!From.Read(Buffer,sizeof(Buffer),&Count)) { - _error->Errno("read", _("Read error from %s process"),Prog); - Failed = true; - break; + To.OpFail(); + return false; } - if (Count == 0) break; - + Hash.Add(Buffer,Count); if (To.Write(Buffer,Count) == false) { Failed = true; - FromGz.Close(); break; } } - // Wait for gzip to finish - if (ExecWait(Process,_config->Find(GzPathOption,Prog).c_str(),false) == false) - { - To.OpFail(); - return false; - } - + From.Close(); To.Close(); if (Failed == true) @@ -165,9 +120,5 @@ int main(int argc, char *argv[]) setlocale(LC_ALL, ""); GzipMethod Mth; - - Prog = strrchr(argv[0],'/'); - Prog++; - return Mth.Run(); } -- cgit v1.2.3 From b09663668c4d8203543e56b25db822ba55d21529 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Jul 2010 21:57:51 +0200 Subject: * methods/bzip2.cc: - add a copycat of the old gzip.cc as we need it for bzip2 and lzma --- methods/bzip2.cc | 177 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ methods/makefile | 20 ++++--- 2 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 methods/bzip2.cc (limited to 'methods') diff --git a/methods/bzip2.cc b/methods/bzip2.cc new file mode 100644 index 000000000..5da214bfc --- /dev/null +++ b/methods/bzip2.cc @@ -0,0 +1,177 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Bzip2 method - Take a file URI in and decompress it into the target + file. + + While the method is named "bzip2" it handles also other compression + types as it calls binaries based on the name of the method, + so it can also be used to handle gzip, lzma and others if named + correctly. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + /*}}}*/ + +const char *Prog; + +class Bzip2Method : public pkgAcqMethod +{ + virtual bool Fetch(FetchItem *Itm); + + public: + + Bzip2Method() : pkgAcqMethod("1.1",SingleInstance | SendConfig) {}; +}; + + +// Bzip2Method::Fetch - Decompress the passed URI /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool Bzip2Method::Fetch(FetchItem *Itm) +{ + URI Get = Itm->Uri; + string Path = Get.Host + Get.Path; // To account for relative paths + + string GzPathOption = "Dir::bin::"+string(Prog); + + FetchResult Res; + Res.Filename = Itm->DestFile; + URIStart(Res); + + // Open the source and destination files + FileFd From(Path,FileFd::ReadOnly); + + // if the file is empty, just rename it and return + if(From.Size() == 0) + { + rename(Path.c_str(), Itm->DestFile.c_str()); + return true; + } + + int GzOut[2]; + if (pipe(GzOut) < 0) + return _error->Errno("pipe",_("Couldn't open pipe for %s"),Prog); + + // Fork bzip2 + pid_t Process = ExecFork(); + if (Process == 0) + { + close(GzOut[0]); + dup2(From.Fd(),STDIN_FILENO); + dup2(GzOut[1],STDOUT_FILENO); + From.Close(); + close(GzOut[1]); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDOUT_FILENO,false); + + const char *Args[3]; + string Tmp = _config->Find(GzPathOption,Prog); + Args[0] = Tmp.c_str(); + Args[1] = "-d"; + Args[2] = 0; + execvp(Args[0],(char **)Args); + _exit(100); + } + From.Close(); + close(GzOut[1]); + + FileFd FromGz(GzOut[0]); // For autoclose + FileFd To(Itm->DestFile,FileFd::WriteEmpty); + To.EraseOnFailure(); + if (_error->PendingError() == true) + return false; + + // Read data from bzip2, generate checksums and write + Hashes Hash; + bool Failed = false; + while (1) + { + unsigned char Buffer[4*1024]; + unsigned long Count; + + Count = read(GzOut[0],Buffer,sizeof(Buffer)); + if (Count < 0 && errno == EINTR) + continue; + + if (Count < 0) + { + _error->Errno("read", _("Read error from %s process"),Prog); + Failed = true; + break; + } + + if (Count == 0) + break; + + Hash.Add(Buffer,Count); + if (To.Write(Buffer,Count) == false) + { + Failed = true; + FromGz.Close(); + break; + } + } + + // Wait for bzip2 to finish + if (ExecWait(Process,_config->Find(GzPathOption,Prog).c_str(),false) == false) + { + To.OpFail(); + return false; + } + + To.Close(); + + if (Failed == true) + return false; + + // Transfer the modification times + struct stat Buf; + if (stat(Path.c_str(),&Buf) != 0) + return _error->Errno("stat",_("Failed to stat")); + + struct utimbuf TimeBuf; + TimeBuf.actime = Buf.st_atime; + TimeBuf.modtime = Buf.st_mtime; + if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0) + return _error->Errno("utime",_("Failed to set modification time")); + + if (stat(Itm->DestFile.c_str(),&Buf) != 0) + return _error->Errno("stat",_("Failed to stat")); + + // Return a Done response + Res.LastModified = Buf.st_mtime; + Res.Size = Buf.st_size; + Res.TakeHashes(Hash); + + URIDone(Res); + + return true; +} + /*}}}*/ + +int main(int argc, char *argv[]) +{ + setlocale(LC_ALL, ""); + + Bzip2Method Mth; + + Prog = strrchr(argv[0],'/'); + Prog++; + + return Mth.Run(); +} diff --git a/methods/makefile b/methods/makefile index eabe85cfd..d94a85340 100644 --- a/methods/makefile +++ b/methods/makefile @@ -86,9 +86,16 @@ LIB_MAKES = apt-pkg/makefile SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) -# SSH and bzip2 method symlink -binary: $(BIN)/ssh $(BIN)/bzip2 $(BIN)/lzma -veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 clean-$(BIN)/lzma +# The gzip method +PROGRAM=bzip2 +SLIBS = -lapt-pkg $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = bzip2.cc +include $(PROGRAM_H) + +# SSH and lzma method symlink +binary: $(BIN)/ssh $(BIN)/lzma +veryclean: clean-$(BIN)/ssh clean-$(BIN)/lzma $(BIN)/ssh: echo "Installing ssh method link" @@ -96,13 +103,8 @@ $(BIN)/ssh: clean-$(BIN)/ssh: -rm $(BIN)/ssh -$(BIN)/bzip2: - echo "Installing bzip2 method link" - ln -fs gzip $(BIN)/bzip2 $(BIN)/lzma: echo "Installing lzma method link" - ln -fs gzip $(BIN)/lzma -clean-$(BIN)/bzip2: - -rm $(BIN)/bzip2 + ln -fs bzip2 $(BIN)/lzma clean-$(BIN)/lzma: -rm $(BIN)/lzma -- cgit v1.2.3 From 22041bd2864b8d0b401b45dde9eda4003a11fec4 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 10 Aug 2010 14:59:56 +0200 Subject: apt-pkg, methods: Convert users of WriteEmpty to WriteAtomic. --- methods/bzip2.cc | 2 +- methods/copy.cc | 2 +- methods/gzip.cc | 2 +- methods/rred.cc | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/bzip2.cc b/methods/bzip2.cc index 5da214bfc..19e49828e 100644 --- a/methods/bzip2.cc +++ b/methods/bzip2.cc @@ -91,7 +91,7 @@ bool Bzip2Method::Fetch(FetchItem *Itm) close(GzOut[1]); FileFd FromGz(GzOut[0]); // For autoclose - FileFd To(Itm->DestFile,FileFd::WriteEmpty); + FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) return false; diff --git a/methods/copy.cc b/methods/copy.cc index 027b59f46..a6bb372a3 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -52,7 +52,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) // See if the file exists FileFd From(File,FileFd::ReadOnly); - FileFd To(Itm->DestFile,FileFd::WriteEmpty); + FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) { diff --git a/methods/gzip.cc b/methods/gzip.cc index 72e3ac909..5b9b66b50 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -55,7 +55,7 @@ bool GzipMethod::Fetch(FetchItem *Itm) return true; } - FileFd To(Itm->DestFile,FileFd::WriteEmpty); + FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) return false; diff --git a/methods/rred.cc b/methods/rred.cc index f42c7a072..d51c45c85 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -446,7 +446,7 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ // the cleanup/closing of the fds) FileFd From(Path,FileFd::ReadOnly); FileFd Patch(Path+".ed",FileFd::ReadOnly); - FileFd To(Itm->DestFile,FileFd::WriteEmpty); + FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) return false; @@ -458,7 +458,7 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ // retry with patchFile lseek(Patch.Fd(), 0, SEEK_SET); lseek(From.Fd(), 0, SEEK_SET); - To.Open(Itm->DestFile,FileFd::WriteEmpty); + To.Open(Itm->DestFile,FileFd::WriteAtomic); if (_error->PendingError() == true) return false; if (patchFile(Patch, From, To, &Hash) != ED_OK) { -- cgit v1.2.3 From 6802b90c0871726f0018b2c3c059bf8c95a4a710 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 16 Aug 2010 12:12:51 +0200 Subject: * methods/bzip2.cc: - fix error checking for read in case of failing bzip2/lzma/whatever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spotted by -Wextra: bzip2.cc: In member function ‘virtual bool Bzip2Method::Fetch(pkgAcqMethod::FetchItem*)’: bzip2.cc:108:19: warning: comparison of unsigned expression < 0 is always false bzip2.cc:111:19: warning: comparison of unsigned expression < 0 is always false --- methods/bzip2.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/bzip2.cc b/methods/bzip2.cc index 19e49828e..241f21c66 100644 --- a/methods/bzip2.cc +++ b/methods/bzip2.cc @@ -102,9 +102,8 @@ bool Bzip2Method::Fetch(FetchItem *Itm) while (1) { unsigned char Buffer[4*1024]; - unsigned long Count; - Count = read(GzOut[0],Buffer,sizeof(Buffer)); + ssize_t Count = read(GzOut[0],Buffer,sizeof(Buffer)); if (Count < 0 && errno == EINTR) continue; -- cgit v1.2.3 From 4368851d55589a55fc10ff347890d8bbf33ee002 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 19 Aug 2010 00:04:05 +0200 Subject: * apt-pkg/init.cc: - set the default values for dir::etc::trusted options correctly --- methods/gpgv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 018e4f622..efe1f73f7 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -69,7 +69,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, { // TRANSLATOR: %s is the trusted keyring parts directory ioprintf(ret, _("No keyring installed in %s."), - _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); + _config->FindDir("Dir::Etc::TrustedParts").c_str()); return ret.str(); } exit(111); -- cgit v1.2.3 From de2b1358926d293c85277e1d9a58ac94915470ec Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 21 Aug 2010 00:52:59 +0200 Subject: * methods/http.cc: - some http servers violate HTTP1.1 by not issuing a Reason-Phrase (or at least a space after the code) especially for 200, but lets be nice and ignore it as we don't need the reason in general --- methods/http.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 9fa74bffa..25e31de9a 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -553,8 +553,14 @@ bool ServerState::HeaderLine(string Line) // Evil servers return no version if (Line[4] == '/') { - if (sscanf(Line.c_str(),"HTTP/%u.%u %u%[^\n]",&Major,&Minor, - &Result,Code) != 4) + int const elements = sscanf(Line.c_str(),"HTTP/%u.%u %u%[^\n]",&Major,&Minor,&Result,Code); + if (elements == 3) + { + Code[0] = '\0'; + if (Debug == true) + clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl; + } + else if (elements != 4) return _error->Error(_("The HTTP server sent an invalid reply header")); } else -- cgit v1.2.3