From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/http.cc | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index c6623c46f..deaa8d0c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,7 +38,6 @@ #include #include #include -#include #include // Internet stuff @@ -57,7 +56,7 @@ time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; bool Debug = false; - +URI Proxy; unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; @@ -990,7 +989,7 @@ void HttpMethod::SigTerm(int) depth. */ bool HttpMethod::Fetch(FetchItem *) { - if (Server == 0) + if (Server == 0) return true; // Queue the requests @@ -1223,13 +1222,5 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - - HttpMethod Mth; - - return Mth.Loop(); -} -- cgit v1.2.3 From 59271f62e4a291c8d96e1f6073203c395734b6ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 17:52:28 +0100 Subject: * use pkgAcqMethod::FailReason() for consistent error reporting --- methods/http.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index deaa8d0c8..01ad14655 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,6 +914,9 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) failure */ if (Srv->Result < 200 || Srv->Result >= 300) { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result); + SetFailReason(err); _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) return 4; -- cgit v1.2.3 From 484befd1518245e795ed92e6c97ad50786f3b4a2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 11 Mar 2008 09:59:32 +0100 Subject: * typo fixes (LP: #107960) --- methods/http.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index 6aa4261ff..61321c850 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -367,8 +367,8 @@ bool ServerState::Close() /*}}}*/ // ServerState::RunHeaders - Get the headers before the data /*{{{*/ // --------------------------------------------------------------------- -/* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header - parse error occured */ +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ int ServerState::RunHeaders() { State = Header; -- cgit v1.2.3 From ebb461fdf02ee3e038d4b3a4ab1a0a60188edf9a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Feb 2009 12:14:36 +0100 Subject: * [ABI break] merge support for http redirects, thanks to Jeff Licquia and Anthony Towns * [ABI break] use int for the package IDs (thanks to Steve Cotton) - Galician updated. Closes: #509151 - Catalan updated. Closes: #509375 - Vietnamese updated. Closes: #509422 - Traditional Chinese added. Closes: #510664 * COPYING: - Actualized. Removed obsolete Qt section, added GPLv2 clause. (Closes: #440049, #509337) * Clarify the --help for 'purge' (LP: #243948) --- methods/http.cc | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index b0fb89fda..44274bd78 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include // Internet stuff @@ -56,6 +57,7 @@ int HttpMethod::FailFd = -1; time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; +bool AllowRedirect = false; bool Debug = false; URI Proxy; @@ -627,6 +629,12 @@ bool ServerState::HeaderLine(string Line) return true; } + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + return true; } /*}}}*/ @@ -899,7 +907,9 @@ bool HttpMethod::ServerDie(ServerState *Srv) 1 - IMS hit 3 - Unrecoverable error 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) */ + 5 - Unrecoverable non-server error (close the connection) + 6 - Try again with a new or changed URI + */ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified @@ -911,6 +921,27 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) return 1; } + /* Redirect + * + * Note that it is only OK for us to treat all redirection the same + * because we *always* use GET, not other HTTP methods. There are + * three redirection codes for which it is not appropriate that we + * redirect. Pass on those codes so the error handling kicks in. + */ + if (AllowRedirect + && (Srv->Result > 300 && Srv->Result < 400) + && (Srv->Result != 300 // Multiple Choices + && Srv->Result != 304 // Not Modified + && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved) + { + if (!Srv->Location.empty()) + { + NextURI = Srv->Location; + return 6; + } + /* else pass through for error message */ + } + /* We have a reply we dont handle. This should indicate a perm server failure */ if (Srv->Result < 200 || Srv->Result >= 300) @@ -1028,6 +1059,7 @@ bool HttpMethod::Configuration(string Message) if (pkgAcqMethod::Configuration(Message) == false) return false; + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); @@ -1041,6 +1073,10 @@ bool HttpMethod::Configuration(string Message) /* */ int HttpMethod::Loop() { + typedef vector StringVector; + typedef vector::iterator StringVectorIterator; + map Redirected; + signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -1227,6 +1263,46 @@ int HttpMethod::Loop() break; } + // Try again with a new URL + case 6: + { + // Clear rest of response if there is content + if (Server->HaveContent) + { + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(); + delete File; + File = 0; + } + + /* Detect redirect loops. No more redirects are allowed + after the same URI is seen twice in a queue item. */ + StringVector &R = Redirected[Queue->DestFile]; + bool StopRedirects = false; + if (R.size() == 0) + R.push_back(Queue->Uri); + else if (R[0] == "STOP" || R.size() > 10) + StopRedirects = true; + else + { + for (StringVectorIterator I = R.begin(); I != R.end(); I++) + if (Queue->Uri == *I) + { + R[0] = "STOP"; + break; + } + + R.push_back(Queue->Uri); + } + + if (StopRedirects == false) + Redirect(NextURI); + else + Fail(); + + break; + } + default: Fail(_("Internal error")); break; -- cgit v1.2.3 From 056539eeb8eb84cea65c0c15466111b978ee156d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Jul 2009 18:25:16 +0200 Subject: methods/http.cc: remove merge artifact --- methods/http.cc | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index 476fb751f..859dbd39c 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -1316,15 +1316,4 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - - HttpMethod Mth; - return Mth.Loop(); -} - -- cgit v1.2.3 From 7273e49443e480d57bd8455f9cf9a0f39ef181f4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 7 May 2010 11:20:31 +0200 Subject: * methods/http.cc: - code cleanup, add (some) doxygen strings --- methods/http.cc | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index b05444691..c05abc862 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -67,7 +67,7 @@ unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; struct timeval CircleBuf::BwReadTick={0,0}; const unsigned int CircleBuf::BW_HZ=10; - + // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -378,7 +378,7 @@ bool ServerState::Close() // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ -int ServerState::RunHeaders() +ServerState::RunHeadersResult ServerState::RunHeaders() { State = Header; @@ -407,7 +407,7 @@ int ServerState::RunHeaders() string::const_iterator J = I; for (; J != Data.end() && *J != '\n' && *J != '\r';J++); if (HeaderLine(string(I,J)) == false) - return 2; + return RUN_HEADERS_PARSE_ERROR; I = J; } @@ -419,11 +419,11 @@ int ServerState::RunHeaders() if (Encoding == Closes && HaveContent == true) Persistent = false; - return 0; + return RUN_HEADERS_OK; } while (Owner->Go(false,this) == true); - return 1; + return RUN_HEADERS_IO_ERROR; } /*}}}*/ // ServerState::RunData - Transfer the data from the socket /*{{{*/ @@ -922,7 +922,8 @@ bool HttpMethod::ServerDie(ServerState *Srv) 5 - Unrecoverable non-server error (close the connection) 6 - Try again with a new or changed URI */ -int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) +HttpMethod::DealWithHeadersResult +HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified if (Srv->Result == 304) @@ -930,7 +931,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) unlink(Queue->DestFile.c_str()); Res.IMSHit = true; Res.LastModified = Queue->LastModified; - return 1; + return IMS_HIT; } /* Redirect @@ -949,7 +950,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) if (!Srv->Location.empty()) { NextURI = Srv->Location; - return 6; + return TRY_AGAIN_OR_REDIRECT; } /* else pass through for error message */ } @@ -960,8 +961,8 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) - return 4; - return 3; + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; } // This is some sort of 2xx 'data follows' reply @@ -972,7 +973,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) delete File; File = new FileFd(Queue->DestFile,FileFd::WriteAny); if (_error->PendingError() == true) - return 5; + return ERROR_NOT_FROM_SERVER; FailFile = Queue->DestFile; FailFile.c_str(); // Make sure we dont do a malloc in the signal handler @@ -1000,13 +1001,13 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false) { _error->Errno("read",_("Problem hashing file")); - return 5; + return ERROR_NOT_FROM_SERVER; } lseek(File->Fd(),0,SEEK_END); } SetNonBlock(File->Fd(),true); - return 0; + return FILE_IS_OPEN; } /*}}}*/ // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/ @@ -1147,11 +1148,11 @@ int HttpMethod::Loop() // Fetch the next URL header data from the server. switch (Server->RunHeaders()) { - case 0: + case ServerState::RUN_HEADERS_OK: break; // The header data is bad - case 2: + case ServerState::RUN_HEADERS_PARSE_ERROR: { _error->Error(_("Bad header data")); Fail(true); @@ -1161,7 +1162,7 @@ int HttpMethod::Loop() // The server closed a connection during the header get.. default: - case 1: + case ServerState::RUN_HEADERS_IO_ERROR: { FailCounter++; _error->Discard(); @@ -1185,7 +1186,7 @@ int HttpMethod::Loop() switch (DealWithHeaders(Res,Server)) { // Ok, the file is Open - case 0: + case FILE_IS_OPEN: { URIStart(Res); @@ -1238,21 +1239,21 @@ int HttpMethod::Loop() } // IMS hit - case 1: + case IMS_HIT: { URIDone(Res); break; } // Hard server error, not found or something - case 3: + case ERROR_UNRECOVERABLE: { Fail(); break; } // Hard internal error, kill the connection and fail - case 5: + case ERROR_NOT_FROM_SERVER: { delete File; File = 0; @@ -1264,7 +1265,7 @@ int HttpMethod::Loop() } // We need to flush the data, the header is like a 404 w/ error text - case 4: + case ERROR_WITH_CONTENT_PAGE: { Fail(); @@ -1277,7 +1278,7 @@ int HttpMethod::Loop() } // Try again with a new URL - case 6: + case TRY_AGAIN_OR_REDIRECT: { // Clear rest of response if there is content if (Server->HaveContent) -- cgit v1.2.3 From f64684a40c97fc52bcd7942bed81bf08b02fdd28 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 25 May 2010 17:45:12 +0200 Subject: methods/http.cc: shorten comment on return values of DealWithHeaders (because we use a enum now) --- methods/http.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index c05abc862..d43dd14c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,13 +914,7 @@ bool HttpMethod::ServerDie(ServerState *Srv) // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ // --------------------------------------------------------------------- /* We look at the header data we got back from the server and decide what - to do. Returns - 0 - File is open, - 1 - IMS hit - 3 - Unrecoverable error - 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) - 6 - Try again with a new or changed URI + to do. Returns DealWithHeadersResult (see http.h for details). */ HttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) -- cgit v1.2.3 From 96cc64a521957d63704de72ed95f1c839698c53c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 00:53:44 +0200 Subject: move the users away from the deprecated StrToTime() method --- methods/http.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index d43dd14c8..5fdc62696 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -631,7 +631,7 @@ bool ServerState::HeaderLine(string Line) if (stringcasecmp(Tag,"Last-Modified:") == 0) { - if (StrToTime(Val,Date) == false) + if (RFC1123StrToTime(Val.c_str(), Date) == false) return _error->Error(_("Unknown date format")); return true; } -- cgit v1.2.3