From 1296bc7c466181a7978c313c40a041b34ce3eaeb Mon Sep 17 00:00:00 2001 From: Robert Edmonds Date: Sun, 22 Mar 2015 00:12:45 -0400 Subject: HttpsMethod::Fetch(): Zero the FetchResult object when leaving due to 404 --- methods/https.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3a5981b58..f2b00dd64 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -402,6 +402,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm) _error->Error("%s", err); // unlink, no need keep 401/404 page content in partial/ unlink(File->Name().c_str()); + Res.Size = 0; + Res.LastModified = 0; return false; } -- cgit v1.2.3 From 3a53f6a1510d332e24c3330a69b987f2341d1a94 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 13 Apr 2015 12:57:22 -0400 Subject: Revert "HttpsMethod::Fetch(): Zero the FetchResult object when leaving due to 404" This reverts commit 1296bc7c466181a7978c313c40a041b34ce3eaeb. --- methods/https.cc | 2 -- 1 file changed, 2 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index f2b00dd64..3a5981b58 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -402,8 +402,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) _error->Error("%s", err); // unlink, no need keep 401/404 page content in partial/ unlink(File->Name().c_str()); - Res.Size = 0; - Res.LastModified = 0; return false; } -- cgit v1.2.3 From ceafe8a6edc815df2923ba892894617829e9d3c2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 May 2015 15:28:53 +0200 Subject: Fix endless loop in apt-get update that can cause disk fillup The apt http code parses Content-Length and Content-Range. For both requests the variable "Size" is used and the semantic for this Size is the total file size. However Content-Length is not the entire file size for partital file requests. For servers that send the Content-Range header first and then the Content-Length header this can lead to globbing of Size so that its less than the real file size. This may lead to a subsequent passing of a negative number into the CircleBuf which leads to a endless loop that writes data. Thanks to Anton Blanchard for the analysis and initial patch. LP: #1445239 --- methods/http.cc | 2 +- methods/server.cc | 20 +++++++++++++++----- methods/server.h | 3 ++- 3 files changed, 18 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 1b996db98..ad90c9891 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -443,7 +443,7 @@ bool HttpServerState::RunData(FileFd * const File) else if (JunkSize != 0) In.Limit(JunkSize); else - In.Limit(Size - StartPos); + In.Limit(DownloadSize); // Just transfer the whole block. do diff --git a/methods/server.cc b/methods/server.cc index e321e0230..ba0a8864b 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -164,15 +164,22 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - unsigned long long * SizePtr = &Size; + unsigned long long * DownloadSizePtr = &DownloadSize; if (Result == 416) - SizePtr = &JunkSize; + DownloadSizePtr = &JunkSize; - *SizePtr = strtoull(Val.c_str(), NULL, 10); - if (*SizePtr >= std::numeric_limits::max()) + *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10); + if (*DownloadSizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (*SizePtr == 0) + else if (*DownloadSizePtr == 0) HaveContent = false; + + // On partial content (206) the Content-Length less than the real + // size, so do not set it here but leave that to the Content-Range + // header instead + if(Result != 206 && Size == 0) + Size = DownloadSize; + return true; } @@ -193,6 +200,9 @@ bool ServerState::HeaderLine(string Line) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) return _error->Error(_("This HTTP server has broken range support")); + + // figure out what we will download + DownloadSize = Size - StartPos; return true; } diff --git a/methods/server.h b/methods/server.h index 1b81e3549..ed3cb456a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long Size; // total size of the usable content (aka: the file) + unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content) unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; -- cgit v1.2.3 From 6291f60e86718697f261519a6818e1d5ee433216 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 May 2015 15:40:18 +0200 Subject: Rename "Size" in ServerState to TotalFileSize The variable "Size" was misleading and caused bug #1445239. To avoid similar issues in the future, rename it to make the meaning more obvious. git-dch: ignore --- methods/https.cc | 4 ++-- methods/server.cc | 20 ++++++++++---------- methods/server.h | 14 ++++++++++---- 3 files changed, 22 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3a5981b58..12fc6c70f 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -55,10 +55,10 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { if (me->Server->Result != 416 && me->Server->StartPos != 0) ; - else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize()) + else if (me->Server->Result == 416 && me->Server->TotalFileSize == me->File->FileSize()) { me->Server->Result = 200; - me->Server->StartPos = me->Server->Size; + me->Server->StartPos = me->Server->TotalFileSize; // the actual size is not important for https as curl will deal with it // by itself and e.g. doesn't bother us with transport-encoding… me->Server->JunkSize = std::numeric_limits::max(); diff --git a/methods/server.cc b/methods/server.cc index ba0a8864b..6c05700a5 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -54,7 +54,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Major = 0; Minor = 0; Result = 0; - Size = 0; + TotalFileSize = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; @@ -177,8 +177,8 @@ bool ServerState::HeaderLine(string Line) // On partial content (206) the Content-Length less than the real // size, so do not set it here but leave that to the Content-Range // header instead - if(Result != 206 && Size == 0) - Size = DownloadSize; + if(Result != 206 && TotalFileSize == 0) + TotalFileSize = DownloadSize; return true; } @@ -194,15 +194,15 @@ bool ServerState::HeaderLine(string Line) HaveContent = true; // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 - if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) + if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1) ; // we got the expected filesize which is all we wanted - else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) + else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); - if ((unsigned long long)StartPos > Size) + if ((unsigned long long)StartPos > TotalFileSize) return _error->Error(_("This HTTP server has broken range support")); // figure out what we will download - DownloadSize = Size - StartPos; + DownloadSize = TotalFileSize - StartPos; return true; } @@ -313,7 +313,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) struct stat SBuf; if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { - if ((unsigned long long)SBuf.st_size == Server->Size) + if ((unsigned long long)SBuf.st_size == Server->TotalFileSize) { // the file is completely downloaded, but was not moved if (Server->HaveContent == true) @@ -323,7 +323,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) Server->RunData(&DevNull); } Server->HaveContent = false; - Server->StartPos = Server->Size; + Server->StartPos = Server->TotalFileSize; Server->Result = 200; } else if (unlink(Queue->DestFile.c_str()) == 0) @@ -349,7 +349,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // This is some sort of 2xx 'data follows' reply Res.LastModified = Server->Date; - Res.Size = Server->Size; + Res.Size = Server->TotalFileSize; // Open the file delete File; diff --git a/methods/server.h b/methods/server.h index ed3cb456a..8c14282b6 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,10 +34,16 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; // total size of the usable content (aka: the file) - unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content) - unsigned long long JunkSize; // size of junk content (aka: server error pages) + + // total size of the usable content (aka: the file) + unsigned long long TotalFileSize; + // size we actually download (can be smaller than Size if we have partial content) + unsigned long long DownloadSize; + // size of junk content (aka: server error pages) + unsigned long long JunkSize; + // The start of the data (for partial content) unsigned long long StartPos; + time_t Date; bool HaveContent; enum {Chunked,Stream,Closes} Encoding; @@ -73,7 +79,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; TotalFileSize = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true;}; virtual bool WriteResponse(std::string const &Data) = 0; -- cgit v1.2.3 From 65759e00eff0513c34f584b99420b72fe0e5073e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 May 2015 16:27:08 +0200 Subject: Update methods/https.cc now that ServerState::Size is renamed Git-Dch: ignore --- methods/https.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 81060122c..c97367323 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,7 +69,7 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) me->File->Truncate(me->Server->StartPos); me->File->Seek(me->Server->StartPos); - me->Res.Size = me->Server->Size; + me->Res.Size = me->Server->TotalFileSize; } else if (me->Server->HeaderLine(line) == false) return 0; -- cgit v1.2.3