From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. (The tests and their binary helpers had to be slightly modified to apply, but the patch to fix the issue itself is unchanged.) Closes: 768797 --- methods/https.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'methods/https.cc') diff --git a/methods/https.cc b/methods/https.cc index 0499af0c5..65a744e2a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -76,13 +79,18 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; - return size*nmemb; + return buffer_size; } int -- cgit v1.2.3 From 9127d7aecf01f2999a2589e4b0503288518b2927 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Jan 2015 10:27:53 +0100 Subject: Fix missing URIStart() for https downloads Add a explicit ReceivedData to HttpsMethod that indicates when we got data from the connection so that we can send URISTart() to the parent. This is needed because URIStart got moved in f9b4f12d from the progress_callback to write_data() and it only checks for Res.Size. In the old code if progress_callback is called by libcurl (and sets Res.Size) before write_data is called then URIStart() is never send. Making this a explicit ReceivedData variable fixes this issue. --- methods/https.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'methods/https.cc') diff --git a/methods/https.cc b/methods/https.cc index 65a744e2a..3a5981b58 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -85,8 +85,12 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if (me->Server->JunkSize != 0) return buffer_size; - if (me->Res.Size == 0) + if (me->ReceivedData == false) + { me->URIStart(me->Res); + me->ReceivedData = true; + } + if(me->File->Write(buffer, buffer_size) != true) return false; @@ -95,7 +99,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) int HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/, - double /*ultotal*/, double /*ulnow*/) + double /*ultotal*/, double /*ulnow*/) { HttpsMethod *me = (HttpsMethod *)clientp; if(dltotal > 0 && me->Res.Size == 0) { @@ -179,6 +183,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) char curl_errorstr[CURL_ERROR_SIZE]; URI Uri = Itm->Uri; string remotehost = Uri.Host; + ReceivedData = false; // TODO: // - http::Pipeline-Depth -- cgit v1.2.3