fail early in http if server answer is too small as well

Failing on too much data is good, but we can do better by checking for
exact filesizes as we know with hashsums how large a file should be, so
if we get a file which has a size we do not expect we can drop it
directly, regardless of if the file is larger or smaller than what we
expect which should catch most cases which would end up as hashsum
errors later now a lot sooner.

4 files changed, 37 insertions, 6 deletions

diff --git a/methods/basehttp.cc b/methods/basehttp.cc
index 47dabf960..cc5039c75 100644
--- a/methods/basehttp.cc
+++ b/methods/basehttp.cc
@@ -660,8 +660,39 @@ int BaseHttpMethod::Loop()
             // so instead we use the size of the biggest item in the queue
             Req.MaximumSize = FindMaximumObjectSizeInQueue();
 
-            if (Req.HaveContent)
-	       Result = Server->RunData(Req);
+	    if (Req.HaveContent)
+	    {
+	       /* If the server provides Content-Length we can figure out with it if
+		  this satisfies any request we have made so far (in the pipeline).
+		  If not we can kill the connection as whatever file the server is trying
+		  to send to us would be rejected with a hashsum mismatch later or triggers
+		  a maximum size error. We don't run the data to /dev/null as this can be MBs
+		  of junk data we would waste bandwidth on and instead just close the connection
+		  to reopen a fresh one which should be more cost/time efficient */
+	       if (Req.DownloadSize > 0)
+	       {
+		  decltype(Queue->ExpectedHashes.FileSize()) const filesize = Req.StartPos + Req.DownloadSize;
+		  bool found = false;
+		  for (FetchItem const *I = Queue; I != 0 && I != QueueBack; I = I->Next)
+		  {
+		     auto const fs = I->ExpectedHashes.FileSize();
+		     if (fs == 0 || fs == filesize)
+		     {
+			found = true;
+			break;
+		     }
+		  }
+		  if (found == false)
+		  {
+		     SetFailReason("MaximumSizeExceeded");
+		     _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
+				   filesize, Queue->ExpectedHashes.FileSize());
+		     Result = false;
+		  }
+	       }
+	       if (Result)
+		  Result = Server->RunData(Req);
+	    }
 
 	    /* If the server is sending back sizeless responses then fill in
 	       the size now */
diff --git a/methods/curl.cc b/methods/curl.cc
index a19318098..71149217a 100644
--- a/methods/curl.cc
+++ b/methods/curl.cc
@@ -139,7 +139,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
       if (TotalWritten > me->https->Queue->MaximumSize)
       {
 	 me->https->SetFailReason("MaximumSizeExceeded");
-	 _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+	 _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
 		       TotalWritten, me->https->Queue->MaximumSize);
 	 return 0;
       }
diff --git a/methods/ftp.cc b/methods/ftp.cc
index dd97458d0..4972337e3 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -940,7 +940,7 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
       if (MaximumSize > 0 && To.Tell() > MaximumSize)
       {
          Owner->SetFailReason("MaximumSizeExceeded");
-	 return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+	 return _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
 			      To.Tell(), MaximumSize);
       }
    }
diff --git a/methods/http.cc b/methods/http.cc
index 9425145dd..db4542981 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -596,7 +596,7 @@ bool HttpServerState::RunData(RequestState &Req)
 	 if (Req.MaximumSize != 0 && Req.DownloadSize > Req.MaximumSize)
 	 {
 	    Owner->SetFailReason("MaximumSizeExceeded");
-	    return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+	    return _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
 				 Req.DownloadSize, Req.MaximumSize);
 	 }
 	 In.Limit(Req.DownloadSize);
@@ -837,7 +837,7 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req)
    if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize)
    {
       Owner->SetFailReason("MaximumSizeExceeded");
-      return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+      return _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
 			   Req.File.Tell(), Req.MaximumSize);
    }