From d7518dba50e2285c41c7002a1d86f876401fd9ea Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 24 Jul 2017 14:30:41 +0200 Subject: fail earlier if server answers with too much data We tend to operate on rather large static files, which means we usually get Content-Length information from the server. If we combine this information with the filesize we are expecting (factoring in pipelining) we can avoid reading a bunch of data we are ending up rejecting anyhow by just closing the connection saving bandwidth and time both for the server as well as the client. --- methods/curl.cc | 4 ++-- methods/ftp.cc | 4 ++-- methods/http.cc | 16 +++++++++++++--- test/integration/test-apt-update-expected-size | 14 ++++++-------- .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 4 ++-- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/methods/curl.cc b/methods/curl.cc index dfdd3b06b..a19318098 100644 --- a/methods/curl.cc +++ b/methods/curl.cc @@ -139,8 +139,8 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if (TotalWritten > me->https->Queue->MaximumSize) { me->https->SetFailReason("MaximumSizeExceeded"); - _error->Error("Writing more data than expected (%llu > %llu)", - TotalWritten, me->https->Queue->MaximumSize); + _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"), + TotalWritten, me->https->Queue->MaximumSize); return 0; } } diff --git a/methods/ftp.cc b/methods/ftp.cc index 9bfe72bc6..dd97458d0 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -940,8 +940,8 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, if (MaximumSize > 0 && To.Tell() > MaximumSize) { Owner->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", - To.Tell(), MaximumSize); + return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"), + To.Tell(), MaximumSize); } } diff --git a/methods/http.cc b/methods/http.cc index 9f01d15b3..9425145dd 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -267,6 +267,7 @@ bool CircleBuf::Write(string &Data) OutP += LeftWrite(); return true; } + /*}}}*/ // CircleBuf::Stats - Print out stats information /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -282,11 +283,12 @@ void CircleBuf::Stats() clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/ } /*}}}*/ -CircleBuf::~CircleBuf() +CircleBuf::~CircleBuf() /*{{{*/ { delete [] Buf; delete Hash; } + /*}}}*/ // UnwrapHTTPConnect - Does the HTTP CONNECT handshake /*{{{*/ // --------------------------------------------------------------------- @@ -590,7 +592,15 @@ bool HttpServerState::RunData(RequestState &Req) if (Req.JunkSize != 0) In.Limit(Req.JunkSize); else if (Req.DownloadSize != 0) + { + if (Req.MaximumSize != 0 && Req.DownloadSize > Req.MaximumSize) + { + Owner->SetFailReason("MaximumSizeExceeded"); + return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"), + Req.DownloadSize, Req.MaximumSize); + } In.Limit(Req.DownloadSize); + } else if (Persistent == false) In.Limit(-1); @@ -827,8 +837,8 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize) { Owner->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", - Req.File.Tell(), Req.MaximumSize); + return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"), + Req.File.Tell(), Req.MaximumSize); } // Handle commands from APT diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 710d05d18..5c73a2396 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -21,7 +21,7 @@ test_inreleasetoobig() { testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 msgtest 'Check that the max write warning is triggered' cp rootdir/tmp/testsuccess.output update.output - testsuccess --nomsg grep -q 'Writing more data than expected' update.output + testsuccess --nomsg grep -q 'File is larger than expected' update.output rm -f update.output # ensure the failed InRelease file got renamed testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED @@ -39,7 +39,7 @@ test_packagestoobig() { touch -d '+1hour' "$pkg" done NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages.gz)" - testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz Writing more data than expected ($NEW_SIZE > $SIZE) + testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz File is larger than expected ($NEW_SIZE > $SIZE). Mirror sync in progress? E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::Transaction=0 testsuccess ls rootdir/var/lib/apt/lists/partial/*Packages*.FAILED testfailure test -e rootdir/var/lib/apt/lists/partial/Old.FAILED @@ -51,29 +51,27 @@ methodtest() { webserverconfig 'aptwebserver::support::last-modified' 'false' "$1" # curl is clever and sees hits here also msgmsg 'Test with' "$1" 'and clean start' - rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good + rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good aptarchive/dists + cp -a aptarchive/dists.good aptarchive/dists # normal update works fine testsuccess aptget update touch rootdir/var/lib/apt/lists/partial/Old.FAILED mv rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good - # starting fresh works test_inreleasetoobig "$1" rm -rf aptarchive/dists rootdir/var/lib/apt/lists cp -a aptarchive/dists.good aptarchive/dists test_packagestoobig "$1" - rm -rf aptarchive/dists rootdir/var/lib/apt/lists - cp -a aptarchive/dists.good aptarchive/dists msgmsg 'Test with' "$1" 'and existing old data' + rm -rf aptarchive/dists rootdir/var/lib/apt/lists cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists test_inreleasetoobig "$1" rm -rf aptarchive/dists rootdir/var/lib/apt/lists cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists cp -a aptarchive/dists.good aptarchive/dists test_packagestoobig "$1" - rm -rf aptarchive/dists - cp -a aptarchive/dists.good aptarchive/dists } changetowebserver diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum index 2636c3140..14e5a20b8 100755 --- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -216,7 +216,7 @@ testmismatch() { Need to get 6 B of source archives. Get:1 http://localhost:${APTHTTPPORT} $1 1.0 (dsc) [2 B] Err:1 http://localhost:${APTHTTPPORT} $1 1.0 (dsc) - Writing more data than expected (3 > 2) + File is larger than expected (3 > 2). Mirror sync in progress? Hashes of expected file: - SHA256:943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a - Filesize:2 [weak] @@ -230,7 +230,7 @@ Err:2 http://localhost:${APTHTTPPORT} $1 1.0 (tar) - SHA256:90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb - Filesize:3 [weak] Last modification reported: $(lastmodification "aptarchive/${1}_1.0.dsc") -E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.dsc Writing more data than expected (3 > 2) +E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.dsc File is larger than expected (3 > 2). Mirror sync in progress? Hashes of expected file: - SHA256:943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a - Filesize:2 [weak] -- cgit v1.2.3