From ed793a19ec00b83254029509bc516e3ba911c75a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. Closes: 768797 --- test/integration/test-partial-file-support | 62 +++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 6 deletions(-) (limited to 'test/integration/test-partial-file-support') diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 98b2f242a..b6b305d25 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -24,13 +24,25 @@ testdownloadfile() { else msgpass fi - cat "$DOWNLOADLOG" | while read field hash; do + sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log + testsuccess test -s receivedhashes.log + local HASHES_OK=0 + local HASHES_BAD=0 + while read field hash; do local EXPECTED case "$field" in 'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";; + 'Checksum-FileSize-Hash:') + #filesize is too weak to check for != + if [ "$4" = '=' ]; then + EXPECTED="$(stat -c '%s' "$TESTFILE")" + else + continue + fi + ;; *) continue;; esac if [ "$4" = '=' ]; then @@ -40,15 +52,41 @@ testdownloadfile() { fi if [ "$EXPECTED" "$4" "$hash" ]; then msgpass + HASHES_OK=$((HASHES_OK+1)); else - cat >&2 "$DOWNLOADLOG" msgfail "expected: $EXPECTED ; got: $hash" + HASHES_BAD=$((HASHES_BAD+1)); fi - done + done < receivedhashes.log + msgtest 'At least one good hash and no bad ones' + if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then + cat >&2 "$DOWNLOADLOG" + msgfail + else + msgpass + fi } TESTFILE='aptarchive/testfile' cp -a ${TESTDIR}/framework $TESTFILE +cp -a ${TESTDIR}/framework "${TESTFILE}2" + +followuprequest() { + local DOWN='./downloaded/testfile' + + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' + testwebserverlaststatuscode '416' "$DOWNLOADLOG" + + copysource $TESTFILE 1M $DOWN + copysource "${TESTFILE}2" 20 "${DOWN}2" + msgtest 'Testing download of files with' 'completely downloaded file + partial file' + testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \ + download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2" + testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output' + testsuccess diff -u "$TESTFILE" "${DOWN}" + testsuccess diff -u "${DOWN}" "${DOWN}2" +} testrun() { webserverconfig 'aptwebserver::support::range' 'true' @@ -66,9 +104,11 @@ testrun() { testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource $TESTFILE 1M $DOWN - testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' - testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::closeOnError' 'false' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'true' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'false' copysource /dev/zero 1M $DOWN testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '=' @@ -86,8 +126,18 @@ testrun() { testwebserverlaststatuscode '200' "$DOWNLOADLOG" } +msgmsg 'http: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'http://localhost:8080' +msgmsg 'http: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'http://localhost:8080' changetohttpswebserver +msgmsg 'https: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'https://localhost:4433' +msgmsg 'https: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'https://localhost:4433' -- cgit v1.2.3 From 146f7715a9f36d246b461255b3c683b479296915 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Apr 2015 10:59:27 +0200 Subject: improve partial/ cleanup in abort and failure cases Especially pdiff-enhanced downloads have the tendency to fail for various reasons from which we can recover and even a successful download used to leave the old unpatched index in partial/. By adding a new method responsible for making the transaction of an individual file happen we can at specialisations especially for abort cases to deal with the cleanup. This also helps in keeping the compressed indexes around if another index failed instead of keeping the decompressed files, which we wouldn't pick up in the next call. --- test/integration/test-partial-file-support | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'test/integration/test-partial-file-support') diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index b6b305d25..85046b3eb 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -126,18 +126,17 @@ testrun() { testwebserverlaststatuscode '200' "$DOWNLOADLOG" } -msgmsg 'http: Test with Content-Length' -webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' -testrun 'http://localhost:8080' -msgmsg 'http: Test with Transfer-Encoding: chunked' -webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' -testrun 'http://localhost:8080' +serverconfigs() { + msgmsg "${1%%:*}: Test with Content-Length" + webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' + testrun "$1" + msgmsg "${1%%:*}: Test with Transfer-Encoding: chunked" + webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' + testrun "$1" +} + +serverconfigs 'http://localhost:8080' changetohttpswebserver -msgmsg 'https: Test with Content-Length' -webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' -testrun 'https://localhost:4433' -msgmsg 'https: Test with Transfer-Encoding: chunked' -webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' -testrun 'https://localhost:4433' +serverconfigs 'https://localhost:4433' -- cgit v1.2.3 From dcbb364fc69e1108b3fea3adb12a7ba83d9af467 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 12 May 2015 00:30:16 +0200 Subject: detect 416 complete file in partial by expected hash If we have the expected hashes we can check with them if the file we have in partial we got a 416 for is the expected file. We detected this with same-size before, but not every server sends a good Content-Range header with a 416 response. --- test/integration/test-partial-file-support | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'test/integration/test-partial-file-support') diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 85046b3eb..c07af7bd0 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -17,8 +17,8 @@ DOWNLOADLOG='rootdir/tmp/testdownloadfile.log' testdownloadfile() { rm -f "$DOWNLOADLOG" - msgtest "Testing download of file $2 with" "$1" - if ! downloadfile "$2" "$3" > "$DOWNLOADLOG"; then + msgtest "Testing download of file $2 with" "$1 $5" + if ! downloadfile "$2" "$3" "$5" > "$DOWNLOADLOG"; then cat >&2 "$DOWNLOADLOG" msgfail else @@ -78,6 +78,12 @@ followuprequest() { testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::support::content-range' 'false' + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' "SHA1:$(sha1sum "$TESTFILE" | cut -d' ' -f 1)" + testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::support::content-range' 'true' + copysource $TESTFILE 1M $DOWN copysource "${TESTFILE}2" 20 "${DOWN}2" msgtest 'Testing download of files with' 'completely downloaded file + partial file' -- cgit v1.2.3