From 27925d82dd0cbae74d48040363fe6f6c2bae5215 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Mar 2015 15:53:43 +0100 Subject: improve https method queue progress reporting The worker expects that the methods tell him when they start or finish downloading a file. Various information pieces are passed along in this report including the (expected) filesize. https was using a "global" struct for reporting which made it 'reuse' incorrect values in some cases like a non-existent InRelease fallbacking to Release{,.gpg} resulting in a size-mismatch warning. Reducing the scope and redesigning the setting of the values we can fix this and related issues. Closes: 777565, 781509 Thanks: Robert Edmonds and Anders Kaseorg for initial patchs --- test/integration/framework | 20 +++++- test/integration/test-apt-https-no-redirect | 15 ++-- test/integration/test-apt-update-expected-size | 86 +++++++++++++++-------- test/integration/test-bug-602412-dequote-redirect | 3 +- 4 files changed, 80 insertions(+), 44 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index ec23e41e6..50c027a2c 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1125,8 +1125,10 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom downloadfile() { local PROTO="${1%%:*}" - apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \ - download-file "$1" "$2" 2>&1 || true + if ! apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \ + download-file "$1" "$2" 2>&1 ; then + return 1 + fi # only if the file exists the download was successful if [ -r "$2" ]; then return 0 @@ -1407,6 +1409,20 @@ testfailureequal() { testfileequal "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" "$CMP" } +testfailuremsg() { + local CMP="$1" + shift + testfailure "$@" + msgtest 'Check that the output of the previous failed command has expected' 'failures and warnings' + grep '^\(W\|E\):' "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" > "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailureequal.output" 2>&1 || true + if echo "$CMP" | checkdiff - "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailureequal.output"; then + msgpass + else + echo '### Complete output ###' + cat "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" + msgfail + fi +} testfilestats() { msgtest "Test that file $1 has $2 $3" "$4" diff --git a/test/integration/test-apt-https-no-redirect b/test/integration/test-apt-https-no-redirect index bc744d6f2..c91c78916 100755 --- a/test/integration/test-apt-https-no-redirect +++ b/test/integration/test-apt-https-no-redirect @@ -14,22 +14,15 @@ echo 'alright' > aptarchive/working changetohttpswebserver -o 'aptwebserver::redirect::replace::/redirectme/=http://localhost:8080/' msgtest 'download of a file works via' 'http' -downloadfile 'http://localhost:8080/working' httpfile >/dev/null 2>&1 && msgpass || msgfail +testsuccess --nomsg downloadfile 'http://localhost:8080/working' httpfile testfileequal httpfile 'alright' msgtest 'download of a file works via' 'https' -downloadfile 'https://localhost:4433/working' httpsfile >/dev/null 2>&1 && msgpass || msgfail +testsuccess --nomsg downloadfile 'https://localhost:4433/working' httpsfile testfileequal httpsfile 'alright' msgtest 'download of a file does not work if' 'https redirected to http' -downloadfile 'https://localhost:4433/redirectme/working' redirectfile >curloutput 2>&1 && msgfail || msgpass +testfailure --nomsg downloadfile 'https://localhost:4433/redirectme/working' redirectfile msgtest 'libcurl has forbidden access in last request to' 'http resource' -if grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' curloutput; then - msgpass -else - cat curloutput - msgfail -fi - - +testsuccess --nomsg grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' rootdir/tmp/testfailure.output diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 9711c293a..22de13ea5 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -10,35 +10,61 @@ configarchitecture "i386" insertpackage 'unstable' 'apt' 'all' '1.0' setupaptarchive --no-update +cp -a aptarchive/dists aptarchive/dists.good + +test_inreleasetoobig() { + # make InRelease really big to trigger fallback + dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null + touch -d '+1hour' aptarchive/dists/unstable/InRelease + testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 + msgtest 'Check that the max write warning is triggered' + cp rootdir/tmp/testsuccess.output update.output + testsuccess --nomsg grep -q 'Writing more data than expected' update.output + rm -f update.output + # ensure the failed InRelease file got renamed + testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED +} + +test_packagestoobig() { + # append junk at the end of the Packages.gz/Packages + SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" + find aptarchive/dists -name 'Packages*' | while read pkg; do + echo "1234567890" >> "$pkg" + touch -d '+1hour' "$pkg" + done + NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" + testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0 +} + +methodtest() { + msgmsg 'Test with' "$1" 'and clean start' + rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good + # normal update works fine + testsuccess aptget update + mv rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good + + # starting fresh works + test_inreleasetoobig "$1" + rm -rf aptarchive/dists rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists + test_packagestoobig "$1" + rm -rf aptarchive/dists rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists + + msgmsg 'Test with' "$1" 'and existing old data' + cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists + test_inreleasetoobig "$1" + rm -rf aptarchive/dists rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists + test_packagestoobig "$1" + rm -rf aptarchive/dists + cp -a aptarchive/dists.good aptarchive/dists +} + changetowebserver +methodtest 'http://localhost:8080' -# normal update works fine -testsuccess aptget update - -# make InRelease really big to trigger fallback -mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good -dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null -touch -d '+1hour' aptarchive/dists/unstable/InRelease -testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 -msgtest 'Check that the max write warning is triggered' -if grep -q "Writing more data than expected" rootdir/tmp/testsuccess.output; then - msgpass -else - cat rootdir/tmp/testsuccess.output - msgfail -fi -# ensure the failed InRelease file got renamed -testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED -mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease - - -# append junk at the end of the Packages.gz/Packages -SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" -find aptarchive -name 'Packages*' | while read pkg; do - echo "1234567890" >> "$pkg" -done -NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" -rm -f rootdir/var/lib/apt/lists/localhost* -testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) - -E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq +changetohttpswebserver +methodtest 'https://localhost:4433' diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect index d3573a79a..384c8b113 100755 --- a/test/integration/test-bug-602412-dequote-redirect +++ b/test/integration/test-bug-602412-dequote-redirect @@ -8,7 +8,7 @@ configarchitecture 'amd64' buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable' -setupaptarchive +setupaptarchive --no-update changetowebserver -o aptwebserver::redirect::replace::/pool/=/newpool/ \ -o aptwebserver::redirect::replace::/dists/=/newdists/ @@ -16,6 +16,7 @@ mv aptarchive/pool aptarchive/newpool mv aptarchive/dists aptarchive/newdists testrun() { + msgmsg 'Test redirection works in method boundaries' "$1" msgtest 'Test redirection works in' 'apt-get update' testsuccess --nomsg aptget update -- cgit v1.2.3