From c48eea97b93920062ea26001081d4fdf7eb967e3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 17:47:30 +0200 Subject: make expected-size a maximum-size check as this is what we want at this point --- test/integration/test-apt-update-expected-size | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 72812336d..c1eecc08a 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -15,6 +15,13 @@ changetowebserver # normal update works fine testsuccess aptget update +# make InRelease really big +mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good +dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 +touch -d '+1hour' aptarchive/dists/unstable/InRelease +aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) + + # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz -- cgit v1.2.3 From 27e6c17a18216e2a02de39a6d1722b83ac823d42 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 20:40:37 +0200 Subject: Add new Acquire::MaxReleaseFileSize=10*1000*1000 option This option controls the maximum size of Release/Release.gpg/InRelease files. The rational is that we do not know the size of these files in advance and we want to protect against a denial of service attack where someone sends us endless amounts of data until the disk is full (we do know the size all other files (Packages/Sources/debs)). --- test/integration/test-apt-update-expected-size | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index c1eecc08a..f8ec24dcc 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -17,10 +17,17 @@ testsuccess aptget update # make InRelease really big mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good -dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 +dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null touch -d '+1hour' aptarchive/dists/unstable/InRelease -aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) - +aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 > output.log +msgtest 'Check that the max write warning is triggered' +if grep -q "Writing more data than expected" output.log; then + msgpass +else + cat output.log + msgfail +fi +mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" -- cgit v1.2.3 From ee27950632c149bb14c9c490e92147579ba4fc2a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 22:36:09 +0200 Subject: Send "Fail-Reason: MaximumSizeExceeded" from the method Communicate the fail reason from the methods to the parent and Rename() failed files. --- test/integration/test-apt-update-expected-size | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index f8ec24dcc..58920f544 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -19,7 +19,7 @@ testsuccess aptget update mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null touch -d '+1hour' aptarchive/dists/unstable/InRelease -aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 > output.log +aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 > output.log msgtest 'Check that the max write warning is triggered' if grep -q "Writing more data than expected" output.log; then msgpass @@ -27,8 +27,11 @@ else cat output.log msgfail fi +# ensure the failed InRelease file got renamed +testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease + # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz -- cgit v1.2.3 From 9d653a6de2ca952247cc4e628256259d225570a6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 13 Oct 2014 09:54:21 +0200 Subject: fix compile and tests error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I am pretty sure I did that before committing broken stuff… Git-Dch: Ignore --- test/integration/test-apt-update-expected-size | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 58920f544..a039e9e1c 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -38,7 +38,7 @@ echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" rm -f rootdir/var/lib/apt/lists/localhost* -testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) [IP: ::1 8080] +testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq -- cgit v1.2.3 From 846bc058cb0c1bf7ce7c2fb30b9c277e96e9eaf7 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Oct 2014 22:46:48 +0200 Subject: check lists/ content in tests doing rollback Git-Dch: Ignore --- test/integration/test-apt-update-expected-size | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index a039e9e1c..b71853406 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -34,8 +34,9 @@ mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" -echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz -echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages +find aptarchive -name 'Packages*' | while read pkg; do + echo "1234567890" >> "$pkg" +done NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" rm -f rootdir/var/lib/apt/lists/localhost* testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) -- cgit v1.2.3 From 1df24acfdb8ba1cd8bbbaa166f170dda480ce41e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 19 Oct 2014 14:14:37 +0200 Subject: check for failure message in testsuccess/failure These functions check the exit code of the command, but for apt commands we can go further and require an error message for non-zero exits and none for zero exits. Git-Dch: Ignore --- test/integration/test-apt-update-expected-size | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index b71853406..2acf56961 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -15,16 +15,16 @@ changetowebserver # normal update works fine testsuccess aptget update -# make InRelease really big +# make InRelease really big to trigger fallback mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null touch -d '+1hour' aptarchive/dists/unstable/InRelease -aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 > output.log +testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 msgtest 'Check that the max write warning is triggered' -if grep -q "Writing more data than expected" output.log; then +if grep -q "Writing more data than expected" rootdir/tmp/testsuccess.output; then msgpass else - cat output.log + cat rootdir/tmp/testsuccess.output msgfail fi # ensure the failed InRelease file got renamed -- cgit v1.2.3 From 4fa34122cbe347d21b3a162ff2fa75dd2e73c3a8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 20 Oct 2014 10:23:41 +0200 Subject: testcases: do not allow warnings in testsuccess Adds a new testwarning which tests for zero exit and the presents of a warning in the output, failing if either is not the case or if an error is found, too. This allows us to change testsuccess to accept only totally successful executions (= without warnings) which should help finding regressions. Git-Dch: Ignore --- test/integration/test-apt-update-expected-size | 1 - 1 file changed, 1 deletion(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 2acf56961..045217a77 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -42,4 +42,3 @@ rm -f rootdir/var/lib/apt/lists/localhost* testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq - -- cgit v1.2.3 From 25b86db159fbc3c043628e285c0c1ef24dec2c6e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Mar 2015 00:59:44 +0100 Subject: test exitcode as well as string equality We use test{success,failure} now all over the place in the framework, so its only consequencial to do this in the situations in which we test for a specific output as well. Git-Dch: Ignore --- test/integration/test-apt-update-expected-size | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 045217a77..9711c293a 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -39,6 +39,6 @@ find aptarchive -name 'Packages*' | while read pkg; do done NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" rm -f rootdir/var/lib/apt/lists/localhost* -testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) +testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq -- cgit v1.2.3 From 27925d82dd0cbae74d48040363fe6f6c2bae5215 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Mar 2015 15:53:43 +0100 Subject: improve https method queue progress reporting The worker expects that the methods tell him when they start or finish downloading a file. Various information pieces are passed along in this report including the (expected) filesize. https was using a "global" struct for reporting which made it 'reuse' incorrect values in some cases like a non-existent InRelease fallbacking to Release{,.gpg} resulting in a size-mismatch warning. Reducing the scope and redesigning the setting of the values we can fix this and related issues. Closes: 777565, 781509 Thanks: Robert Edmonds and Anders Kaseorg for initial patchs --- test/integration/test-apt-update-expected-size | 86 +++++++++++++++++--------- 1 file changed, 56 insertions(+), 30 deletions(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 9711c293a..22de13ea5 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -10,35 +10,61 @@ configarchitecture "i386" insertpackage 'unstable' 'apt' 'all' '1.0' setupaptarchive --no-update +cp -a aptarchive/dists aptarchive/dists.good + +test_inreleasetoobig() { + # make InRelease really big to trigger fallback + dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null + touch -d '+1hour' aptarchive/dists/unstable/InRelease + testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 + msgtest 'Check that the max write warning is triggered' + cp rootdir/tmp/testsuccess.output update.output + testsuccess --nomsg grep -q 'Writing more data than expected' update.output + rm -f update.output + # ensure the failed InRelease file got renamed + testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED +} + +test_packagestoobig() { + # append junk at the end of the Packages.gz/Packages + SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" + find aptarchive/dists -name 'Packages*' | while read pkg; do + echo "1234567890" >> "$pkg" + touch -d '+1hour' "$pkg" + done + NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" + testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0 +} + +methodtest() { + msgmsg 'Test with' "$1" 'and clean start' + rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good + # normal update works fine + testsuccess aptget update + mv rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good + + # starting fresh works + test_inreleasetoobig "$1" + rm -rf aptarchive/dists rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists + test_packagestoobig "$1" + rm -rf aptarchive/dists rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists + + msgmsg 'Test with' "$1" 'and existing old data' + cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists + test_inreleasetoobig "$1" + rm -rf aptarchive/dists rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists + cp -a aptarchive/dists.good aptarchive/dists + test_packagestoobig "$1" + rm -rf aptarchive/dists + cp -a aptarchive/dists.good aptarchive/dists +} + changetowebserver +methodtest 'http://localhost:8080' -# normal update works fine -testsuccess aptget update - -# make InRelease really big to trigger fallback -mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good -dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null -touch -d '+1hour' aptarchive/dists/unstable/InRelease -testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 -msgtest 'Check that the max write warning is triggered' -if grep -q "Writing more data than expected" rootdir/tmp/testsuccess.output; then - msgpass -else - cat rootdir/tmp/testsuccess.output - msgfail -fi -# ensure the failed InRelease file got renamed -testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED -mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease - - -# append junk at the end of the Packages.gz/Packages -SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" -find aptarchive -name 'Packages*' | while read pkg; do - echo "1234567890" >> "$pkg" -done -NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" -rm -f rootdir/var/lib/apt/lists/localhost* -testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) - -E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq +changetohttpswebserver +methodtest 'https://localhost:4433' -- cgit v1.2.3 From ba6b79bd0090077724fa1272ea4d3a31706fcd5a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 12 Apr 2015 17:08:46 +0200 Subject: a hit on Release files means the indexes will be hits too If we get a IMSHit for the Transaction-Manager (= the InRelease file or as its still supported fallback Release + Release.gpg combo) we can assume that every file we would queue based on this manager, but already have locally is current and hence would get an IMSHit, too. We therefore save us and the server the trouble and skip the queuing in this case. Beside speeding up repetative executions of 'apt-get update' this way we also avoid hitting hashsum errors if the indexes are in fact already updated, but the Release file isn't yet as it is the case on well behaving mirrors as Release files is updated last. The implementation is a bit harder than the theory makes it sound as we still have to keep reverifying the Release files (e.g. to detect now expired once to avoid an attacker being able to silently stale us) and have to handle cases in which the Release file hits, but some indexes aren't present (e.g. user added a new foreign architecture). --- test/integration/test-apt-update-expected-size | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 22de13ea5..9f58308aa 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -38,6 +38,10 @@ E: Some index files failed to download. They have been ignored, or old ones used } methodtest() { + # less complicated test setup this way + webserverconfig 'aptwebserver::support::modified-since' 'false' "$1" + webserverconfig 'aptwebserver::support::last-modified' 'false' "$1" # curl is clever and sees hits here also + msgmsg 'Test with' "$1" 'and clean start' rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good # normal update works fine -- cgit v1.2.3 From 146f7715a9f36d246b461255b3c683b479296915 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Apr 2015 10:59:27 +0200 Subject: improve partial/ cleanup in abort and failure cases Especially pdiff-enhanced downloads have the tendency to fail for various reasons from which we can recover and even a successful download used to leave the old unpatched index in partial/. By adding a new method responsible for making the transaction of an individual file happen we can at specialisations especially for abort cases to deal with the cleanup. This also helps in keeping the compressed indexes around if another index failed instead of keeping the decompressed files, which we wouldn't pick up in the next call. --- test/integration/test-apt-update-expected-size | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 9f58308aa..7efccaa57 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -34,7 +34,7 @@ test_packagestoobig() { done NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) -E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0 +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::Transaction=0 } methodtest() { -- cgit v1.2.3 From 8eafc759544298211cd0bfaa3919afc0fadd47d1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 13 May 2015 16:09:12 +0200 Subject: detect Releasefile IMS hits even if the server doesn't Not all servers we are talking to support If-Modified-Since and some are not even sending Last-Modified for us, so in an effort to detect such hits we run a hashsum check on the 'old' compared to the 'new' file, we got the hashes for the 'new' already for "free" from the methods anyway and hence just need to calculated the old ones. This allows us to detect hits even with unsupported servers, which in turn means we benefit from all the new hit behavior also here. --- test/integration/test-apt-update-expected-size | 1 + 1 file changed, 1 insertion(+) (limited to 'test/integration/test-apt-update-expected-size') diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 7efccaa57..55a5da848 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -26,6 +26,7 @@ test_inreleasetoobig() { } test_packagestoobig() { + redatereleasefiles '+1hour' # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" find aptarchive/dists -name 'Packages*' | while read pkg; do -- cgit v1.2.3