From 4fa34122cbe347d21b3a162ff2fa75dd2e73c3a8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 20 Oct 2014 10:23:41 +0200 Subject: testcases: do not allow warnings in testsuccess Adds a new testwarning which tests for zero exit and the presents of a warning in the output, failing if either is not the case or if an error is found, too. This allows us to change testsuccess to accept only totally successful executions (= without warnings) which should help finding regressions. Git-Dch: Ignore --- test/integration/framework | 34 ++++++++++++++++++++-- test/integration/test-apt-by-hash-update | 4 +-- test/integration/test-apt-get-source-authenticated | 2 +- test/integration/test-apt-update-expected-size | 1 - test/integration/test-apt-update-ims | 15 ++++++---- test/integration/test-apt-update-nofallback | 8 ++--- test/integration/test-apt-update-rollback | 2 +- test/integration/test-apt-update-unauth | 2 +- .../test-bug-596498-trusted-unsigned-repo | 4 +-- ...17690-allow-unauthenticated-makes-all-untrusted | 2 +- test/integration/test-http-pipeline-messup | 2 +- test/integration/test-sourceslist-trusted-options | 12 ++++---- 12 files changed, 60 insertions(+), 28 deletions(-) diff --git a/test/integration/framework b/test/integration/framework index 617daa283..9ce300d55 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1205,12 +1205,43 @@ testsuccess() { msgtest 'Test for successful execution of' "$*" fi local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output" + if "$@" >${OUTPUT} 2>&1; then + if expr match "$1" '^apt.*' >/dev/null; then + if grep -q -E '^[WE]: ' "$OUTPUT"; then + echo >&2 + cat >&2 $OUTPUT + msgfail 'successful run, but output contains warnings/errors' + else + msgpass + fi + else + msgpass + fi + else + local EXITCODE=$? + echo >&2 + cat >&2 $OUTPUT + msgfail "exitcode $EXITCODE" + fi + aptautotest 'testsuccess' "$@" +} +testwarning() { + if [ "$1" = '--nomsg' ]; then + shift + else + msgtest 'Test for successful execution with warnings of' "$*" + fi + local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output" if "$@" >${OUTPUT} 2>&1; then if expr match "$1" '^apt.*' >/dev/null; then if grep -q -E '^E: ' "$OUTPUT"; then echo >&2 cat >&2 $OUTPUT msgfail 'successful run, but output contains errors' + elif ! grep -q -E '^W: ' "$OUTPUT"; then + echo >&2 + cat >&2 $OUTPUT + msgfail 'successful run, but output contains no warnings' else msgpass fi @@ -1223,9 +1254,8 @@ testsuccess() { cat >&2 $OUTPUT msgfail "exitcode $EXITCODE" fi - aptautotest 'testsuccess' "$@" + aptautotest 'testwarning' "$@" } - testfailure() { if [ "$1" = '--nomsg' ]; then shift diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 6e1ecdaff..d9d0b146f 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -34,7 +34,7 @@ Building dependency tree... E: Unable to locate package foo" aptget install -q -s foo # ensure we can apt-get update by hash -testsuccess aptget update -o APT::Acquire::By-Hash=1 +testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none # ensure it works testequal "Inst foo (1.0 unstable [all]) @@ -45,5 +45,5 @@ MAGIC="Acquire-By-Hash: true" sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release signreleasefiles # ... and verify that it fetches by hash now -testsuccess aptget update +testsuccess aptget update -o Acquire::Languages=none diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated index d833ddd85..fedfc8308 100755 --- a/test/integration/test-apt-get-source-authenticated +++ b/test/integration/test-apt-get-source-authenticated @@ -21,7 +21,7 @@ APTARCHIVE=$(readlink -f ./aptarchive) rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated InRelease file -testsuccess aptget update --allow-insecure-repositories +testwarning aptget update --allow-insecure-repositories # this all should fail testfailure aptget install -y foo diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 2acf56961..045217a77 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -42,4 +42,3 @@ rm -f rootdir/var/lib/apt/lists/localhost* testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Writing more data than expected ($NEW_SIZE > $SIZE) E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq - diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index eece0c84c..afae99563 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -12,9 +12,15 @@ setupaptarchive --no-update changetowebserver runtest() { + configallowinsecurerepositories "${1:-false}" + rm -f rootdir/var/lib/apt/lists/localhost* - testsuccess aptget update + if [ "$1" = 'true' ]; then + testwarning aptget update + else + testsuccess aptget update + fi # ensure no leftovers in partial testfailure ls "rootdir/var/lib/apt/lists/partial/*" @@ -62,10 +68,7 @@ runtest echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex runtest - # no Release.gpg or InRelease -configallowinsecurerepositories "true" - msgmsg "Release only" EXPECT="Ign http://localhost:8080 unstable InRelease 404 Not Found @@ -81,7 +84,7 @@ W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Pac find aptarchive -name 'Release.gpg' -delete echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex -runtest +runtest "true" echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex -runtest +runtest "true" diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback index 12977129f..831fc67eb 100755 --- a/test/integration/test-apt-update-nofallback +++ b/test/integration/test-apt-update-nofallback @@ -54,8 +54,7 @@ assert_repo_is_intact() setupaptarchive_with_lists_clean() { setupaptarchive --no-update - rm -f rootdir/var/lib/apt/lists/_* - #rm -rf rootdir/var/lib/apt/lists + rm -rf rootdir/var/lib/apt/lists } test_from_inrelease_to_unsigned() @@ -87,14 +86,15 @@ test_from_inrelease_to_unsigned_with_override() { # setup archive with InRelease file setupaptarchive_with_lists_clean - testsuccess aptget update + # FIXME: is not what the server reported 4104 4106 + testsuccess aptget update #-o Debug::pkgAcquire::Worker=1 # simulate moving to a unsigned but otherwise valid repo simulate_mitm_and_inject_evil_package generatereleasefiles # and ensure we can update to it (with enough force) - testsuccess aptget update --allow-insecure-repositories \ + testwarning aptget update --allow-insecure-repositories \ -o Acquire::AllowDowngradeToInsecureRepositories=1 # but that the individual packages are still considered untrusted testequal "WARNING: The following packages cannot be authenticated! diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback index d33411da4..9efc194a0 100755 --- a/test/integration/test-apt-update-rollback +++ b/test/integration/test-apt-update-rollback @@ -117,7 +117,7 @@ test_unauthenticated_to_invalid_inrelease() { rm $APTARCHIVE/dists/unstable/InRelease rm $APTARCHIVE/dists/unstable/Release.gpg - testsuccess aptget update --allow-insecure-repositories + testwarning aptget update --allow-insecure-repositories listcurrentlistsdirectory > lists.before testequal "WARNING: The following packages cannot be authenticated! old diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index 1b5dbc6c4..7347f7d10 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -27,7 +27,7 @@ runtest() { rm -rf rootdir/var/lib/apt/lists/ find aptarchive/ -name '*Release*' -delete - testsuccess aptget update --allow-insecure-repositories + testwarning aptget update --allow-insecure-repositories # become authenticated generatereleasefiles diff --git a/test/integration/test-bug-596498-trusted-unsigned-repo b/test/integration/test-bug-596498-trusted-unsigned-repo index a08c153f8..1e5e75b0e 100755 --- a/test/integration/test-bug-596498-trusted-unsigned-repo +++ b/test/integration/test-bug-596498-trusted-unsigned-repo @@ -12,7 +12,7 @@ setupaptarchive aptgetupdate() { rm -rf rootdir/var/lib/apt/ rootdir/var/cache/apt/*.bin - testsuccess aptget update --allow-insecure-repositories + ${1:-testwarning} aptget update --allow-insecure-repositories } PKGTEXT="$(aptget install cool --assume-no -d | head -n 7)" @@ -25,7 +25,7 @@ testequal "$PKGTEXT Download complete and in download only mode" aptget install cool --assume-no -d --allow-unauthenticated sed -i -e 's#deb#deb [trusted=no]#' $DEBFILE -aptgetupdate +aptgetupdate 'testsuccess' testequal "$PKGTEXT WARNING: The following packages cannot be authenticated! diff --git a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted index 0736bb6dc..164dca070 100755 --- a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted +++ b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted @@ -34,7 +34,7 @@ testrun() { testsuccess aptget download cool --allow-unauthenticated testfileexists 'cool_1.0_i386.deb' else - testsuccess aptget update --allow-insecure-repositories + testwarning aptget update --allow-insecure-repositories testfailure aptget download cool testfilemissing 'cool_1.0_i386.deb' diff --git a/test/integration/test-http-pipeline-messup b/test/integration/test-http-pipeline-messup index 405574e8a..dda8ef7eb 100755 --- a/test/integration/test-http-pipeline-messup +++ b/test/integration/test-http-pipeline-messup @@ -35,7 +35,7 @@ testfailure test -f pkga_1.0_all.deb echo 'Acquire::http::Pipeline-Depth 10;' > rootdir/etc/apt/apt.conf.d/99enable-pipeline # the output is a bit strange: it looks like it has downloaded pkga 4 times -testsuccess aptget download pkga pkgb pkgc pkgd +testwarning aptget download pkga pkgb pkgc pkgd for pkg in 'pkga' 'pkgb' 'pkgc' 'pkgd'; do testsuccess test -f ${pkg}_1.0_all.deb testsuccess cmp incoming/${pkg}_1.0_all.deb ${pkg}_1.0_all.deb diff --git a/test/integration/test-sourceslist-trusted-options b/test/integration/test-sourceslist-trusted-options index c954f2f4f..28415dd62 100755 --- a/test/integration/test-sourceslist-trusted-options +++ b/test/integration/test-sourceslist-trusted-options @@ -57,14 +57,14 @@ aptgetupdate() { # note that insecure with trusted=yes are allowed # as the trusted=yes indicates that security is provided by # something above the understanding of apt - testsuccess aptget update --no-allow-insecure-repositories + ${1:-testsuccess} aptget update --no-allow-insecure-repositories } insecureaptgetupdate() { rm -rf rootdir/var/lib/apt/lists testfailure aptget update --no-allow-insecure-repositories rm -rf rootdir/var/lib/apt/lists - testsuccess aptget update --allow-insecure-repositories + testwarning aptget update --allow-insecure-repositories } msgmsg 'Test without trusted option and good sources' @@ -103,7 +103,7 @@ everythingsucceeds -t testing msgmsg 'Test with trusted=yes option and good and unsigned sources' cp -a rootdir/etc/apt/sources.list.d.bak/* rootdir/etc/apt/sources.list.d/ sed -i 's#^deb\(-src\)\? #deb\1 [trusted=yes] #' rootdir/etc/apt/sources.list.d/* -aptgetupdate +aptgetupdate 'testwarning' everythingsucceeds everythingsucceeds -t stable everythingsucceeds -t testing @@ -128,7 +128,7 @@ everythingsucceeds -t testing msgmsg 'Test with trusted=yes option and good and unknown sources' cp -a rootdir/etc/apt/sources.list.d.bak/* rootdir/etc/apt/sources.list.d/ sed -i 's#^deb\(-src\)\? #deb\1 [trusted=yes] #' rootdir/etc/apt/sources.list.d/* -aptgetupdate +aptgetupdate 'testwarning' everythingsucceeds everythingsucceeds -t stable everythingsucceeds -t testing @@ -154,7 +154,7 @@ everythingsucceeds -t testing msgmsg 'Test with trusted=yes option and good and expired sources' cp -a rootdir/etc/apt/sources.list.d.bak/* rootdir/etc/apt/sources.list.d/ sed -i 's#^deb\(-src\)\? #deb\1 [trusted=yes] #' rootdir/etc/apt/sources.list.d/* -aptgetupdate +aptgetupdate 'testwarning' everythingsucceeds everythingsucceeds -t stable everythingsucceeds -t testing @@ -181,7 +181,7 @@ everythingfails -t testing msgmsg 'Test with trusted=yes option and unsigned and good sources' cp -a rootdir/etc/apt/sources.list.d.bak/* rootdir/etc/apt/sources.list.d/ sed -i 's#^deb\(-src\)\? #deb\1 [trusted=yes] #' rootdir/etc/apt/sources.list.d/* -aptgetupdate +aptgetupdate 'testwarning' everythingsucceeds everythingsucceeds -t stable everythingsucceeds -t testing -- cgit v1.2.3