summaryrefslogtreecommitdiff
path: root/test
diff options
context:
space:
mode:
authorDavid Kalnischkies <david@kalnischkies.de>2014-04-28 10:02:27 +0200
committerDavid Kalnischkies <david@kalnischkies.de>2014-05-09 13:06:27 +0200
commit895417ef99bb1371d8970da1afe87c6d64382f67 (patch)
tree4f0e940d5b9d907f110c17d9649292f495557e72 /test
parentd003a557a516e3063de3190950e911c61c3dd53e (diff)
reenable pipelining via hashsum reordering support
Now that methods have the expected hashes available they can check if the response from the server is what they expected. Pipelining is one of those areas in which servers can mess up by not supporting it properly, which forced us to disable it for the time being. Now, we check if we got a response out of order, which we can not only use to disable pipelining automatically for the next requests, but we can fix it up just like the server responded in proper order for the current requests. To ensure that this little trick works pipelining is only attempt if we have hashsums for all the files in the chain which in theory reduces the use of pipelining usage even on the many servers which work properly, but in practice only the InRelease file (or similar such) will be requested without a hashsum – and as it is the only file requested in that stage it can't be pipelined even if we wanted to. Some minor annoyances remain: The display of the progress we have doesn't reflect this change, so it looks like the same package gets downloaded multiple times while others aren't at all. Further more, partial files are not supported in this recovery as the received data was appended to the wrong file, so the hashsum doesn't match. Both seem to be minor enough to reenable pipelining by default until further notice through to test if it really solves the problem. This therefore reverts commit 8221431757c775ee875a061b184b5f6f2330f928.
Diffstat (limited to 'test')
-rwxr-xr-xtest/integration/test-http-pipeline-messup43
-rwxr-xr-xtest/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum116
2 files changed, 102 insertions, 57 deletions
diff --git a/test/integration/test-http-pipeline-messup b/test/integration/test-http-pipeline-messup
new file mode 100755
index 000000000..9c59e1825
--- /dev/null
+++ b/test/integration/test-http-pipeline-messup
@@ -0,0 +1,43 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+buildsimplenativepackage 'pkga' 'all' '1.0' 'stable'
+buildsimplenativepackage 'pkgb' 'all' '1.0' 'stable'
+buildsimplenativepackage 'pkgc' 'all' '1.0' 'stable'
+buildsimplenativepackage 'pkgd' 'all' '1.0' 'stable'
+
+setupaptarchive --no-update
+
+# simulate (and be a predictable) pipeline mess-up by the server/proxy
+changetowebserver \
+ -o 'aptwebserver::overwrite::.*pkga.*::filename=/pool/pkgd_1.0_all.deb' \
+ -o 'aptwebserver::overwrite::.*pkgc.*::filename=/pool/pkgb_1.0_all.deb' \
+ -o 'aptwebserver::overwrite::.*pkgb.*::filename=/pool/pkgc_1.0_all.deb' \
+ -o 'aptwebserver::overwrite::.*pkgd.*::filename=/pool/pkga_1.0_all.deb'
+
+testsuccess aptget update -o Debug::Acquire::http=1 -o Debug::pkgAcquire::Worker=1
+
+# messup is bigger than pipeline: checks if fixup isn't trying to hard
+testfailure aptget download pkga pkgb pkgc pkgd "$@" -o Acquire::http::Pipeline-Depth=2
+testfailure test -f pkga_1.0_all.deb
+
+# ensure that pipeling is enabled for rest of this test
+echo 'Acquire::http::Pipeline-Depth 10;' > rootdir/etc/apt/apt.conf.d/99enable-pipeline
+
+# the output is a bit strange: it looks like it has downloaded pkga 4 times
+testsuccess aptget download pkga pkgb pkgc pkgd -o Debug::Acquire::http=1 -o Debug::pkgAcquire::Worker=1
+for pkg in 'pkga' 'pkgb' 'pkgc' 'pkgd'; do
+ testsuccess test -f ${pkg}_1.0_all.deb
+ testsuccess cmp incoming/${pkg}_1.0_all.deb ${pkg}_1.0_all.deb
+ rm -f ${pkg}_1.0_all.deb
+done
+
+# while hashes will pass (as none are available), sizes will not match, so failure
+# checks that no hashes means that pipeline depth is ignored as we can't fixup
+testfailure aptget download pkga pkgb pkgc pkgd "$@" --allow-unauthenticated -o Acquire::ForceHash=ROT26
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index 9bdc81264..8c9c9c767 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -14,8 +14,8 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Files:
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.tar.gz
+ 9604ba9427a280db542279d9ed78400b 3 pkg-md5-ok_1.0.dsc
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-md5-ok_1.0.tar.gz
Package: pkg-sha256-ok
Binary: pkg-sha256-ok
@@ -23,14 +23,14 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Files:
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.tar.gz
+ 9604ba9427a280db542279d9ed78400b 3 pkg-sha256-ok_1.0.dsc
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-sha256-ok_1.0.tar.gz
Checksums-Sha1:
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.dsc
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.tar.gz
+ 324f464e6151a92cf57b26ef95dcfcf2059a8c44 3 pkg-sha256-ok_1.0.dsc
+ 680254bad1d7ca0d65ec46aaa315d363abf6a50a 3 pkg-sha256-ok_1.0.tar.gz
Checksums-Sha256:
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.dsc
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.tar.gz
+ 943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a 3 pkg-sha256-ok_1.0.dsc
+ 90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb 3 pkg-sha256-ok_1.0.tar.gz
Package: pkg-sha256-bad
Binary: pkg-sha256-bad
@@ -38,14 +38,14 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Files:
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.tar.gz
+ 9604ba9427a280db542279d9ed78400b 3 pkg-sha256-bad_1.0.dsc
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-sha256-bad_1.0.tar.gz
Checksums-Sha1:
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.dsc
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.tar.gz
+ 324f464e6151a92cf57b26ef95dcfcf2059a8c44 3 pkg-sha256-bad_1.0.dsc
+ 680254bad1d7ca0d65ec46aaa315d363abf6a50a 3 pkg-sha256-bad_1.0.tar.gz
Checksums-Sha256:
- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-bad_1.0.dsc
- bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-bad_1.0.tar.gz
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 3 pkg-sha256-bad_1.0.dsc
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 3 pkg-sha256-bad_1.0.tar.gz
Package: pkg-no-md5
Binary: pkg-no-md5
@@ -53,11 +53,11 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Checksums-Sha1:
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.dsc
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.tar.gz
+ 324f464e6151a92cf57b26ef95dcfcf2059a8c44 3 pkg-no-md5_1.0.dsc
+ 680254bad1d7ca0d65ec46aaa315d363abf6a50a 3 pkg-no-md5_1.0.tar.gz
Checksums-Sha256:
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.dsc
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.tar.gz
+ 943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a 3 pkg-no-md5_1.0.dsc
+ 90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb 3 pkg-no-md5_1.0.tar.gz
Package: pkg-mixed-ok
Binary: pkg-mixed-ok
@@ -65,9 +65,9 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Checksums-Sha1:
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-ok_1.0.tar.gz
+ 680254bad1d7ca0d65ec46aaa315d363abf6a50a 3 pkg-mixed-ok_1.0.tar.gz
Checksums-Sha256:
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-ok_1.0.dsc
+ 943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a 3 pkg-mixed-ok_1.0.dsc
Package: pkg-mixed-sha1-bad
Binary: pkg-mixed-sha1-bad
@@ -75,9 +75,9 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Checksums-Sha1:
- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-mixed-sha1-bad_1.0.dsc
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 3 pkg-mixed-sha1-bad_1.0.dsc
Checksums-Sha256:
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-sha1-bad_1.0.tar.gz
+ 90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb 3 pkg-mixed-sha1-bad_1.0.tar.gz
Package: pkg-mixed-sha2-bad
Binary: pkg-mixed-sha2-bad
@@ -85,9 +85,9 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Checksums-Sha1:
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-sha2-bad_1.0.dsc
+ 324f464e6151a92cf57b26ef95dcfcf2059a8c44 3 pkg-mixed-sha2-bad_1.0.dsc
Checksums-Sha256:
- bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-mixed-sha2-bad_1.0.tar.gz
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 3 pkg-mixed-sha2-bad_1.0.tar.gz
Package: pkg-md5-disagree
Binary: pkg-md5-disagree
@@ -95,10 +95,10 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Files:
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.tar.gz
- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-md5-disagree_1.0.dsc
- bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-md5-disagree_1.0.tar.gz
+ 9604ba9427a280db542279d9ed78400b 3 pkg-md5-disagree_1.0.dsc
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-md5-disagree_1.0.tar.gz
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 3 pkg-md5-disagree_1.0.dsc
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 3 pkg-md5-disagree_1.0.tar.gz
Package: pkg-md5-agree
Binary: pkg-md5-agree
@@ -106,10 +106,10 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Files:
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc
+ 9604ba9427a280db542279d9ed78400b 3 pkg-md5-agree_1.0.dsc
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-md5-agree_1.0.tar.gz
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-md5-agree_1.0.tar.gz
+ 9604ba9427a280db542279d9ed78400b 3 pkg-md5-agree_1.0.dsc
Package: pkg-sha256-disagree
Binary: pkg-sha256-disagree
@@ -117,23 +117,24 @@ Version: 1.0
Maintainer: Joe Sixpack <joe@example.org>
Architecture: all
Files:
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.tar.gz
+ 9604ba9427a280db542279d9ed78400b 3 pkg-sha256-disagree_1.0.dsc
+ db5570bf61464b46e2bde31ed61a7dc6 3 pkg-sha256-disagree_1.0.tar.gz
Checksums-Sha1:
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.dsc
- da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.tar.gz
+ 324f464e6151a92cf57b26ef95dcfcf2059a8c44 3 pkg-sha256-disagree_1.0.dsc
+ 680254bad1d7ca0d65ec46aaa315d363abf6a50a 3 pkg-sha256-disagree_1.0.tar.gz
Checksums-Sha256:
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.dsc
- e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.tar.gz
- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-disagree_1.0.dsc
- bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-disagree_1.0.tar.gz
+ 943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a 3 pkg-sha256-disagree_1.0.dsc
+ 90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb 3 pkg-sha256-disagree_1.0.tar.gz
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 3 pkg-sha256-disagree_1.0.dsc
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 3 pkg-sha256-disagree_1.0.tar.gz
EOF
# create fetchable files
for x in 'pkg-md5-ok' 'pkg-sha256-ok' 'pkg-sha256-bad' 'pkg-no-md5' \
'pkg-mixed-ok' 'pkg-mixed-sha1-bad' 'pkg-mixed-sha2-bad' \
'pkg-md5-agree' 'pkg-md5-disagree' 'pkg-sha256-disagree'; do
- touch aptarchive/${x}_1.0.dsc aptarchive/${x}_1.0.tar.gz
+ echo -n 'dsc' > aptarchive/${x}_1.0.dsc
+ echo -n 'tar' > aptarchive/${x}_1.0.tar.gz
done
setupaptarchive
@@ -144,9 +145,9 @@ testok() {
rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
testequal "Reading package lists...
Building dependency tree...
-Need to get 0 B of source archives.
-Get:1 http://localhost:8080/ $1 1.0 (dsc)
-Get:2 http://localhost:8080/ $1 1.0 (tar)
+Need to get 6 B of source archives.
+Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B]
+Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B]
Download complete and in download only mode" aptget source -d "$@"
msgtest 'Files were successfully downloaded for' "$1"
testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz
@@ -154,7 +155,8 @@ Download complete and in download only mode" aptget source -d "$@"
}
testkeep() {
- touch ${1}_1.0.dsc ${1}_1.0.tar.gz
+ echo -n 'dsc' > ${1}_1.0.dsc
+ echo -n 'tar' > ${1}_1.0.tar.gz
testequal "Reading package lists...
Building dependency tree...
Skipping already downloaded file '${1}_1.0.dsc'
@@ -170,9 +172,9 @@ testmismatch() {
rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
testequal "Reading package lists...
Building dependency tree...
-Need to get 0 B of source archives.
-Get:1 http://localhost:8080/ $1 1.0 (dsc)
-Get:2 http://localhost:8080/ $1 1.0 (tar)
+Need to get 6 B of source archives.
+Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B]
+Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B]
E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch
E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz Hash Sum mismatch
@@ -194,9 +196,9 @@ Download complete and in download only mode" aptget source -d "$@" -o Acquire::F
rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
testequal "Reading package lists...
Building dependency tree...
-Need to get 0 B of source archives.
-Get:1 http://localhost:8080/ $1 1.0 (dsc)
-Get:2 http://localhost:8080/ $1 1.0 (tar)
+Need to get 6 B of source archives.
+Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B]
+Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B]
Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26
msgtest 'Files were downloaded unauthenticated as user allowed it' "$1"
testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz
@@ -231,9 +233,9 @@ testfailure --nomsg test -e pkg-no-md5_1.0.dsc -a -e pkg-no-md5_1.0.tar.gz
testok pkg-mixed-ok
testequal 'Reading package lists...
Building dependency tree...
-Need to get 0 B of source archives.
-Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar)
-Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc)
+Need to get 6 B of source archives.
+Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) [3 B]
+Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) [3 B]
E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch
E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad
@@ -241,9 +243,9 @@ msgtest 'Only tar file is downloaded as the dsc has hashsum mismatch' 'pkg-mixed
testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-bad_1.0.tar.gz
testequal 'Reading package lists...
Building dependency tree...
-Need to get 0 B of source archives.
-Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar)
-Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc)
+Need to get 6 B of source archives.
+Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) [3 B]
+Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) [3 B]
E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch
E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad