From 08ea7806458de0995414eaae852e0a5985875642 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 13:16:43 +0200 Subject: deal better with acquiring the same URI multiple times This is an unlikely event for indexes and co, but it can happen quiet easily e.g. for changelogs where you want to get the changelogs for multiple binary package(version)s which happen to all be built from a single source. The interesting part is that the Acquire system actually detected this already and set the item requesting the URI again to StatDone - expect that this is hardly sufficient: an Item must be Complete=true as well to be considered truely done and that is only the tip of the ::Done handling iceberg. So instead of this StatDone hack we allow QItems to be owned by multiple items and notify all owners about everything now, so that for the point of each item they got it downloaded just for them. --- .../test-acquire-same-file-multiple-times | 80 ++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 test/integration/test-acquire-same-file-multiple-times (limited to 'test/integration/test-acquire-same-file-multiple-times') diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times new file mode 100755 index 000000000..d329a39cb --- /dev/null +++ b/test/integration/test-acquire-same-file-multiple-times @@ -0,0 +1,80 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' + +TESTFILE="$TESTDIR/framework" +cp $TESTFILE aptarchive/foo +APTARCHIVE="$(readlink -f ./aptarchive)" + +filedown() { + msgtest 'Downloading the same URI twice over file' "$1" + testsuccess --nomsg apthelper download-file file:///$APTARCHIVE/foo ./downloaded/foo1 '' file:///$APTARCHIVE/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + cp rootdir/tmp/testsuccess.output download.log + #cat download.log + testsuccess cmp $TESTFILE ./downloaded/foo1 + testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 + #testequal '1' grep -c '200%20URI%20Start' ./download.log + testequal '1' grep -c '201%20URI%20Done' ./download.log + rm -f ./downloaded/foo1 ./downloaded/foo2 +} + +testrun() { + $1 'no partial' + cp $TESTFILE ./downloaded/foo1 + $1 'complete partial 1' + cp $TESTFILE ./downloaded/foo2 + $1 'complete partial 2' + cp $TESTFILE ./downloaded/foo1 + cp $TESTFILE ./downloaded/foo2 + $1 'complete partial 1+2' + dd if=$TESTFILE of=./downloaded/foo1 bs=500 count=1 2>/dev/null + $1 'partial partial 1' + dd if=$TESTFILE of=./downloaded/foo2 bs=500 count=1 2>/dev/null + $1 'partial partial 2' + dd if=$TESTFILE of=./downloaded/foo1 bs=500 count=1 2>/dev/null + dd if=$TESTFILE of=./downloaded/foo2 bs=500 count=1 2>/dev/null + $1 'partial partial 1+2' +} +testrun 'filedown' + +changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo + +httpdown() { + msgtest 'Downloading the same URI to different files' 'twice over http' + testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + cp rootdir/tmp/testsuccess.output download.log + testsuccess cmp $TESTDIR/framework ./downloaded/foo1 + testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 + testequal '1' grep -c '200%20URI%20Start' ./download.log + testequal '1' grep -c '201%20URI%20Done' ./download.log + rm -f ./downloaded/foo1 ./downloaded/foo2 +} +testrun 'httpdown' + +httpredirectdown() { + msgtest 'Redirect leads' 'first URI to the second URI' + testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + cp rootdir/tmp/testsuccess.output download.log + testsuccess cmp $TESTDIR/framework ./downloaded/foo1 + testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 + testequal '1' grep -c '200%20URI%20Start' ./download.log + testequal '1' grep -c '103%20Redirect' ./download.log + testequal '1' grep -c '201%20URI%20Done' ./download.log + rm -f ./downloaded/foo1 ./downloaded/foo2 +} +testrun 'httpredirectdown' + +httpsamedown() { + msgtest 'Downloading two files via the same URI to' 'the same file' + testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo1 '' -o Debug::pkgAcquire::Worker=1 + cp rootdir/tmp/testsuccess.output download.log + testsuccess cmp $TESTDIR/framework ./downloaded/foo1 + testequal '1' grep -c '200%20URI%20Start' ./download.log + testequal '1' grep -c '201%20URI%20Done' ./download.log + rm -f ./downloaded/foo1 +} +testrun 'httpsamedown' -- cgit v1.2.3 From 9d2a8a7388cf3b0bbbe92f6b0b30a533e1167f40 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 23:06:56 +0200 Subject: condense parallel requests with the same hashes to one It shouldn't be too common, but sometimes people have multiple mirrors in the sources or otherwise repositories with the same content. Now that we gracefully can handle multiple requests to the same URI, we can also fold multiple requests with the same expected hashes into one. Note that this isn't trying to find oppertunities for merging, but just merges if it happens to encounter the oppertunity for it. This is most obvious in the new testcase actually as it needs to delay the action to give the acquire system enough time to figure out that they can be merged. --- test/integration/test-acquire-same-file-multiple-times | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/integration/test-acquire-same-file-multiple-times') diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times index d329a39cb..526765521 100755 --- a/test/integration/test-acquire-same-file-multiple-times +++ b/test/integration/test-acquire-same-file-multiple-times @@ -45,7 +45,7 @@ changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo httpdown() { msgtest 'Downloading the same URI to different files' 'twice over http' - testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 cp rootdir/tmp/testsuccess.output download.log testsuccess cmp $TESTDIR/framework ./downloaded/foo1 testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 @@ -57,7 +57,7 @@ testrun 'httpdown' httpredirectdown() { msgtest 'Redirect leads' 'first URI to the second URI' - testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 cp rootdir/tmp/testsuccess.output download.log testsuccess cmp $TESTDIR/framework ./downloaded/foo1 testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 -- cgit v1.2.3