From 546dbfc82ad9ff0308b365bca3731a1118b1d251 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 21:26:20 +0100 Subject: disable the lock disabling in the tests We create our own directories here and work without root in them, so we can also test the locking with them as it is how we usually operate. Git-Dch: Ignore --- test/integration/test-apt-update-transactions | 1 + 1 file changed, 1 insertion(+) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index fe352c762..bf425a22e 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -49,6 +49,7 @@ testsetup() { msgmsg 'Test with no initial data over' "$1" rm -rf rootdir/var/lib/apt/lists mkdir -m 700 -p rootdir/var/lib/apt/lists/partial + touch rootdir/var/lib/apt/lists/lock if [ "$(id -u)" = '0' ]; then chown _apt:root rootdir/var/lib/apt/lists/partial fi -- cgit v1.2.3 From 8fe964f148344b8a55252fe52b6292a4ab86ea98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 14 Nov 2014 18:01:09 +0100 Subject: create our cache and lib directory always with mode 755 We autocreate for a while now the last two directories in /var/lib/apt/lists (similar for /var/cache/apt/archives) which is very nice for systems having any of those on tmpfs or other non-persistent storage. This also means though that this creation is effected by the default umask, so for people with aggressive umasks like 027 the directories will be created with 750, which means all non-root users are left out, which is usually exactly what we want then this umask is set, but the cache and lib directories contain public knowledge. There isn't any need to protect them from viewers and they render apt completely useless if not readable. --- test/integration/test-apt-update-transactions | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index bf425a22e..b325733ac 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -48,11 +48,7 @@ testrun() { testsetup() { msgmsg 'Test with no initial data over' "$1" rm -rf rootdir/var/lib/apt/lists - mkdir -m 700 -p rootdir/var/lib/apt/lists/partial - touch rootdir/var/lib/apt/lists/lock - if [ "$(id -u)" = '0' ]; then - chown _apt:root rootdir/var/lib/apt/lists/partial - fi + mkdir -p rootdir/var/lib/apt/lists/partial listcurrentlistsdirectory > listsdir.lst testrun 'listsdir.lst' -- cgit v1.2.3 From ba6b79bd0090077724fa1272ea4d3a31706fcd5a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 12 Apr 2015 17:08:46 +0200 Subject: a hit on Release files means the indexes will be hits too If we get a IMSHit for the Transaction-Manager (= the InRelease file or as its still supported fallback Release + Release.gpg combo) we can assume that every file we would queue based on this manager, but already have locally is current and hence would get an IMSHit, too. We therefore save us and the server the trouble and skip the queuing in this case. Beside speeding up repetative executions of 'apt-get update' this way we also avoid hitting hashsum errors if the indexes are in fact already updated, but the Release file isn't yet as it is the case on well behaving mirrors as Release files is updated last. The implementation is a bit harder than the theory makes it sound as we still have to keep reverifying the Release files (e.g. to detect now expired once to avoid an attacker being able to silently stale us) and have to handle cases in which the Release file hits, but some indexes aren't present (e.g. user added a new foreign architecture). --- test/integration/test-apt-update-transactions | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index b325733ac..f028ac0c7 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -59,6 +59,13 @@ testsetup() { } testsetup 'file' + changetowebserver +webserverconfig 'aptwebserver::support::modified-since' 'false' "$1" +webserverconfig 'aptwebserver::support::last-modified' 'false' "$1" # curl is clever and sees hits here also + testsetup 'http' +changetohttpswebserver + +testsetup 'https' -- cgit v1.2.3 From dcbb364fc69e1108b3fea3adb12a7ba83d9af467 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 12 May 2015 00:30:16 +0200 Subject: detect 416 complete file in partial by expected hash If we have the expected hashes we can check with them if the file we have in partial we got a 416 for is the expected file. We detected this with same-size before, but not every server sends a good Content-Range header with a 416 response. --- test/integration/test-apt-update-transactions | 1 + 1 file changed, 1 insertion(+) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index f028ac0c7..67dd633f9 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -63,6 +63,7 @@ testsetup 'file' changetowebserver webserverconfig 'aptwebserver::support::modified-since' 'false' "$1" webserverconfig 'aptwebserver::support::last-modified' 'false' "$1" # curl is clever and sees hits here also +webserverconfig 'aptwebserver::support::range' 'false' "$1" testsetup 'http' -- cgit v1.2.3 From 8eafc759544298211cd0bfaa3919afc0fadd47d1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 13 May 2015 16:09:12 +0200 Subject: detect Releasefile IMS hits even if the server doesn't Not all servers we are talking to support If-Modified-Since and some are not even sending Last-Modified for us, so in an effort to detect such hits we run a hashsum check on the 'old' compared to the 'new' file, we got the hashes for the 'new' already for "free" from the methods anyway and hence just need to calculated the old ones. This allows us to detect hits even with unsupported servers, which in turn means we benefit from all the new hit behavior also here. --- test/integration/test-apt-update-transactions | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index 67dd633f9..63b318056 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -47,6 +47,7 @@ testrun() { testsetup() { msgmsg 'Test with no initial data over' "$1" + redatereleasefiles 'now' rm -rf rootdir/var/lib/apt/lists mkdir -p rootdir/var/lib/apt/lists/partial listcurrentlistsdirectory > listsdir.lst @@ -55,6 +56,7 @@ testsetup() { msgmsg 'Test with initial data over' "$1" testsuccess aptget update listcurrentlistsdirectory > listsdir.lst + redatereleasefiles '+1hour' testrun 'listsdir.lst' } -- cgit v1.2.3 From 58702f8563a443a7c6e66253b259c2488b877290 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 19 May 2015 10:40:55 +0200 Subject: don't try other compressions on hashsum mismatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we e.g. fail on hash verification for Packages.xz its highly unlikely that it will be any better with Packages.gz, so we just waste download bandwidth and time. It also causes us always to fallback to the uncompressed Packages file for which the error will finally be reported, which in turn confuses users as the file usually doesn't exist on the mirrors, so a bug in apt is suspected for even trying it… --- test/integration/test-apt-update-transactions | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index 63b318056..152e1617a 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -8,6 +8,7 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'i386' +configcompression '.' 'gz' insertpackage 'unstable' 'foo' 'all' '1.0' insertsource 'unstable' 'foo' 'all' '1.0' @@ -15,7 +16,8 @@ insertsource 'unstable' 'foo' 'all' '1.0' setupaptarchive --no-update breakfile() { - mv "$1" "${1}.bak" + mv "${1}" "${1}.bak" + mv "${1}.gz" "${1}.gz.bak" cat > "$1" < listsdir.lst redatereleasefiles '+1hour' testrun 'listsdir.lst' -- cgit v1.2.3 From 8d041b4f4f353079268039dcbfd8b5e575196b66 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 8 Jun 2015 15:22:01 +0200 Subject: do not request files if we expect an IMS hit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we have a file on disk and the hashes are the same in the new Release file and the old one we have on disk we know that if we ask the server for the file, we will at best get an IMS hit – at worse the server doesn't support this and sends us the (unchanged) file and we have to run all our checks on it again for nothing. So, we can save ourselves (and the servers) some unneeded requests if we figure this out on our own. --- test/integration/test-apt-update-transactions | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'test/integration/test-apt-update-transactions') diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index 152e1617a..a5dac1737 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -29,6 +29,12 @@ restorefile() { } testrun() { + rm -rf aptarchive/dists.good + cp -a aptarchive/dists aptarchive/dists.good + insertpackage 'unstable' 'bar' 'all' '1.0' + insertsource 'unstable' 'bar' 'all' '1.0' + buildaptarchivefromfiles '+1 hour' + # produce an unsigned repository find aptarchive \( -name 'Release.gpg' -o -name 'InRelease' \) -delete testfailure aptget update --no-allow-insecure-repositories @@ -37,20 +43,27 @@ testrun() { # signed but broken signreleasefiles + onehashbroken() { + testfailure aptget update + # each file generates two messages with this string + testequal '2' grep --count 'Hash Sum mismatch' rootdir/tmp/testfailure.output + testfileequal "$1" "$(listcurrentlistsdirectory)" + } + breakfile aptarchive/dists/unstable/main/binary-i386/Packages - testfailure aptget update - testfileequal "$1" "$(listcurrentlistsdirectory)" + onehashbroken "$1" restorefile aptarchive/dists/unstable/main/binary-i386/Packages breakfile aptarchive/dists/unstable/main/source/Sources - testfailure aptget update - testfileequal "$1" "$(listcurrentlistsdirectory)" + onehashbroken "$1" restorefile aptarchive/dists/unstable/main/source/Sources + + rm -rf aptarchive/dists + cp -a aptarchive/dists.good aptarchive/dists } testsetup() { msgmsg 'Test with no initial data over' "$1" - redatereleasefiles 'now' rm -rf rootdir/var/lib/apt/lists mkdir -p rootdir/var/lib/apt/lists/partial listcurrentlistsdirectory > listsdir.lst @@ -60,7 +73,6 @@ testsetup() { rm -rf rootdir/var/lib/apt/lists testsuccess aptget update -o Debug::pkgAcquire::Worker=1 listcurrentlistsdirectory > listsdir.lst - redatereleasefiles '+1hour' testrun 'listsdir.lst' } -- cgit v1.2.3