From ba6b79bd0090077724fa1272ea4d3a31706fcd5a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 12 Apr 2015 17:08:46 +0200 Subject: a hit on Release files means the indexes will be hits too If we get a IMSHit for the Transaction-Manager (= the InRelease file or as its still supported fallback Release + Release.gpg combo) we can assume that every file we would queue based on this manager, but already have locally is current and hence would get an IMSHit, too. We therefore save us and the server the trouble and skip the queuing in this case. Beside speeding up repetative executions of 'apt-get update' this way we also avoid hitting hashsum errors if the indexes are in fact already updated, but the Release file isn't yet as it is the case on well behaving mirrors as Release files is updated last. The implementation is a bit harder than the theory makes it sound as we still have to keep reverifying the Release files (e.g. to detect now expired once to avoid an attacker being able to silently stale us) and have to handle cases in which the Release file hits, but some indexes aren't present (e.g. user added a new foreign architecture). --- test/integration/test-apt-update-not-modified | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 test/integration/test-apt-update-not-modified (limited to 'test/integration/test-apt-update-not-modified') diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified new file mode 100755 index 000000000..2dc56e76c --- /dev/null +++ b/test/integration/test-apt-update-not-modified @@ -0,0 +1,45 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' 'i386' + +insertpackage 'unstable' 'apt' 'all' '1.0' + +setupaptarchive --no-update + +methodtest() { + msgmsg 'Test with' "$1" + rm -rf rootdir/var/lib/apt/lists + # get our cache populated + testsuccess aptget update + listcurrentlistsdirectory > listsdir.lst + + # hit again with a good cache + testsuccessequal "Hit $1 unstable InRelease +Reading package lists..." aptget update + testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" + + # drop an architecture, which means the file should be gone now + configarchitecture 'i386' + sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst + testsuccessequal "Hit $1 unstable InRelease +Reading package lists..." aptget update + testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)" + + # readd arch so its downloaded again + configarchitecture 'amd64' 'i386' + testsuccessequal "Hit $1 unstable InRelease +Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B] +Reading package lists..." aptget update + testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" +} + +changetowebserver +methodtest 'http://localhost:8080' + +changetohttpswebserver +methodtest 'https://localhost:4433' -- cgit v1.2.3 From 8eafc759544298211cd0bfaa3919afc0fadd47d1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 13 May 2015 16:09:12 +0200 Subject: detect Releasefile IMS hits even if the server doesn't Not all servers we are talking to support If-Modified-Since and some are not even sending Last-Modified for us, so in an effort to detect such hits we run a hashsum check on the 'old' compared to the 'new' file, we got the hashes for the 'new' already for "free" from the methods anyway and hence just need to calculated the old ones. This allows us to detect hits even with unsupported servers, which in turn means we benefit from all the new hit behavior also here. --- test/integration/test-apt-update-not-modified | 58 ++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) (limited to 'test/integration/test-apt-update-not-modified') diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified index 2dc56e76c..a67ecb760 100755 --- a/test/integration/test-apt-update-not-modified +++ b/test/integration/test-apt-update-not-modified @@ -12,7 +12,7 @@ insertpackage 'unstable' 'apt' 'all' '1.0' setupaptarchive --no-update methodtest() { - msgmsg 'Test with' "$1" + msgmsg 'Test InRelease with' "$1" rm -rf rootdir/var/lib/apt/lists # get our cache populated testsuccess aptget update @@ -36,6 +36,62 @@ Reading package lists..." aptget update Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B] Reading package lists..." aptget update testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" + + webserverconfig 'aptwebserver::support::modified-since' 'false' + webserverconfig 'aptwebserver::support::last-modified' 'false' + testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B] +Reading package lists..." aptget update + webserverconfig 'aptwebserver::support::modified-since' 'true' + webserverconfig 'aptwebserver::support::last-modified' 'true' + + msgmsg 'Test Release.gpg with' "$1" + rm -rf rootdir/var/lib/apt/lists + cp -a aptarchive/dists aptarchive/dists.good + find aptarchive/dists -name 'InRelease' -delete + # get our cache populated + testsuccess aptget update + listcurrentlistsdirectory > listsdir.lst + + # hit again with a good cache + testsuccessequal "Ign $1 unstable InRelease + 404 Not Found +Hit $1 unstable Release +Hit $1 unstable Release.gpg +Reading package lists..." aptget update + testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" + + # drop an architecture, which means the file should be gone now + configarchitecture 'i386' + sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst + testsuccessequal "Ign $1 unstable InRelease + 404 Not Found +Hit $1 unstable Release +Hit $1 unstable Release.gpg +Reading package lists..." aptget update + testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)" + + # readd arch so its downloaded again + configarchitecture 'amd64' 'i386' + testsuccessequal "Ign $1 unstable InRelease + 404 Not Found +Hit $1 unstable Release +Hit $1 unstable Release.gpg +Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B] +Reading package lists..." aptget update + testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" + + webserverconfig 'aptwebserver::support::modified-since' 'false' + webserverconfig 'aptwebserver::support::last-modified' 'false' + testsuccessequal "Ign $1 unstable InRelease + 404 Not Found +Get:1 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B] +Get:2 $1 unstable Release.gpg [$(stat -c '%s' 'aptarchive/dists/unstable/Release.gpg') B] +Reading package lists..." aptget update + webserverconfig 'aptwebserver::support::modified-since' 'true' + webserverconfig 'aptwebserver::support::last-modified' 'true' + + rm -rf aptarchive/dists + cp -a aptarchive/dists.good aptarchive/dists } changetowebserver -- cgit v1.2.3 From 6bf93605fdb8e858d3f0a79a124c1d39f760094d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 18 May 2015 22:15:06 +0200 Subject: treat older Release files than we already have as an IMSHit Valid-Until protects us from long-living downgrade attacks, but not all repositories have it and an attacker could still use older but still valid files to downgrade us. While this makes it sounds like a security improvement now, its a bit theoretical at best as an attacker with capabilities to pull this off could just as well always keep us days (but in the valid period) behind and always knows which state we have, as we tell him with the If-Modified-Since header. This is also why this is 'silently' ignored and treated as an IMSHit rather than screamed at the user as this can at best be an annoyance for attackers. An error here would 'regularily' be encountered by users by out-of-sync mirrors serving a single run (e.g. load balancer) or in two consecutive runs on the other hand, so it would just help teaching people ignore it. That said, most of the code churn is caused by enforcing this additional requirement. Crisscross from InRelease to Release.gpg is e.g. very unlikely in practice, but if we would ignore it an attacker could sidestep it this way. --- test/integration/test-apt-update-not-modified | 4 ---- 1 file changed, 4 deletions(-) (limited to 'test/integration/test-apt-update-not-modified') diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified index a67ecb760..b1d55c156 100755 --- a/test/integration/test-apt-update-not-modified +++ b/test/integration/test-apt-update-not-modified @@ -56,7 +56,6 @@ Reading package lists..." aptget update testsuccessequal "Ign $1 unstable InRelease 404 Not Found Hit $1 unstable Release -Hit $1 unstable Release.gpg Reading package lists..." aptget update testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" @@ -66,7 +65,6 @@ Reading package lists..." aptget update testsuccessequal "Ign $1 unstable InRelease 404 Not Found Hit $1 unstable Release -Hit $1 unstable Release.gpg Reading package lists..." aptget update testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)" @@ -75,7 +73,6 @@ Reading package lists..." aptget update testsuccessequal "Ign $1 unstable InRelease 404 Not Found Hit $1 unstable Release -Hit $1 unstable Release.gpg Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B] Reading package lists..." aptget update testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)" @@ -85,7 +82,6 @@ Reading package lists..." aptget update testsuccessequal "Ign $1 unstable InRelease 404 Not Found Get:1 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B] -Get:2 $1 unstable Release.gpg [$(stat -c '%s' 'aptarchive/dists/unstable/Release.gpg') B] Reading package lists..." aptget update webserverconfig 'aptwebserver::support::modified-since' 'true' webserverconfig 'aptwebserver::support::last-modified' 'true' -- cgit v1.2.3