From b3501edb7091ca3aa6c2d6d96dc667b8161dd2b9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 19 Aug 2013 00:00:23 +0200 Subject: use HashStringList in the acquire system It is not very extensible to have the supported Hashes hardcoded everywhere and especially if it is part of virtual method names. It is also possible that a method does not support the 'best' hash (yet), so we might end up not being able to verify a file even though we have a common subset of supported hashes. And those are just two of the cases in which it is handy to have a more dynamic selection. The downside is that this is a MAJOR API break, but the HashStringList has a string constructor for compatibility, so with a bit of luck the few frontends playing with the acquire system directly are okay. --- test/integration/test-pdiff-usage | 6 ------ 1 file changed, 6 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index afe1ad443..74749d6ab 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -13,12 +13,6 @@ changetowebserver PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" -echo '#!/bin/sh -touch merge-was-used -/usr/bin/diffindex-rred "$@"' > extrred -chmod +x extrred -echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred - wasmergeused() { msgtest 'Test for successful execution of' "$*" local OUTPUT=$(mktemp) -- cgit v1.2.3 From 03bfbc965443393b92b2d6d82613472fa3a5067f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 23 Sep 2014 23:48:19 +0200 Subject: make pdiff transactional (but at the cost of a CopyFile() --- test/integration/test-pdiff-usage | 1 + 1 file changed, 1 insertion(+) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 74749d6ab..e86963f28 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -159,6 +159,7 @@ SHA1-Patches: " aptcache show apt newstuff } echo 'Debug::pkgAcquire::Diffs "true"; +Debug::Acquire::Transaction "true"; Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 -- cgit v1.2.3 From f6d4ab9ad8a2cfe52737ab620dd252cf8ceec43d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 29 Sep 2014 22:45:52 +0200 Subject: support parsing of all hashes for pdiff The fileformat of a pdiff index stores currently only SHA1 hashes. With this change, we look for all other hashes we support as well and take what we get, so that we can work after the release of jessie to get right of SHA1 if we want to. Note that the completely patched file is and was checked against the hashes collected from the Release file, so this transition isn't mission critical. --- test/integration/test-pdiff-usage | 48 ++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 74749d6ab..0d5261429 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -76,8 +76,15 @@ SHA1-History: 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) SHA1-Patches: - 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 - $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) +SHA256-Current: $(sha256sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA256-History: + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 + $(sha256sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA256-Patches: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 + $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX generatereleasefiles '+1hour' signreleasefiles find aptarchive -name 'Packages*' -type f -delete @@ -119,9 +126,18 @@ SHA1-History: $(sha1sum ${PKGFILE} | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}) $(basename ${PATCHFILE}) $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) $(basename ${PATCHFILE2}) SHA1-Patches: - 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28 $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) - $(sha1sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX + $(sha1sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2}) +SHA256-Current: $(sha256sum aptarchive/Packages | cut -d' ' -f 1) $(stat -c%s aptarchive/Packages) +SHA256-History: + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 + $(sha256sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) + $(sha256sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) $(basename ${PATCHFILE2}) +SHA256-Patches: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 + $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) + $(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX generatereleasefiles '+2hour' signreleasefiles cp -a aptarchive/Packages Packages-future @@ -147,8 +163,15 @@ SHA1-History: 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) SHA1-Patches: - 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 - $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) +SHA256-Current: $(sha256sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA256-History: + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 + $(sha256sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA256-Patches: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 + $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX echo 'I am Mallory and I change files' >> $PATCHFILE cat $PATCHFILE | gzip > ${PATCHFILE}.gz generatereleasefiles '+1hour' @@ -165,3 +188,16 @@ testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0 testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=0 + +sha256sum() { + echo '01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b -' +} +testrun -o Acquire::PDiffs::Merge=0 -o Acquire::ForceHash=SHA1 +testrun -o Acquire::PDiffs::Merge=1 -o Acquire::ForceHash=SHA1 + +unset -f sha256sum +sha1sum() { + echo 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc -' +} +testrun -o Acquire::PDiffs::Merge=0 -o Acquire::ForceHash=SHA256 +testrun -o Acquire::PDiffs::Merge=1 -o Acquire::ForceHash=SHA256 -- cgit v1.2.3 From c355ea30383486e74635cd0248d3b53ed9759c39 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:04:29 +0100 Subject: reenable patchsize limit option for pdiffs One word: "doh!" Commit f6d4ab9ad8a2cfe52737ab620dd252cf8ceec43d disabled the check to prevent apt from downloading bigger patches than the index it tries to patch. Happens rarly of course, but still. Detected by scan-build complaining about a dead assignment. To make up for the mistake a test is included as well. --- test/integration/test-pdiff-usage | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 5bad90214..d773dcd66 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -180,6 +180,40 @@ SHA256-Patches: testnopackage oldstuff testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff + + msgmsg "Testcase: pdiff patch bigger than index itself: $*" + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + cp ${PKGFILE}-new aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + cat $PATCHFILE | gzip > ${PATCHFILE}.gz + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA1-History: + 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 + $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA1-Patches: + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE) +SHA256-Current: $(sha256sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA256-History: + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 + $(sha256sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA256-Patches: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 + $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX + generatereleasefiles '+1hour' + signreleasefiles + #find aptarchive -name 'Packages*' -type f -delete + testsuccess aptget update -o Debug::pkgAcquire::Diffs=1 "$@" + cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output + testsuccess grep 'bytes (Limit is' rootdir/tmp/aptgetupdate.output + testnopackage oldstuff + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt newstuff } echo 'Debug::pkgAcquire::Diffs "true"; Debug::Acquire::Transaction "true"; -- cgit v1.2.3 From 25b86db159fbc3c043628e285c0c1ef24dec2c6e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Mar 2015 00:59:44 +0100 Subject: test exitcode as well as string equality We use test{success,failure} now all over the place in the framework, so its only consequencial to do this in the situations in which we test for a specific output as well. Git-Dch: Ignore --- test/integration/test-pdiff-usage | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index d773dcd66..4de07f1ad 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -60,7 +60,7 @@ testrun() { testsuccess aptget update "$@" cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak testnopackage newstuff - testequal "$(cat ${PKGFILE}) + testsuccessequal "$(cat ${PKGFILE}) " aptcache show apt oldstuff msgmsg "Testcase: apply with one patch: $*" @@ -90,13 +90,13 @@ SHA256-Patches: find aptarchive -name 'Packages*' -type f -delete wasmergeused "$@" testnopackage oldstuff - testequal "$(cat ${PKGFILE}-new) + testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff msgmsg "Testcase: index is already up-to-date: $*" find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete testsuccess aptget update "$@" - testequal "$(cat ${PKGFILE}-new) + testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff msgmsg "Testcase: apply with two patches: $*" @@ -146,7 +146,7 @@ SHA256-Patches: cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists wasmergeused "$@" testnopackage oldstuff - testequal "$(cat Packages-future) + testsuccessequal "$(cat Packages-future) " aptcache show apt newstuff futurestuff msgmsg "Testcase: patch applying fails, but successful fallback: $*" @@ -178,7 +178,7 @@ SHA256-Patches: signreleasefiles testsuccess aptget update "$@" testnopackage oldstuff - testequal "$(cat ${PKGFILE}-new) + testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff msgmsg "Testcase: pdiff patch bigger than index itself: $*" @@ -212,7 +212,7 @@ SHA256-Patches: cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output testsuccess grep 'bytes (Limit is' rootdir/tmp/aptgetupdate.output testnopackage oldstuff - testequal "$(cat ${PKGFILE}-new) + testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff } echo 'Debug::pkgAcquire::Diffs "true"; -- cgit v1.2.3 From ba6b79bd0090077724fa1272ea4d3a31706fcd5a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 12 Apr 2015 17:08:46 +0200 Subject: a hit on Release files means the indexes will be hits too If we get a IMSHit for the Transaction-Manager (= the InRelease file or as its still supported fallback Release + Release.gpg combo) we can assume that every file we would queue based on this manager, but already have locally is current and hence would get an IMSHit, too. We therefore save us and the server the trouble and skip the queuing in this case. Beside speeding up repetative executions of 'apt-get update' this way we also avoid hitting hashsum errors if the indexes are in fact already updated, but the Release file isn't yet as it is the case on well behaving mirrors as Release files is updated last. The implementation is a bit harder than the theory makes it sound as we still have to keep reverifying the Release files (e.g. to detect now expired once to avoid an attacker being able to silently stale us) and have to handle cases in which the Release file hits, but some indexes aren't present (e.g. user added a new foreign architecture). --- test/integration/test-pdiff-usage | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 4de07f1ad..5e759e50e 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -14,16 +14,7 @@ changetowebserver PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" wasmergeused() { - msgtest 'Test for successful execution of' "$*" - local OUTPUT=$(mktemp) - addtrap "rm $OUTPUT;" - if aptget update "$@" >${OUTPUT} 2>&1; then - msgpass - else - echo - cat $OUTPUT - msgfail - fi + testsuccess aptget update "$@" msgtest 'No intermediate patch files' 'still exist' local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')" @@ -36,7 +27,7 @@ wasmergeused() { fi msgtest 'Check if the right pdiff merger was used' - if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' $OUTPUT; then + if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' rootdir/tmp/testsuccess.output; then if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then msgpass else @@ -96,6 +87,8 @@ SHA256-Patches: msgmsg "Testcase: index is already up-to-date: $*" find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete testsuccess aptget update "$@" + testequal 'Hit http://localhost:8080 InRelease +Reading package lists...' aptget update "$@" -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Diffs=0 testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff -- cgit v1.2.3 From 146f7715a9f36d246b461255b3c683b479296915 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Apr 2015 10:59:27 +0200 Subject: improve partial/ cleanup in abort and failure cases Especially pdiff-enhanced downloads have the tendency to fail for various reasons from which we can recover and even a successful download used to leave the old unpatched index in partial/. By adding a new method responsible for making the transaction of an individual file happen we can at specialisations especially for abort cases to deal with the cleanup. This also helps in keeping the compressed indexes around if another index failed instead of keeping the decompressed files, which we wouldn't pick up in the next call. --- test/integration/test-pdiff-usage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 5e759e50e..7d72a6944 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -47,7 +47,7 @@ testrun() { compressfile 'aptarchive/Packages' generatereleasefiles signreleasefiles - rm -rf aptarchive/Packages.diff rootdir/var/lib/apt/lists + rm -rf aptarchive/Packages.diff rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak testsuccess aptget update "$@" cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak testnopackage newstuff -- cgit v1.2.3