From 3679515479136179e0d95325a6559fcc6d0af7f8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 6 Jun 2015 19:16:45 +0200 Subject: check patch hashes in rred worker instead of in the handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rred is responsible for unpacking and reading the patch files in one go, but we currently only have hashes for the uncompressed patch files, so the handler read the entire patch file before dispatching it to the worker which would read it again – both with an implicit uncompress. Worse, while the workers operate in parallel the handler is the central orchestration unit, so having it busy with work means the workers do (potentially) nothing. This means rred is working with 'untrusted' data, which is bad. Yet, having the unpack in the handler meant that the untrusted uncompress was done as root which isn't better either. Now, we have it at least contained in a binary which we can harden a bit better. In the long run, we want hashes for the compressed patch files through to be safe. --- test/integration/test-pdiff-usage | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 7d72a6944..73df61895 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -170,6 +170,8 @@ SHA256-Patches: generatereleasefiles '+1hour' signreleasefiles testsuccess aptget update "$@" + cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output + testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output testnopackage oldstuff testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff -- cgit v1.2.3 From 6d3e5bd8e08564c5eb12ecd869de5bd71e25f59d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 7 Jun 2015 02:17:15 +0200 Subject: add more parsing error checking for rred The rred parser is very accepting regarding 'invalid' files. Given that we can't trust the input it might be a bit too relaxed. In any case, checking for more errors can't hurt given that we support only a very specific subset of ed commands. --- test/integration/test-pdiff-usage | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 73df61895..7a9f6496b 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -165,7 +165,8 @@ SHA256-History: SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX - echo 'I am Mallory and I change files' >> $PATCHFILE + # needs to look like a valid command, otherwise the parser will fail before hashes are checked + echo '1d' >> $PATCHFILE cat $PATCHFILE | gzip > ${PATCHFILE}.gz generatereleasefiles '+1hour' signreleasefiles -- cgit v1.2.3 From 4f51fd8636592a96aecf17c8bf4cfdb3ea2207cc Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 8 Jun 2015 00:06:41 +0200 Subject: support hashes for compressed pdiff files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment we only have hashes for the uncompressed pdiff files, but via the new '$HASH-Download' field in the .diff/Index hashes can be provided for the .gz compressed pdiff file, which apt will pick up now and use to verify the download. Now, we "just" need a buy in from the creators of repositories… --- test/integration/test-pdiff-usage | 46 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 7a9f6496b..3295d5497 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -42,6 +42,8 @@ wasmergeused() { testrun() { msgmsg "Testcase: setup the base with: $*" + local DOWNLOADHASH=true + if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi find aptarchive -name 'Packages*' -type f -delete cp ${PKGFILE} aptarchive/Packages compressfile 'aptarchive/Packages' @@ -76,6 +78,15 @@ SHA256-History: SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX + if $DOWNLOADHASH; then + echo "SHA1-Download: + 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz + $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz) +SHA256-Download: + d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz + $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX + fi + generatereleasefiles '+1hour' signreleasefiles find aptarchive -name 'Packages*' -type f -delete @@ -131,6 +142,17 @@ SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) $(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX + if $DOWNLOADHASH; then + echo "SHA1-Download: + 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz + $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz) + $(sha1sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz) +SHA256-Download: + d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz + $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz) + $(sha256sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)" >> $PATCHINDEX + fi + generatereleasefiles '+2hour' signreleasefiles cp -a aptarchive/Packages Packages-future @@ -150,6 +172,7 @@ SHA256-Patches: mkdir -p aptarchive/Packages.diff PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + cat $PATCHFILE | gzip > ${PATCHFILE}.gz PATCHINDEX='aptarchive/Packages.diff/Index' echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) SHA1-History: @@ -165,14 +188,22 @@ SHA256-History: SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX + if $DOWNLOADHASH; then + echo "SHA1-Download: + 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz + $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz) +SHA256-Download: + d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz + $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX + fi # needs to look like a valid command, otherwise the parser will fail before hashes are checked - echo '1d' >> $PATCHFILE + echo '1d' > $PATCHFILE cat $PATCHFILE | gzip > ${PATCHFILE}.gz generatereleasefiles '+1hour' signreleasefiles testsuccess aptget update "$@" cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output - testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output + testsuccess grep 'Hash Sum mismatch' rootdir/tmp/aptgetupdate.output testnopackage oldstuff testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff @@ -201,6 +232,14 @@ SHA256-History: SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX + if $DOWNLOADHASH; then + echo "SHA1-Download: + 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz + $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz) +SHA256-Download: + d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz + $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX + fi generatereleasefiles '+1hour' signreleasefiles #find aptarchive -name 'Packages*' -type f -delete @@ -215,6 +254,9 @@ echo 'Debug::pkgAcquire::Diffs "true"; Debug::Acquire::Transaction "true"; Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf +testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 +testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 + testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0 -- cgit v1.2.3 From 1eb1836f4b5397497bd34f0cf516e6e4e73117bf Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 16:41:43 +0200 Subject: show item ID in Hit, Ign and Err lines as well Again, consistency is the main sellingpoint here, but this way it is now also easier to explain that some files move through different stages and lines are printed for them hence multiple times: That is a bit hard to believe if the number is changing all the time, but now that it keeps consistent. --- test/integration/test-pdiff-usage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 3295d5497..e5fe21e0f 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -98,7 +98,7 @@ SHA256-Download: msgmsg "Testcase: index is already up-to-date: $*" find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete testsuccess aptget update "$@" - testequal 'Hit http://localhost:8080 InRelease + testequal 'Hit:1 http://localhost:8080 InRelease Reading package lists...' aptget update "$@" -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Diffs=0 testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff -- cgit v1.2.3