From 47d2bc78adb49f3182f9a3d7a4baea363e772d64 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 6 Dec 2013 12:17:48 +0100 Subject: implement POC client-side merging of pdiffs via apt-file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idea of pdiffs is to avoid downloading the hole file by patching the existing index. This works very well, but becomes slow if a lot of patches needs to be applied to reconstruct an up-to-date index and in recent years more and more dinstall (or similar) runs are executed creating more and more pdiffs in the same amount of time, so pdiffs became less useful. The solution is simple: Reduce the amount of patches (which are very small) which need to be applied on top of the index we have available (which is usually pretty big). This can be done in two ways: Either merge the patches on the server-side so that the client has to download only one patch or the patches are all downloaded and merged on the client-side. The first needs a client who is doing one step at a time who can also skip patches if it needs (APT supports this for a long time now). The later is implemented by this commit, but depends on the server NOT merging the patches and the patches being in a strict order in which no patch is skipped. This is traditionally the case for dak, but other repository creators support merging – e.g. reprepro (which helpfully adds a flag indicating that the patches are merged). To support both or even mixes a client needs more information which isn't available for now. This POC uses the external diffindex-rred included in apt-file to do the heavy lifting of merging & applying all patches in one pass, hence to test this feature apt-file needs to be installed. --- test/integration/test-pdiff-usage | 147 ++++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 20 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index ac0563b7f..5a06e0ccb 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -5,39 +5,146 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'i386' buildaptarchive setupflataptarchive changetowebserver -signreleasefiles -testsuccess aptget update -testnopackage newstuff PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" -testequal "$(cat ${PKGFILE}) + +echo '#!/bin/sh +touch merge-was-used +/usr/bin/diffindex-rred "$@"' > extrred +chmod +x extrred +echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred + +wasmergeused() { + testsuccess aptget update "$@" + msgtest 'Check if the right pdiff merger was used' + if [ -e ./merge-was-used ]; then + rm -f ./merge-was-used + if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then + msgpass + else + msgfail "Merge shouldn't have been used, but was" + fi + elif echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then + msgfail "Merge should have been used, but wasn't" + else + msgpass + fi +} + +testrun() { + # setup the base + find aptarchive -name 'Packages*' -type f -delete + cp ${PKGFILE} aptarchive/Packages + compressfile 'aptarchive/Packages' + generatereleasefiles + signreleasefiles + rm -rf aptarchive/Packages.diff rootdir/var/lib/apt/lists + testsuccess aptget update "$@" + cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak + testnopackage newstuff + testequal "$(cat ${PKGFILE}) " aptcache show apt oldstuff -cp ${PKGFILE}-new aptarchive/Packages -compressfile 'aptarchive/Packages' -rm -rf aptarchive/Packages.diff -mkdir -p aptarchive/Packages.diff -PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" -diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true -cat $PATCHFILE | gzip > ${PATCHFILE}.gz -PATCHINDEX="aptarchive/Packages.diff/Index" -echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) + # apply with one patch + cp ${PKGFILE}-new aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + cat $PATCHFILE | gzip > ${PATCHFILE}.gz + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) SHA1-History: 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) SHA1-Patches: 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX -generatereleasefiles '+1hour' -signreleasefiles -find aptarchive -name 'Packages*' -type f -delete -testsuccess aptget update + generatereleasefiles '+1hour' + signreleasefiles + find aptarchive -name 'Packages*' -type f -delete + wasmergeused "$@" + testnopackage oldstuff + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt newstuff -testnopackage oldstuff -testequal "$(cat ${PKGFILE}-new) + # index is already up-to-date + find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete + testsuccess aptget update "$@" + testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff + + # apply with two patches + cp ${PKGFILE}-new aptarchive/Packages + echo ' +Package: futurestuff +Version: 1.0 +Architecture: i386 +Maintainer: Joe Sixpack +Installed-Size: 202 +Filename: pool/futurestuff_1.0_i386.deb +Size: 202200 +MD5sum: 311aeeaaae5ba33aff1ceaf3e1f76671 +SHA1: 3c695e028f7a1ae324deeaae5ba332desa81088c +SHA256: b46fd154615edaae5ba33c56a5cc0e7deaef23e2da3e4f129727fd660f28f050 +Description: some cool and shiny future stuff + This package will appear in the next next mirror update +Description-md5: d5f89fbbc2ce34c455dfee9b67d82b6b' >> aptarchive/Packages + + compressfile 'aptarchive/Packages' + PATCHFILE2="aptarchive/Packages.diff/$(date -d 'now + 1hour' '+%Y-%m-%d-%H%M.%S')" + diff -e ${PKGFILE}-new aptarchive/Packages > ${PATCHFILE2} || true + cat $PATCHFILE2 | gzip > ${PATCHFILE2}.gz + echo "SHA1-Current: $(sha1sum aptarchive/Packages | cut -d' ' -f 1) $(stat -c%s aptarchive/Packages) +SHA1-History: + 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 + $(sha1sum ${PKGFILE} | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}) $(basename ${PATCHFILE}) + $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) $(basename ${PATCHFILE2}) +SHA1-Patches: + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE) + $(sha1sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX + generatereleasefiles '+2hour' + signreleasefiles + cp -a aptarchive/Packages Packages-future + find aptarchive -name 'Packages*' -type f -delete + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + wasmergeused "$@" + testnopackage oldstuff + testequal "$(cat Packages-future) +" aptcache show apt newstuff futurestuff + + # patch applying fails, but successful fallback + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + cp ${PKGFILE}-new aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA1-History: + 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 + $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA1-Patches: + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX + echo 'I am Mallory and I change files' >> $PATCHFILE + cat $PATCHFILE | gzip > ${PATCHFILE}.gz + generatereleasefiles '+1hour' + signreleasefiles + testsuccess aptget update "$@" + testnopackage oldstuff + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt newstuff +} + +testrun -o Debug::pkgAcquire::Diffs=1 -o Debug::pkgAcquire::rred=1 -o Acquire::PDiffs::Merge=0 +testrun -o Debug::pkgAcquire::Diffs=1 -o Debug::pkgAcquire::rred=1 -o Acquire::PDiffs::Merge=1 -- cgit v1.2.3 From 50bd6fd3794dd1f61185302129dc6cd218d20b98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jan 2014 17:23:05 +0100 Subject: integrate Anthonys rred with POC for client-side merge Providing the benefits of both without the downsides :) (ABI breaks or external dependencies) For this Anthonys rred is equipped with: - magic-filename-pickup of patches rather than explicit messages - use of FileFd instead of FILE* to get on-the-fly uncompress of the gzip compressed pdiff patches The acquire code in turn stops checking for apt-file's helper as our own rred is now clever enough for our needs. --- test/integration/test-pdiff-usage | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'test/integration/test-pdiff-usage') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 5a06e0ccb..ad31511b9 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -20,10 +20,19 @@ chmod +x extrred echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred wasmergeused() { - testsuccess aptget update "$@" + msgtest 'Test for successful execution of' "$*" + local OUTPUT=$(mktemp) + addtrap "rm $OUTPUT;" + if aptget update "$@" >${OUTPUT} 2>&1; then + msgpass + else + echo + cat $OUTPUT + msgfail + fi + msgtest 'Check if the right pdiff merger was used' - if [ -e ./merge-was-used ]; then - rm -f ./merge-was-used + if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' $OUTPUT; then if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then msgpass else @@ -50,7 +59,7 @@ testrun() { testequal "$(cat ${PKGFILE}) " aptcache show apt oldstuff - # apply with one patch + msgmsg 'Testcase: apply with one patch' cp ${PKGFILE}-new aptarchive/Packages compressfile 'aptarchive/Packages' mkdir -p aptarchive/Packages.diff @@ -73,13 +82,13 @@ SHA1-Patches: testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff - # index is already up-to-date + msgmsg 'Testcase: index is already up-to-date' find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete testsuccess aptget update "$@" testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff - # apply with two patches + msgmsg 'Testcase: apply with two patches' cp ${PKGFILE}-new aptarchive/Packages echo ' Package: futurestuff @@ -120,7 +129,7 @@ SHA1-Patches: testequal "$(cat Packages-future) " aptcache show apt newstuff futurestuff - # patch applying fails, but successful fallback + msgmsg 'Testcase: patch applying fails, but successful fallback' rm -rf rootdir/var/lib/apt/lists cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists cp ${PKGFILE}-new aptarchive/Packages -- cgit v1.2.3