From 0179cfa83cf0042235eda41db7f35c420781c63e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 7 Jan 2016 20:32:09 +0100 Subject: keep compressed indexes in a low-cost format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Downloading and storing are two different operations were different compression types can be preferred. For downloading we provide the choice via Acquire::CompressionTypes::Order as there is a choice to be made between download size and speed – and limited by whats available in the repository. Storage on the other hand has all compressions currently supported by apt available and to reduce runtime of tools accessing these files the compression type should be a low-cost format in terms of decompression. apt traditionally stores its indexes uncompressed on disk, but has options to keep them compressed. Now that apt downloads additional files we also deal with files which simply can't be stored uncompressed as they are just too big (like Contents for apt-file). Traditionally they are downloaded in a low-cost format (gz) as repositories do not provide other formats, but there might be even lower-cost formats and for download we could introduce higher-cost in the repositories. Downloading an entire index potentially requires recompression to another format, so an update takes potentially longer – but big files are usually updated via pdiffs which has to de- and re-compress anyhow and does it on the fly anyhow, so there is no extra time needed and in general it seems to be benefitial to invest the time in update to save time later on file access. --- test/integration/framework | 2 +- test/integration/test-apt-acquire-additional-files | 15 ++++++++------- test/integration/test-apt-update-ims | 8 ++++---- test/integration/test-apt-update-unauth | 10 +++++++--- test/integration/test-compressed-indexes | 9 +++++---- test/integration/test-pdiff-usage | 3 ++- 6 files changed, 27 insertions(+), 20 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index e2d545ec2..c97500ffd 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1826,7 +1826,7 @@ listcurrentlistsdirectory() { } forallsupportedcompressors() { for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do - if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = 'lz4' ]; then continue; fi + if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi "$@" "$COMP" done } diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index e9bbdbce6..bf7850a7a 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -10,6 +10,7 @@ configarchitecture 'amd64' # note that in --print-uri we talk about .xz because that is the default. # This doesn't mean it is actually attempt to download it. configcompression '.' 'gz' +LOWCOSTEXT='lz4' buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' @@ -101,14 +102,14 @@ Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B] Reading package lists..." aptget update -testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" find rootdir/var/lib/apt/lists -name '*Contents-amd64*' -testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" find rootdir/var/lib/apt/lists -name '*Contents-all*' -testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' -testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" 'aptarchive/dists/unstable/main/Contents-amd64.gz' -testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" 'aptarchive/dists/unstable/main/Contents-all.gz' +testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-amd64*' +testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*' +testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' +testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz' +testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT -rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz -rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz +rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT +rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' msgmsg "Compressed Contents file" diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index f175ca355..d433baeb5 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -24,15 +24,15 @@ runtest() { rm -rf rootdir/var/lib/apt/lists/ local TEST="test${1:-success}" - $TEST aptget update $APTOPT + $TEST aptget update $APTOPT -o Debug::pkgAcquire::Worker=1 if [ "$1" = 'failure' ]; then # accept the outdated Release file so we can check Hit behaviour "test${2:-success}" aptget update -o Acquire::Min-ValidTime=9999999 $APTOPT fi listcurrentlistsdirectory > listsdir.lst - testsuccess grep '_Packages\(\.gz\)\?$' listsdir.lst - testsuccess grep '_Sources\(\.gz\)\?$' listsdir.lst - testsuccess grep '_Translation-en\(\.gz\)\?$' listsdir.lst + testsuccess grep '_Packages\(\.[0-9a-z]\+\)\?$' listsdir.lst + testsuccess grep '_Sources\(\.[0-9a-z]\+\)\?$' listsdir.lst + testsuccess grep '_Translation-en\(\.[0-9a-z]\+\)\?$' listsdir.lst # ensure no leftovers in partial testfailure ls 'rootdir/var/lib/apt/lists/partial/*' diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth index 2ececc4f6..ef1e4eca7 100755 --- a/test/integration/test-apt-update-unauth +++ b/test/integration/test-apt-update-unauth @@ -12,6 +12,7 @@ umask 022 setupenvironment configarchitecture "i386" +configcompression '.' 'xz' insertpackage 'unstable' 'foo' 'i386' '1.0' insertsource 'unstable' 'foo' 'any' '1.0' @@ -45,7 +46,7 @@ runtest() { listcurrentlistsdirectory > lists.before # update and ensure all is reverted on the hashsum failure - testfailure aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0 + testfailure aptget update -o Debug::Acquire::Transaction=1 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0 # ensure we have before what we have after msgtest 'Check rollback on going from' 'unauth -> auth' @@ -53,10 +54,13 @@ runtest() { if cmp lists.before lists.after; then msgpass else + echo >&2 echo >&2 '### Output of previous apt-get update ###' - cat >&2 rootdir/tmp/testfailure.output + cat >&2 rootdir/tmp/testfailure.output || true echo >&2 '### Changes in the lists-directory: ###' - diff -u >&2 lists.before lists.after + diff -u >&2 lists.before lists.after || true + echo >&2 '### Contents of the lists-directory: ###' + ls -l rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists/partial || true msgfail fi diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes index b97e1244b..573a86511 100755 --- a/test/integration/test-compressed-indexes +++ b/test/integration/test-compressed-indexes @@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")" setupenvironment configcompression '.' $(aptconfig dump APT::Compressor --format '%t %v%n' | sed -n 's#^Extension \.\(.*\)$#\1#p') configarchitecture 'i386' +LOWCOSTEXT='lz4' buildsimplenativepackage 'testpkg' 'i386' '1.0' @@ -35,10 +36,10 @@ testrun() { ! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1 ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1 ! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1 - test -e rootdir/var/lib/apt/lists/*i386_Packages.${COMPRESS} || F=1 - test -e rootdir/var/lib/apt/lists/*all_Packages.${COMPRESS} || F=1 - test -e rootdir/var/lib/apt/lists/*_Sources.${COMPRESS} || F=1 - test -e rootdir/var/lib/apt/lists/*_Translation-en.${COMPRESS} || F=1 + test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1 + test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1 + test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1 + test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1 # there is no point in trying pdiff if we have compressed indexes # as we can't patch compressed files (well, we can, but what is the point?) ! test -e rootdir/var/lib/apt/lists/*diff_Index || F=1 diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 547242cad..32240da4e 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")" setupenvironment configarchitecture 'i386' confighashes 'SHA1' 'SHA256' +LOWCOSTEXT='lz4' buildaptarchive setupflataptarchive @@ -274,5 +275,5 @@ testcase() { } aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages"; } testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=false -aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.gz"; } +aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.$LOWCOSTEXT"; } testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=true -- cgit v1.2.3