From b2fd852459a6b9234255644730f48f071ccad64d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 14 Apr 2016 17:32:17 +0200 Subject: silently skip acquire of empty index files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is just no point in taking the time to acquire empty files – especially as it will be tiny non-empty compressed files usually. --- apt-pkg/acquire-item.cc | 12 ++++-- test/integration/framework | 6 +++ test/integration/test-apt-acquire-additional-files | 3 +- test/integration/test-apt-get-build-dep-file | 1 + test/integration/test-apt-update-empty-files | 6 ++- .../test-bug-595691-empty-and-broken-archive-files | 50 +++++++--------------- test/integration/test-compressed-indexes | 5 +-- .../test-handle-redirect-as-used-mirror-change | 5 +-- 8 files changed, 41 insertions(+), 47 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 0569c6dda..bc5e91fed 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1151,11 +1151,17 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify) /*{{{*/ else { auto const hashes = GetExpectedHashesFor(Target->MetaKey); - if (hashes.usable() == false && hashes.empty() == false) + if (hashes.empty() == false) { - _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"), + if (hashes.usable() == false) + { + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"), Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str()); - continue; + continue; + } + // empty files are skipped as acquiring the very small compressed files is a waste of time + else if (hashes.FileSize() == 0) + continue; } } diff --git a/test/integration/framework b/test/integration/framework index fc59c6450..213169a98 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -566,6 +566,11 @@ forcecompressor() { local CONFFILE="${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor" echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; }; Dir::Bin::uncompressed \"/does/not/exist\";" > "$CONFFILE" + for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do + if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = "$COMPRESSOR" ]; then continue; fi + echo "Dir::Bin::${COMP} \"/does/not/exist\";" >> "$CONFFILE" + echo "APT::Compressor::${COMP}::Name \"${COMP}-disabled\";" >> "$CONFFILE" + done } setupsimplenativepackage() { @@ -1843,6 +1848,7 @@ listcurrentlistsdirectory() { } | sort } forallsupportedcompressors() { + rm -f "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor" for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi "$@" "$COMP" diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 522b3adcb..afeeadd4f 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -15,6 +15,7 @@ configcompression '.' 'gz' LOWCOSTEXT='lz4' buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' +buildsimplenativepackage 'bar' 'all' '1' 'unstable' setupaptarchive --no-update changetowebserver @@ -110,7 +111,7 @@ testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_mai testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*' testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz' -testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT +testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-all.gz' rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT diff --git a/test/integration/test-apt-get-build-dep-file b/test/integration/test-apt-get-build-dep-file index 2cd4f57ec..233aed934 100755 --- a/test/integration/test-apt-get-build-dep-file +++ b/test/integration/test-apt-get-build-dep-file @@ -10,6 +10,7 @@ configarchitecture 'i386' insertpackage 'stable' 'debhelper' 'i386' '7' insertpackage 'stable' 'build-essential' 'i386' '1' insertpackage 'stable' 'build-depends' 'i386' '1' +insertsource 'stable' 'unrelated' 'any' '1' insertinstalledpackage 'build-conflict' 'i386' '1' setupaptarchive diff --git a/test/integration/test-apt-update-empty-files b/test/integration/test-apt-update-empty-files index e4f6d62bd..42329bb4b 100755 --- a/test/integration/test-apt-update-empty-files +++ b/test/integration/test-apt-update-empty-files @@ -17,9 +17,13 @@ msgmsg 'Test with file' rm -rf rootdir/var/lib/apt/lists testsuccess apt update -o Debug::pkgAcquire::Worker=1 cp rootdir/tmp/testsuccess.output apt.output -testsuccess grep '%0aAlt-Filename:%20/.*/Sources%0a' apt.output +testfailure grep '%0a\(Alt\)\?Filename:%20/.*/Sources\(\.gz\)\?%0a' apt.output +testempty find rootdir/var/lib/apt/lists -name '*_Sources' msgmsg 'Test with http' changetowebserver rm -rf rootdir/var/lib/apt/lists testsuccess apt update -o Debug::pkgAcquire::Worker=1 +cp rootdir/tmp/testsuccess.output apt.output +testfailure grep 'http:600.*Sources' apt.output +testempty find rootdir/var/lib/apt/lists -name '*_Sources' diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index 8d06f09c0..d71bd2ce5 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -10,35 +10,28 @@ buildaptarchive touch aptarchive/Packages setupflataptarchive -testaptgetupdate() { +failureupdate() { rm -rf rootdir/var/lib/apt - aptget update >testaptgetupdate.diff 2>&1 || true - sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff - GIVEN="$1" - shift - msgtest "Test for correctness of" "apt-get update with $*" - if [ -z "$GIVEN" ]; then - echo -n "" | checkdiff - testaptgetupdate.diff && msgpass || msgfail - else - echo "$GIVEN" | checkdiff - testaptgetupdate.diff && msgpass || msgfail - fi - rm testaptgetupdate.diff + testfailure aptget update --allow-insecure-repositories + testsuccess grep "^E: Failed to fetch store:${1} Empty files can't be valid archives$" rootdir/tmp/testfailure.output +} +successupdate() { + rm -rf rootdir/var/lib/apt + testwarning aptget update --allow-insecure-repositories } createemptyarchive() { find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete touch aptarchive/Packages echo -n "" | $COMPRESSOR_CMD > aptarchive/${1}.$COMPRESS - generatereleasefiles - signreleasefiles + find aptarchive -name '*Release*' -delete rm -f aptarchive/Packages } createemptyfile() { find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete touch aptarchive/Packages aptarchive/${1}.$COMPRESS - generatereleasefiles - signreleasefiles + find aptarchive -name '*Release*' -delete rm -f aptarchive/Packages } @@ -46,34 +39,21 @@ testoverfile() { local APTARCHIVE="$(readlink -f ./aptarchive)" forcecompressor "$1" + msgmsg 'archive over file' "Packages.$COMPRESS" createemptyarchive 'Packages' - testaptgetupdate "Get:2 file:$APTARCHIVE Packages [] -Reading package lists..." "empty archive Packages.$COMPRESS over file" - + successupdate createemptyfile 'Packages' - testaptgetupdate "Get:2 file:$APTARCHIVE Packages -Err:2 file:$APTARCHIVE Packages - Empty files can't be valid archives -Reading package lists... -E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS}) Empty files can't be valid archives -E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file" + failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS})" } testoverhttp() { forcecompressor "$1" + msgmsg 'archive over http' "Packages.$COMPRESS" createemptyarchive 'Packages' - testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT} Packages [] -Reading package lists..." "empty archive Packages.$COMPRESS over http" - + successupdate createemptyfile 'Packages' - #FIXME: we should response with a good error message instead - testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT} Packages -Err:2 http://localhost:${APTHTTPPORT} Packages - Empty files can't be valid archives -Reading package lists... -E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS}) Empty files can't be valid archives -E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http" + failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS})" } forallsupportedcompressors testoverfile diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes index 573a86511..7ddf2e2e2 100755 --- a/test/integration/test-compressed-indexes +++ b/test/integration/test-compressed-indexes @@ -33,11 +33,9 @@ testrun() { msgtest 'Check if all index files are' "${1:-uncompressed}" if [ "$1" = 'compressed' ]; then ! test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1 - ! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1 ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1 ! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1 test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1 - test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1 test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1 test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1 # there is no point in trying pdiff if we have compressed indexes @@ -47,11 +45,9 @@ testrun() { # clear the faked pdiff indexes so the glob below works rm -f rootdir/var/lib/apt/lists/*diff_Index test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1 - test -e rootdir/var/lib/apt/lists/*all_Packages || F=1 test -e rootdir/var/lib/apt/lists/*_Sources || F=1 test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1 ! test -e rootdir/var/lib/apt/lists/*i386_Packages.* || F=1 - ! test -e rootdir/var/lib/apt/lists/*all_Packages.* || F=1 ! test -e rootdir/var/lib/apt/lists/*_Sources.* || F=1 ! test -e rootdir/var/lib/apt/lists/*_Translation-en.* || F=1 fi @@ -112,6 +108,7 @@ testovermethod() { INDCOMP='uncompressed' else INDCOMP='compressed' + echo 'APT::Compressor::lz4::Name "lz4";' >> rootdir/etc/apt/apt.conf.d/02compressindex fi msgmsg "${1}: ${COMPRESSOR}: Test with $INDCOMP indexes gzip=$INDEX" diff --git a/test/integration/test-handle-redirect-as-used-mirror-change b/test/integration/test-handle-redirect-as-used-mirror-change index 41dd0772e..3a8b1fecd 100755 --- a/test/integration/test-handle-redirect-as-used-mirror-change +++ b/test/integration/test-handle-redirect-as-used-mirror-change @@ -16,9 +16,8 @@ rewritesourceslist "http://localhost:${APTHTTPPORT}/redirectme" testsuccessequal "Get:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease [$(stat -c %s aptarchive/dists/unstable/InRelease) B] Get:2 http://0.0.0.0:${APTHTTPPORT} unstable/main Sources [$(stat -c %s aptarchive/dists/unstable/main/source/Sources.gz) B] -Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main amd64 Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B] -Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B] -Get:5 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B] +Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B] +Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B] Reading package lists..." aptget update testsuccessequal "Hit:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease -- cgit v1.2.3