From feb674aba51dcb26f5281b5b38fbc5893f757170 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 15 Jan 2016 02:45:35 +0100 Subject: revert file-hash based action-merging in acquire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduced in 9d2a8a7388cf3b0bbbe92f6b0b30a533e1167f40 apt tries to merge actions like downloading the same (as judged by hashes) file into doing it once. The implementation was very simple in that it isn't planing at all. Turns out that it works 90% of the time just fine, but has issues in more complicated situations in which items can be in different stages downloading different files emitting potentially the "wrong" hash – like while pdiffs are worked on we might end up copying the patch instead of the result file giving us very strange errors in return. Reverting the change until we can implement a better planing solution seems to be the best course of action even if its sad. Closes: 810046 --- apt-pkg/acquire.cc | 3 +- .../skip-acquire-same-repository-multiple-times | 85 ++++++++++++++++++++++ .../test-acquire-same-repository-multiple-times | 85 ---------------------- 3 files changed, 86 insertions(+), 87 deletions(-) create mode 100755 test/integration/skip-acquire-same-repository-multiple-times delete mode 100755 test/integration/test-acquire-same-repository-multiple-times diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 7a483f272..e515255ae 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -839,9 +839,8 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) { QItem **I = &Items; // move to the end of the queue and check for duplicates here - HashStringList const hsl = Item.Owner->GetExpectedHashes(); for (; *I != 0; I = &(*I)->Next) - if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes()) + if (Item.URI == (*I)->URI) { if (_config->FindB("Debug::pkgAcquire::Worker",false) == true) std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl; diff --git a/test/integration/skip-acquire-same-repository-multiple-times b/test/integration/skip-acquire-same-repository-multiple-times new file mode 100755 index 000000000..c8372bd41 --- /dev/null +++ b/test/integration/skip-acquire-same-repository-multiple-times @@ -0,0 +1,85 @@ +#!/bin/sh +set -e + +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" +setupenvironment +configarchitecture 'amd64' + +TESTFILE="$TESTDIR/framework" +cp "$TESTFILE" aptarchive/foo +APTARCHIVE="$(readlink -f ./aptarchive)" + +getcodenamefromsuite() { echo "jessie"; } +buildsimplenativepackage 'foo' 'all' '1.0' 'stable' +setupaptarchive --no-update +ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie" +for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do + sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g') +done + +# install a slowed down file: otherwise its to fast to reproduce combining +NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods" +OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)" +rm "$NEWMETHODS" +mkdir "$NEWMETHODS" +backupIFS="$IFS" +IFS="$(printf "\n\b")" +for METH in $(find "$OLDMETHODS" ! -type d); do + ln -s "$OLDMETHODS/$(basename "$METH")" "$NEWMETHODS" +done +IFS="$backupIFS" +rm "${NEWMETHODS}/file" "${NEWMETHODS}/http" +cat >"${NEWMETHODS}/file" < file.lst + testequal "$(find "$(readlink -f ./rootdir/var/lib/apt/lists)" -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst + testsuccess aptcache policy + testequal "foo: + Installed: (none) + Candidate: 1.0 + Version table: + 1.0 500 + 500 $1:$2 jessie/main all Packages + 500 $1:$2 stable/main all Packages" aptcache policy foo + testfailure aptcache show foo/unstable + testsuccess aptcache show foo/stable + testsuccess aptcache show foo/jessie +} + +tworepos 'file' "$APTARCHIVE" 'no partial' +testequal '14' grep -c '200%20URI%20Start' ./download.log +testequal '14' grep -c '201%20URI%20Done' ./download.log +testequal '8' grep -c '^ @ Queue: Action combined' ./download.log +tworepos 'file' "$APTARCHIVE" 'hit' +testequal '6' grep -c '200%20URI%20Start' ./download.log +testequal '6' grep -c '201%20URI%20Done' ./download.log +testequal '0' grep -c '^ @ Queue: Action combined' ./download.log +rm -rf rootdir/var/lib/apt/lists + +changetowebserver + +tworepos 'http' "//localhost:${APTHTTPPORT}" 'no partial' +testequal '12' grep -c '200%20URI%20Start' ./download.log +testequal '12' grep -c '201%20URI%20Done' ./download.log +testequal '8' grep -c '^ @ Queue: Action combined' ./download.log +tworepos 'http' "//localhost:${APTHTTPPORT}" 'hit' +testequal '2' grep -c '200%20URI%20Start' ./download.log +testequal '4' grep -c '201%20URI%20Done' ./download.log +testequal '0' grep -c '^ @ Queue: Action combined' ./download.log +rm -rf rootdir/var/lib/apt/lists diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times deleted file mode 100755 index c8372bd41..000000000 --- a/test/integration/test-acquire-same-repository-multiple-times +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -set -e - -TESTDIR="$(readlink -f "$(dirname "$0")")" -. "$TESTDIR/framework" -setupenvironment -configarchitecture 'amd64' - -TESTFILE="$TESTDIR/framework" -cp "$TESTFILE" aptarchive/foo -APTARCHIVE="$(readlink -f ./aptarchive)" - -getcodenamefromsuite() { echo "jessie"; } -buildsimplenativepackage 'foo' 'all' '1.0' 'stable' -setupaptarchive --no-update -ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie" -for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do - sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g') -done - -# install a slowed down file: otherwise its to fast to reproduce combining -NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods" -OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)" -rm "$NEWMETHODS" -mkdir "$NEWMETHODS" -backupIFS="$IFS" -IFS="$(printf "\n\b")" -for METH in $(find "$OLDMETHODS" ! -type d); do - ln -s "$OLDMETHODS/$(basename "$METH")" "$NEWMETHODS" -done -IFS="$backupIFS" -rm "${NEWMETHODS}/file" "${NEWMETHODS}/http" -cat >"${NEWMETHODS}/file" < file.lst - testequal "$(find "$(readlink -f ./rootdir/var/lib/apt/lists)" -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst - testsuccess aptcache policy - testequal "foo: - Installed: (none) - Candidate: 1.0 - Version table: - 1.0 500 - 500 $1:$2 jessie/main all Packages - 500 $1:$2 stable/main all Packages" aptcache policy foo - testfailure aptcache show foo/unstable - testsuccess aptcache show foo/stable - testsuccess aptcache show foo/jessie -} - -tworepos 'file' "$APTARCHIVE" 'no partial' -testequal '14' grep -c '200%20URI%20Start' ./download.log -testequal '14' grep -c '201%20URI%20Done' ./download.log -testequal '8' grep -c '^ @ Queue: Action combined' ./download.log -tworepos 'file' "$APTARCHIVE" 'hit' -testequal '6' grep -c '200%20URI%20Start' ./download.log -testequal '6' grep -c '201%20URI%20Done' ./download.log -testequal '0' grep -c '^ @ Queue: Action combined' ./download.log -rm -rf rootdir/var/lib/apt/lists - -changetowebserver - -tworepos 'http' "//localhost:${APTHTTPPORT}" 'no partial' -testequal '12' grep -c '200%20URI%20Start' ./download.log -testequal '12' grep -c '201%20URI%20Done' ./download.log -testequal '8' grep -c '^ @ Queue: Action combined' ./download.log -tworepos 'http' "//localhost:${APTHTTPPORT}" 'hit' -testequal '2' grep -c '200%20URI%20Start' ./download.log -testequal '4' grep -c '201%20URI%20Done' ./download.log -testequal '0' grep -c '^ @ Queue: Action combined' ./download.log -rm -rf rootdir/var/lib/apt/lists -- cgit v1.2.3