From 4a808deaac462e7714a345dac676c6da294a2ee0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 14 Mar 2016 01:09:32 +0100 Subject: require $(HASH)-Download field in .diff/Index files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we ignore SHA1-only files it makes sense to require also the provision of hashes for the compressed patches as this was introduced in the same patchset as support for non-SHA1 hashes in the file itself in dak and adding support in other archive creators (if they support pdiffs at all) will likely be in the same batch. The reason for the change itself is simple: If you are 'scared' enough about the security of SHA1, you shouldn't uncompress a file you haven't verified at all – after all, it could be exploiting a bug or a zip bomb. --- apt-pkg/acquire-item.cc | 41 +++++++++------------ test/integration/test-pdiff-usage | 77 +++++++++++++++++++++------------------ 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2cf6b60a8..45f8cb76c 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -206,12 +206,12 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const { - /* We don't always have the diff of the downloaded pdiff file. - What we have for sure is hashes for the uncompressed file, - but rred uncompresses them on the fly while parsing, so not handled here. - Hashes are (also) checked while searching for (next) patch to apply. */ + /* We can't check hashes of rred result as we don't know what the + hash of the file will be. We just know the hash of the patch(es), + the hash of the file they will apply on and the hash of the resulting + file. */ if (State == StateFetchDiff) - return available_patches[0].download_hashes.empty() == false; + return true; return false; } HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const @@ -227,7 +227,7 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const we can check the rred result after all patches are applied as we know the expected result rather than potentially apply more patches */ if (State == StateFetchDiff) - return patch.download_hashes.empty() == false; + return true; return State == StateApplyDiff; } HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const @@ -2022,6 +2022,17 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ return false; } + for (auto const &patch: available_patches) + if (patch.result_hashes.usable() == false || + patch.patch_hashes.usable() == false || + patch.download_hashes.usable() == false) + { + if (Debug) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file + << " so fallback to complete download" << std::endl; + return false; + } + // patching with too many files is rather slow compared to a fast download unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); if (fileLimit != 0 && fileLimit < available_patches.size()) @@ -2036,7 +2047,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100); if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr) { - // compressed case unsigned long long downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { return T + I.download_hashes.FileSize(); @@ -2065,23 +2075,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ return false; } } - // uncompressed case - downloadSize = std::accumulate(available_patches.begin(), - available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { - return T + I.patch_hashes.FileSize(); - }); - if (downloadSize != 0) - { - unsigned long long const downloadSizeIdx = ServerSize; - unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent; - if ((sizeLimit/100) < downloadSize) - { - if (Debug) - std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", " - << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl; - return false; - } - } } // we have something, queue the diffs diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index e2330d065..2318448f5 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -30,7 +30,7 @@ wasmergeused() { if echo "$*" | grep -q -- '-o test::cannot-use-pdiff=1'; then msgtest 'Check if pdiff was' 'not used' cp -a rootdir/tmp/testsuccess.output rootdir/tmp/aptupdate.output - testsuccess --nomsg grep 'diff_Index: Did not find a good hashsum in the index' rootdir/tmp/aptupdate.output + testsuccess --nomsg grep "diff/Index with Message: Couldn't parse pdiff index" rootdir/tmp/aptupdate.output return; fi @@ -51,8 +51,6 @@ wasmergeused() { testrun() { configcompression '.' 'xz' msgmsg "Testcase: setup the base with: $*" - local DOWNLOADHASH=true - if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi find aptarchive -name 'Packages*' -type f -delete cp "${PKGFILE}" aptarchive/Packages compressfile 'aptarchive/Packages' @@ -84,12 +82,10 @@ SHA256-History: $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE") SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 - $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX" - if $DOWNLOADHASH; then - echo "SHA256-Download: + $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE") +SHA256-Download: d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz - $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX" - fi + $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX" generatereleasefiles '+1hour' signreleasefiles @@ -99,7 +95,7 @@ SHA256-Patches: testsuccessequal "$(cat "${PKGFILE}-new") " aptcache show apt newstuff - msgmsg "Testcase: apply with one patch and SHA1 only: $*" + msgmsg "Testcase: SHA1-only patches are not used: $*" find aptarchive -name 'Packages*' -type f -delete cp "${PKGFILE}-new" aptarchive/Packages compressfile 'aptarchive/Packages' @@ -114,13 +110,35 @@ SHA1-History: $(sha1sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE") SHA1-Patches: 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28 - $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX" - if $DOWNLOADHASH; then - echo "SHA1-Download: + $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE") +SHA1-Download: 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz - $(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX" - fi + $(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX" + generatereleasefiles '+1hour' + signreleasefiles + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + wasmergeused "$@" -o test::cannot-use-pdiff=1 + testnopackage oldstuff + testsuccessequal "$(cat "${PKGFILE}-new") +" aptcache show apt newstuff + msgmsg "Testcase: no download-hashes patches are not used: $*" + find aptarchive -name 'Packages*' -type f -delete + cp "${PKGFILE}-new" aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true + cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz" + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new") +SHA256-History: + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 + $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE") +SHA256-Patches: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 + $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX" generatereleasefiles '+1hour' signreleasefiles rm -rf rootdir/var/lib/apt/lists @@ -157,13 +175,11 @@ SHA256-History: SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE") - $(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")" > "$PATCHINDEX" - if $DOWNLOADHASH; then - echo "SHA256-Download: + $(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}") +SHA256-Download: d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz") - $(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" >> "$PATCHINDEX" - fi + $(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" > "$PATCHINDEX" generatereleasefiles '+2hour' signreleasefiles @@ -205,12 +221,10 @@ SHA256-History: $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE") SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 - $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > $PATCHINDEX - if $DOWNLOADHASH; then - echo "SHA256-Download: + $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE") +SHA256-Download: d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz - $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> $PATCHINDEX - fi + $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX" # needs to look like a valid command, otherwise the parser will fail before hashes are checked echo '1d' > "$PATCHFILE" cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz" @@ -236,22 +250,16 @@ SHA256-Patches: diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz" PATCHINDEX='aptarchive/Packages.diff/Index' - BIGSIZE="$(stat -c%s "$PATCHFILE")" - if ! $DOWNLOADHASH; then - BIGSIZE="${BIGSIZE}000" - fi echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new") SHA256-History: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE") SHA256-Patches: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 - $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $BIGSIZE $(basename "$PATCHFILE")" > "$PATCHINDEX" - if $DOWNLOADHASH; then - echo "SHA256-Download: + $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE") +SHA256-Download: d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz - $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX" - fi + $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX" generatereleasefiles '+1hour' signreleasefiles testsuccess apt update -o Debug::pkgAcquire::Diffs=1 "$@" @@ -266,9 +274,6 @@ Debug::Acquire::Transaction "true"; Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf testcase() { - testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@" - testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@" - testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@" testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@" testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0 "$@" -- cgit v1.2.3