summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--apt-pkg/acquire-item.cc72
-rw-r--r--apt-pkg/acquire-item.h11
-rw-r--r--apt-pkg/contrib/hashes.cc11
-rw-r--r--apt-pkg/contrib/hashes.h7
-rw-r--r--apt-pkg/indexrecords.cc4
-rw-r--r--methods/rred.cc2
-rwxr-xr-xtest/integration/test-pdiff-usage46
7 files changed, 125 insertions, 28 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 7b69ee993..a3f47242f 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -152,14 +152,18 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const
APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
{
- /* FIXME: We have only hashes for uncompressed pdiffs.
- rred uncompresses them on the fly while parsing.
- In StateFetchDiff state we also uncompress on the fly for hash check.
- Hashes are checked while searching for (next) patch to apply. */
+ /* We don't always have the diff of the downloaded pdiff file.
+ What we have for sure is hashes for the uncompressed file,
+ but rred uncompresses them on the fly while parsing, so not handled here.
+ Hashes are (also) checked while searching for (next) patch to apply. */
+ if (State == StateFetchDiff)
+ return available_patches[0].download_hashes.empty() == false;
return false;
}
HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
{
+ if (State == StateFetchDiff)
+ return available_patches[0].download_hashes;
return HashStringList();
}
@@ -168,11 +172,15 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
/* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
we can check the rred result after all patches are applied as
we know the expected result rather than potentially apply more patches */
+ if (State == StateFetchDiff)
+ return patch.download_hashes.empty() == false;
return State == StateApplyDiff;
}
HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
{
- if (State == StateApplyDiff)
+ if (State == StateFetchDiff)
+ return patch.download_hashes;
+ else if (State == StateApplyDiff)
return GetExpectedHashesFor(Target->MetaKey);
return HashStringList();
}
@@ -1618,7 +1626,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
std::vector<DiffInfo>::iterator cur = available_patches.begin();
for (; cur != available_patches.end(); ++cur)
{
- if (cur->file != filename || unlikely(cur->result_size != size))
+ if (cur->file != filename)
continue;
cur->result_hashes.push_back(HashString(*type, hash));
break;
@@ -1630,8 +1638,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
DiffInfo next;
next.file = filename;
next.result_hashes.push_back(HashString(*type, hash));
- next.result_size = size;
- next.patch_size = 0;
+ next.result_hashes.FileSize(size);
available_patches.push_back(next);
}
else
@@ -1679,10 +1686,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
{
if (cur->file != filename)
continue;
- if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
- continue;
+ if (cur->patch_hashes.empty())
+ cur->patch_hashes.FileSize(size);
cur->patch_hashes.push_back(HashString(*type, hash));
- cur->patch_size = size;
break;
}
if (cur != available_patches.end())
@@ -1694,6 +1700,48 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
}
}
+ for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+ {
+ std::string tagname = *type;
+ tagname.append("-Download");
+ std::string const tmp = Tags.FindS(tagname.c_str());
+ if (tmp.empty() == true)
+ continue;
+
+ string hash, filename;
+ unsigned long long size;
+ std::stringstream ss(tmp);
+
+ // FIXME: all of pdiff supports only .gz compressed patches
+ while (ss >> hash >> size >> filename)
+ {
+ if (unlikely(hash.empty() == true || filename.empty() == true))
+ continue;
+ if (unlikely(APT::String::Endswith(filename, ".gz") == false))
+ continue;
+ filename.erase(filename.length() - 3);
+
+ // see if we have a record for this file already
+ std::vector<DiffInfo>::iterator cur = available_patches.begin();
+ for (; cur != available_patches.end(); ++cur)
+ {
+ if (cur->file != filename)
+ continue;
+ if (cur->download_hashes.empty())
+ cur->download_hashes.FileSize(size);
+ cur->download_hashes.push_back(HashString(*type, hash));
+ break;
+ }
+ if (cur != available_patches.end())
+ continue;
+ if (Debug == true)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+ << " wasn't in the list for the first parsed hash! (download)" << std::endl;
+ break;
+ }
+ }
+
+
bool foundStart = false;
for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
cur != available_patches.end(); ++cur)
@@ -1729,7 +1777,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
unsigned long long patchesSize = 0;
for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
cur != available_patches.end(); ++cur)
- patchesSize += cur->patch_size;
+ patchesSize += cur->patch_hashes.FileSize();
unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
{
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index f24af1aec..910e4131b 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -705,17 +705,14 @@ struct APT_HIDDEN DiffInfo { /*{{{*/
/** The filename of the diff. */
std::string file;
- /** The hashes of the diff */
+ /** The hashes of the file after the diff is applied */
HashStringList result_hashes;
- /** The hashes of the file after the diff is applied */
+ /** The hashes of the diff */
HashStringList patch_hashes;
- /** The size of the file after the diff is applied */
- unsigned long long result_size;
-
- /** The size of the diff itself */
- unsigned long long patch_size;
+ /** The hashes of the compressed diff */
+ HashStringList download_hashes;
};
/*}}}*/
/** \brief An item that is responsible for fetching client-merge patches {{{
diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 11a7e479b..46cf0ba08 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -188,6 +188,13 @@ unsigned long long HashStringList::FileSize() const /*{{{*/
return strtoull(hv.c_str(), NULL, 10);
}
/*}}}*/
+bool HashStringList::FileSize(unsigned long long const Size) /*{{{*/
+{
+ std::string size;
+ strprintf(size, "%llu", Size);
+ return push_back(HashString("Checksum-FileSize", size));
+}
+ /*}}}*/
bool HashStringList::supported(char const * const type) /*{{{*/
{
for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t)
@@ -361,9 +368,7 @@ APT_IGNORE_DEPRECATED_PUSH
if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
APT_IGNORE_DEPRECATED_POP
- std::string SizeStr;
- strprintf(SizeStr, "%llu", d->FileSize);
- hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+ hashes.FileSize(d->FileSize);
return hashes;
}
APT_IGNORE_DEPRECATED_PUSH
diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index 176ce4faa..e8d84da9e 100644
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -96,6 +96,13 @@ class HashStringList
*/
unsigned long long FileSize() const;
+ /** sets the filesize hash
+ *
+ * @param Size of the file
+ * @return @see #push_back
+ */
+ bool FileSize(unsigned long long const Size);
+
/** check if the given hash type is supported
*
* @param type to check
diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index c26868cac..7e6da9558 100644
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -121,9 +121,7 @@ bool indexRecords::Load(const string Filename) /*{{{*/
indexRecords::checkSum *Sum = new indexRecords::checkSum;
Sum->MetaKeyFilename = Name;
Sum->Size = Size;
- std::string SizeStr;
- strprintf(SizeStr, "%llu", Size);
- Sum->Hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+ Sum->Hashes.FileSize(Size);
APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);)
Entries[Name] = Sum;
}
diff --git a/methods/rred.cc b/methods/rred.cc
index 81ecf8553..12cf2b4a5 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -627,7 +627,7 @@ class RredMethod : public pkgAcqMethod {
p.Close();
HashStringList const hsl = patch_hash.GetHashStringList();
if (hsl != I->ExpectedHashes)
- return _error->Error("Patch %s doesn't have the expected hashsum", patch_name.c_str());
+ return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
}
if (Debug == true)
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 7a9f6496b..3295d5497 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -42,6 +42,8 @@ wasmergeused() {
testrun() {
msgmsg "Testcase: setup the base with: $*"
+ local DOWNLOADHASH=true
+ if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
find aptarchive -name 'Packages*' -type f -delete
cp ${PKGFILE} aptarchive/Packages
compressfile 'aptarchive/Packages'
@@ -76,6 +78,15 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+ if $DOWNLOADHASH; then
+ echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+ fi
+
generatereleasefiles '+1hour'
signreleasefiles
find aptarchive -name 'Packages*' -type f -delete
@@ -131,6 +142,17 @@ SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)
$(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX
+ if $DOWNLOADHASH; then
+ echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha1sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha256sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)" >> $PATCHINDEX
+ fi
+
generatereleasefiles '+2hour'
signreleasefiles
cp -a aptarchive/Packages Packages-future
@@ -150,6 +172,7 @@ SHA256-Patches:
mkdir -p aptarchive/Packages.diff
PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
+ cat $PATCHFILE | gzip > ${PATCHFILE}.gz
PATCHINDEX='aptarchive/Packages.diff/Index'
echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
SHA1-History:
@@ -165,14 +188,22 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+ if $DOWNLOADHASH; then
+ echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+ fi
# needs to look like a valid command, otherwise the parser will fail before hashes are checked
- echo '1d' >> $PATCHFILE
+ echo '1d' > $PATCHFILE
cat $PATCHFILE | gzip > ${PATCHFILE}.gz
generatereleasefiles '+1hour'
signreleasefiles
testsuccess aptget update "$@"
cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
- testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output
+ testsuccess grep 'Hash Sum mismatch' rootdir/tmp/aptgetupdate.output
testnopackage oldstuff
testsuccessequal "$(cat ${PKGFILE}-new)
" aptcache show apt newstuff
@@ -201,6 +232,14 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX
+ if $DOWNLOADHASH; then
+ echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+ fi
generatereleasefiles '+1hour'
signreleasefiles
#find aptarchive -name 'Packages*' -type f -delete
@@ -215,6 +254,9 @@ echo 'Debug::pkgAcquire::Diffs "true";
Debug::Acquire::Transaction "true";
Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
+testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
+testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
+
testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-06 19:56:14 +0000