From 50d98a1be2e15f44dea23a5d3840c79366a42fe0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 22 May 2014 10:49:35 +0200 Subject: Implement simple by-hash for apt update This implements a apt update schema that get the indexfiles by the hash instead of the name. The rational is that updates to the archive servers/mirrors are not atomic so the client may have the previous version of the Release file when the server updates to a new Release file and new Packages/Sources/Translations indexes. By keeping the files around by their hash we can still get the previous indexfile without a hashsum mismatch. Enable with APT::Acquire::By-Hash=1 --- apt-pkg/acquire-item.cc | 21 +++++++++++++++++ apt-pkg/indexrecords.cc | 22 ++++++++++-------- test/integration/test-apt-by-hash-update | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 9 deletions(-) create mode 100755 test/integration/test-apt-by-hash-update diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 99013d649..1f3d83941 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1020,6 +1020,27 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S FileSize = Record->Size; } + // do the request by-hash + if(_config->FindB("APT::Acquire::By-Hash", false) == true && + MetaIndexParser) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + { + // FIXME: make the hash used a config option or read from release file + const HashString *TargetHash = Record->Hashes.find("SHA256"); + std::string ByHash = "/by-hash/" + TargetHash->HashValue(); + size_t trailing_slash = Desc.URI.find_last_of("/"); + Desc.URI = Desc.URI.replace(trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); + std::cerr << Desc.URI << std::endl; + } else { + _error->Warning("By-Hash requested but can not find record for %s", + MetaKey.c_str()); + } + } + Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index 5dcaadd76..122194e94 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -90,8 +90,8 @@ bool indexRecords::Load(const string Filename) /*{{{*/ Suite = Section.FindS("Suite"); Dist = Section.FindS("Codename"); - int i; - for (i=0;HashString::SupportedHashes()[i] != NULL; i++) + bool FoundHashSum = false; + for (int i=0;HashString::SupportedHashes()[i] != NULL; i++) { if (!Section.Find(HashString::SupportedHashes()[i], Start, End)) continue; @@ -103,16 +103,20 @@ bool indexRecords::Load(const string Filename) /*{{{*/ { if (!parseSumData(Start, End, Name, Hash, Size)) return false; - indexRecords::checkSum *Sum = new indexRecords::checkSum; - Sum->MetaKeyFilename = Name; - Sum->Hashes.push_back(HashString(HashString::SupportedHashes()[i],Hash)); - Sum->Size = Size; - Entries[Name] = Sum; + + if (Entries.find(Name) == Entries.end()) + { + indexRecords::checkSum *Sum = new indexRecords::checkSum; + Sum->MetaKeyFilename = Name; + Sum->Size = Size; + Entries[Name] = Sum; + } + Entries[Name]->Hashes.push_back(HashString(HashString::SupportedHashes()[i],Hash)); + FoundHashSum = true; } - break; } - if(HashString::SupportedHashes()[i] == NULL) + if(FoundHashSum == false) { strprintf(ErrorText, _("No Hash entry in Release file %s"), Filename.c_str()); return false; diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update new file mode 100755 index 000000000..6b3032ad6 --- /dev/null +++ b/test/integration/test-apt-by-hash-update @@ -0,0 +1,39 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +insertpackage 'unstable' 'foo' 'all' '1.0' + +setupaptarchive --no-update + +APTARCHIVE=$(readlink -f ./aptarchive) + +# make Packages *only* accessable by-hash for this test +mkdir -p aptarchive/dists/unstable/main/binary-i386/by-hash +(cd aptarchive/dists/unstable/main/binary-i386/by-hash && + mv ../Packages* . && + ln -s Packages.gz $(sha256sum Packages.gz|cut -f1 -d' ') ) + +# add sources +mkdir -p aptarchive/dists/unstable/main/source/by-hash +(cd aptarchive/dists/unstable/main/source/by-hash && + ln -s ../Sources.gz $(sha256sum ../Sources.gz|cut -f1 -d' ') +) + + +# ensure we do not know about "foo" +testequal "Reading package lists... +Building dependency tree... +E: Unable to locate package foo" aptget install -q -s foo + +# ensure we can apt-get update by hash +testsuccess aptget update -o APT::Acquire::By-Hash=1 + +# ensure it keeps working +testequal "Inst foo (1.0 unstable [all]) +Conf foo (1.0 unstable [all])" aptget install -qq -s foo \ No newline at end of file -- cgit v1.2.3 From ffa8189668e54572318d1834881e470c238f3642 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 22 May 2014 11:09:11 +0200 Subject: add TODO --- apt-pkg/acquire-item.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 1f3d83941..7fc556b7c 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1021,6 +1021,11 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S } // do the request by-hash + // TODO: + // - add Support-By-Hash hashtype into Release time so that we + // know we can use it + // - (maybe?) add support for by-hash into the sources.list as flag + // - make apt-ftparchive generate the hashes (and expire?) if(_config->FindB("APT::Acquire::By-Hash", false) == true && MetaIndexParser) { -- cgit v1.2.3 From a2fdb57ff93c1b1f35b796c3c99878ec3ae54a06 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 22 May 2014 17:36:09 +0200 Subject: Add APT::Acquire::$(host)::By-Hash=1 knob, add Acquire-By-Hash to Release file The by-hash can be configured on a per-hostname basis and a Release file can indicate that it has by-hash support via a new flag. The location of the hash now matches the AptByHash spec --- apt-pkg/acquire-item.cc | 46 ++++++++++++++++---------------- apt-pkg/indexrecords.cc | 9 ++++++- apt-pkg/indexrecords.h | 4 +++ apt-pkg/tagfile.cc | 11 ++++++++ apt-pkg/tagfile.h | 3 ++- test/integration/test-apt-by-hash-update | 28 ++++++++++++------- 6 files changed, 67 insertions(+), 34 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7fc556b7c..b7ca82c0c 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1018,31 +1018,31 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); if(Record) FileSize = Record->Size; - } - // do the request by-hash - // TODO: - // - add Support-By-Hash hashtype into Release time so that we - // know we can use it - // - (maybe?) add support for by-hash into the sources.list as flag - // - make apt-ftparchive generate the hashes (and expire?) - if(_config->FindB("APT::Acquire::By-Hash", false) == true && - MetaIndexParser) - { - indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); - if(Record) + // TODO: + // - (maybe?) add support for by-hash into the sources.list as flag + // - make apt-ftparchive generate the hashes (and expire?) + // do the request by-hash + std::string HostKnob = "APT::Acquire::" + ::URI(URI).Host + "::By-Hash"; + if(_config->FindB("APT::Acquire::By-Hash", false) == true || + _config->FindB(HostKnob, false) == true || + MetaIndexParser->GetSupportsAcquireByHash()) { - // FIXME: make the hash used a config option or read from release file - const HashString *TargetHash = Record->Hashes.find("SHA256"); - std::string ByHash = "/by-hash/" + TargetHash->HashValue(); - size_t trailing_slash = Desc.URI.find_last_of("/"); - Desc.URI = Desc.URI.replace(trailing_slash, - Desc.URI.substr(trailing_slash+1).size()+1, - ByHash); - std::cerr << Desc.URI << std::endl; - } else { - _error->Warning("By-Hash requested but can not find record for %s", - MetaKey.c_str()); + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + { + // FIXME: should we really use the best hash here? or a fixed one? + const HashString *TargetHash = Record->Hashes.find(""); + std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + size_t trailing_slash = Desc.URI.find_last_of("/"); + Desc.URI = Desc.URI.replace( + trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); + } else { + _error->Warning("By-Hash requested but can not find record for %s", + MetaKey.c_str()); + } } } diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index 122194e94..5b53e796d 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -37,6 +37,11 @@ APT_PURE string indexRecords::GetSuite() const return this->Suite; } +APT_PURE bool indexRecords::GetSupportsAcquireByHash() const +{ + return this->SupportsAcquireByHash; +} + APT_PURE bool indexRecords::CheckDist(const string MaybeDist) const { return (this->Dist == MaybeDist @@ -86,6 +91,8 @@ bool indexRecords::Load(const string Filename) /*{{{*/ strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str()); return false; } + // FIXME: find better tag name + SupportsAcquireByHash = Section.FindB("Acquire-By-Hash", false); Suite = Section.FindS("Suite"); Dist = Section.FindS("Codename"); @@ -243,6 +250,6 @@ indexRecords::indexRecords() } indexRecords::indexRecords(const string ExpectedDist) : - ExpectedDist(ExpectedDist), ValidUntil(0) + ExpectedDist(ExpectedDist), ValidUntil(0), SupportsAcquireByHash(false) { } diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h index 14b03c4d5..bb0fd5564 100644 --- a/apt-pkg/indexrecords.h +++ b/apt-pkg/indexrecords.h @@ -26,12 +26,15 @@ class indexRecords public: struct checkSum; std::string ErrorText; + // dpointer (for later9 + void * d; protected: std::string Dist; std::string Suite; std::string ExpectedDist; time_t ValidUntil; + bool SupportsAcquireByHash; std::map Entries; @@ -49,6 +52,7 @@ class indexRecords virtual bool Load(std::string Filename); std::string GetDist() const; std::string GetSuite() const; + bool GetSupportsAcquireByHash() const; time_t GetValidUntil() const; virtual bool CheckDist(const std::string MaybeDist) const; std::string GetExpectedDist() const; diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 52f4da2d5..7085e7d69 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -518,6 +518,17 @@ unsigned long long pkgTagSection::FindULL(const char *Tag, unsigned long long co return Result; } /*}}}*/ +// TagSection::FindB - Find boolean value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgTagSection::FindB(const char *Tag, bool const &Default) const +{ + const char *Start, *Stop; + if (Find(Tag, Start, Stop) == false) + return Default; + return StringToBool(string(Start, Stop)); +} + /*}}}*/ // TagSection::FindFlag - Locate a yes/no type flag /*{{{*/ // --------------------------------------------------------------------- /* The bits marked in Flag are masked on/off in Flags */ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index b0cfab759..db43bfbf9 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -62,7 +62,8 @@ class pkgTagSection bool Find(const char *Tag,const char *&Start, const char *&End) const; bool Find(const char *Tag,unsigned int &Pos) const; std::string FindS(const char *Tag) const; - signed int FindI(const char *Tag,signed long Default = 0) const ; + signed int FindI(const char *Tag,signed long Default = 0) const; + bool FindB(const char *Tag, bool const &Default = false) const; unsigned long long FindULL(const char *Tag, unsigned long long const &Default = 0) const; bool FindFlag(const char *Tag,unsigned long &Flags, unsigned long Flag) const; diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 6b3032ad6..23282bf86 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -14,17 +14,19 @@ setupaptarchive --no-update APTARCHIVE=$(readlink -f ./aptarchive) # make Packages *only* accessable by-hash for this test -mkdir -p aptarchive/dists/unstable/main/binary-i386/by-hash -(cd aptarchive/dists/unstable/main/binary-i386/by-hash && - mv ../Packages* . && - ln -s Packages.gz $(sha256sum Packages.gz|cut -f1 -d' ') ) +mkdir -p aptarchive/dists/unstable/main/binary-i386/by-hash/SHA512 +(cd aptarchive/dists/unstable/main/binary-i386/by-hash/SHA512 && + mv ../../Packages* . && + ln -s Packages.gz $(sha512sum Packages.gz|cut -f1 -d' ') ) # add sources -mkdir -p aptarchive/dists/unstable/main/source/by-hash -(cd aptarchive/dists/unstable/main/source/by-hash && - ln -s ../Sources.gz $(sha256sum ../Sources.gz|cut -f1 -d' ') +mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512 +(cd aptarchive/dists/unstable/main/source/by-hash/SHA512 && + ln -s ../../Sources.gz $(sha512sum ../../Sources.gz|cut -f1 -d' ') ) +# we moved the Packages file away, normal update won't work +testfailure aptget upate # ensure we do not know about "foo" testequal "Reading package lists... @@ -34,6 +36,14 @@ E: Unable to locate package foo" aptget install -q -s foo # ensure we can apt-get update by hash testsuccess aptget update -o APT::Acquire::By-Hash=1 -# ensure it keeps working +# ensure it works testequal "Inst foo (1.0 unstable [all]) -Conf foo (1.0 unstable [all])" aptget install -qq -s foo \ No newline at end of file +Conf foo (1.0 unstable [all])" aptget install -qq -s foo + +# add magic string to Release file ... +MAGIC="Acquire-By-Hash: true" +sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release +signreleasefiles +# ... and verify that it fetches by hash now +testsuccess aptget update + -- cgit v1.2.3 From 59194959326dbf114a5c894e4279c04844b4a793 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 22 May 2014 17:47:22 +0200 Subject: move ByHash into its own function --- apt-pkg/acquire-item.cc | 60 ++++++++++++++++++++++++++++--------------------- apt-pkg/acquire-item.h | 3 +++ 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b7ca82c0c..e111ed4ca 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1018,32 +1018,8 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); if(Record) FileSize = Record->Size; - - // TODO: - // - (maybe?) add support for by-hash into the sources.list as flag - // - make apt-ftparchive generate the hashes (and expire?) - // do the request by-hash - std::string HostKnob = "APT::Acquire::" + ::URI(URI).Host + "::By-Hash"; - if(_config->FindB("APT::Acquire::By-Hash", false) == true || - _config->FindB(HostKnob, false) == true || - MetaIndexParser->GetSupportsAcquireByHash()) - { - indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); - if(Record) - { - // FIXME: should we really use the best hash here? or a fixed one? - const HashString *TargetHash = Record->Hashes.find(""); - std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); - size_t trailing_slash = Desc.URI.find_last_of("/"); - Desc.URI = Desc.URI.replace( - trailing_slash, - Desc.URI.substr(trailing_slash+1).size()+1, - ByHash); - } else { - _error->Warning("By-Hash requested but can not find record for %s", - MetaKey.c_str()); - } - } + + InitByHashIfNeeded(MetaKey); } Desc.Description = URIDesc; @@ -1053,6 +1029,38 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S QueueURI(Desc); } /*}}}*/ +// AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) +{ + // TODO: + // - (maybe?) add support for by-hash into the sources.list as flag + // - make apt-ftparchive generate the hashes (and expire?) + std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash"; + if(_config->FindB("APT::Acquire::By-Hash", false) == true || + _config->FindB(HostKnob, false) == true || + MetaIndexParser->GetSupportsAcquireByHash()) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + { + // FIXME: should we really use the best hash here? or a fixed one? + const HashString *TargetHash = Record->Hashes.find(""); + std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + size_t trailing_slash = Desc.URI.find_last_of("/"); + Desc.URI = Desc.URI.replace( + trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); + } else { + _error->Warning( + "Fetching ByHash requested but can not find record for %s", + MetaKey.c_str()); + } + } +} + /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3d863874c..cda92e84f 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -713,6 +713,9 @@ class pkgAcqIndex : public pkgAcqBaseIndex */ std::string CompressionExtension; + /** \brief Do the changes needed to fetch via AptByHash (if needed) */ + void InitByHashIfNeeded(const std::string MetaKey); + public: // Specialized action members -- cgit v1.2.3