diff options
| -rw-r--r-- | apt-pkg/acquire-item.cc | 88 | ||||
| -rw-r--r-- | apt-pkg/acquire-item.h | 147 | ||||
| -rw-r--r-- | ftparchive/writer.cc | 1 | ||||
| -rwxr-xr-x | test/integration/test-pdiff-usage | 29 |
4 files changed, 151 insertions, 114 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b8786e34b..7abb7b206 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -451,7 +451,11 @@ std::string pkgAcquire::Item::GetFinalFilename() const } std::string pkgAcqDiffIndex::GetFinalFilename() const { - return GetFinalFileNameFromURI(GetDiffIndexURI(Target)); + std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target)); + // we don't want recompress, so lets keep whatever we got + if (CurrentCompressionExtension == "uncompressed") + return FinalFile; + return FinalFile + "." + CurrentCompressionExtension; } std::string pkgAcqIndex::GetFinalFilename() const { @@ -488,7 +492,10 @@ std::string pkgAcqIndex::GetMetaKey() const } std::string pkgAcqDiffIndex::GetMetaKey() const { - return GetDiffIndexFileName(Target.MetaKey); + auto const metakey = GetDiffIndexFileName(Target.MetaKey); + if (CurrentCompressionExtension == "uncompressed") + return metakey; + return metakey + "." + CurrentCompressionExtension; } /*}}}*/ //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/ @@ -1971,44 +1978,33 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {} pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) - : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL) + : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL) { // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire ExpectedAdditionalItems = 40; - Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Desc.Owner = this; - Desc.Description = GetDiffIndexFileName(Target.Description); - Desc.ShortDesc = Target.ShortDesc; - Desc.URI = GetDiffIndexURI(Target); - - DestFile = GetPartialFileNameFromURI(Desc.URI); + CompressionExtensions.clear(); + { + std::vector<std::string> types = APT::Configuration::getCompressionTypes(); + if (types.empty() == false) + { + std::ostringstream os; + std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) { + if (type == "uncompressed") + return true; + return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type); + }); + os << *types.rbegin(); + CompressionExtensions = os.str(); + } + } + if (Target.Option(IndexTarget::COMPRESSIONTYPES).find("by-hash") != std::string::npos) + CompressionExtensions = "by-hash " + CompressionExtensions; + Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc); if(Debug) std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; - - QueueURI(Desc); -} - /*}}}*/ -// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ -// --------------------------------------------------------------------- -/* The only header we use is the last-modified header. */ -string pkgAcqDiffIndex::Custom600Headers() const -{ - if (TransactionManager->LastMetaIndexParser != NULL) - return "\nIndex-File: true"; - - string const Final = GetFinalFilename(); - - if(Debug) - std::clog << "Custom600Header-IMS: " << Final << std::endl; - - struct stat Buf; - if (stat(Final.c_str(),&Buf) != 0) - return "\nIndex-File: true"; - - return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false); } /*}}}*/ void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/ @@ -2039,7 +2035,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile << std::endl; - FileFd Fd(IndexDiffFile,FileFd::ReadOnly); + FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension); pkgTagFile TF(&Fd); if (Fd.IsOpen() == false || Fd.Failed()) return false; @@ -2411,7 +2407,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ /*}}}*/ void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ { - pkgAcqBaseIndex::Failed(Message,Cnf); + if (CommonFailed(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Message, Cnf)) + return; + Status = StatDone; ExpectedAdditionalItems = 0; @@ -2829,10 +2827,12 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {} // AcqIndex::AcqIndex - Constructor /*{{{*/ pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, - IndexTarget const &Target) + IndexTarget const &Target, bool const Derived) : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD), CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES)) { + if (Derived) + return; Init(Target.URI, Target.Description, Target.ShortDesc); if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -2938,7 +2938,8 @@ string pkgAcqIndex::Custom600Headers() const } /*}}}*/ // AcqIndex::Failed - getting the indexfile failed /*{{{*/ -void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, std::string const TargetDesc, + std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) { pkgAcqBaseIndex::Failed(Message,Cnf); @@ -2950,10 +2951,9 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * CompressionExtensions = "by-hash " + CompressionExtensions; else CompressionExtensions = CurrentCompressionExtension + ' ' + CompressionExtensions; - Desc.Description = Target.Description; - Init(Target.URI, Desc.Description, Desc.ShortDesc); + Init(TargetURI, TargetDesc, Desc.ShortDesc); Status = StatIdle; - return; + return true; } // authorisation matches will not be fixed by other compression types @@ -2961,11 +2961,17 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * { if (CompressionExtensions.empty() == false) { - Init(Target.URI, Desc.Description, Desc.ShortDesc); + Init(TargetURI, Desc.Description, Desc.ShortDesc); Status = StatIdle; - return; + return true; } } + return false; +} +void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + if (CommonFailed(Target.URI, Target.Description, Message, Cnf)) + return; if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD) Status = StatDone; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index e6e5ea12b..71a11bcde 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -611,6 +611,79 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem virtual ~pkgAcqBaseIndex(); }; /*}}}*/ +/** \brief An acquire item that is responsible for fetching an index {{{ + * file (e.g., Packages or Sources). + * + * \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans + * + * \todo Why does pkgAcqIndex have protected members? + */ +class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex +{ + void * const d; + + protected: + + /** \brief The stages the method goes through + * + * The method first downloads the indexfile, then its decompressed (or + * copied) and verified + */ + enum AllStages { + STAGE_DOWNLOAD, + STAGE_DECOMPRESS_AND_VERIFY, + }; + AllStages Stage; + + /** \brief Handle what needs to be done when the download is done */ + void StageDownloadDone(std::string const &Message); + + /** \brief Handle what needs to be done when the decompression/copy is + * done + */ + void StageDecompressDone(); + + /** \brief If \b set, this partially downloaded file will be + * removed when the download completes. + */ + std::string EraseFileName; + + /** \brief The compression-related file extensions that are being + * added to the downloaded file one by one if first fails (e.g., "gz bz2"). + */ + std::string CompressionExtensions; + + /** \brief The actual compression extension currently used */ + std::string CurrentCompressionExtension; + + /** \brief Do the changes needed to fetch via AptByHash (if needed) */ + void InitByHashIfNeeded(); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + + public: + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + virtual std::string GetMetaKey() const APT_OVERRIDE; + + pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, bool const Derived = false) APT_NONNULL(2, 3); + virtual ~pkgAcqIndex(); + + protected: + APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc, + std::string const &ShortDesc); + APT_HIDDEN bool CommonFailed(std::string const &TargetURI, std::string const TargetDesc, + std::string const &Message, pkgAcquire::MethodConfig const * const Cnf); +}; + /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ * package list diffs and starting the package list's download. * @@ -620,7 +693,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem * * \sa pkgAcqIndexDiffs, pkgAcqIndex */ -class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex +class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqIndex { void * const d; std::vector<pkgAcqIndexMergeDiffs*> * diffs; @@ -646,7 +719,6 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";}; - virtual std::string Custom600Headers() const APT_OVERRIDE; virtual std::string GetMetaKey() const APT_OVERRIDE; /** \brief Parse the Index file for a set of Packages diffs. @@ -887,77 +959,6 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex virtual ~pkgAcqIndexDiffs(); }; /*}}}*/ -/** \brief An acquire item that is responsible for fetching an index {{{ - * file (e.g., Packages or Sources). - * - * \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans - * - * \todo Why does pkgAcqIndex have protected members? - */ -class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex -{ - void * const d; - - protected: - - /** \brief The stages the method goes through - * - * The method first downloads the indexfile, then its decompressed (or - * copied) and verified - */ - enum AllStages { - STAGE_DOWNLOAD, - STAGE_DECOMPRESS_AND_VERIFY, - }; - AllStages Stage; - - /** \brief Handle what needs to be done when the download is done */ - void StageDownloadDone(std::string const &Message); - - /** \brief Handle what needs to be done when the decompression/copy is - * done - */ - void StageDecompressDone(); - - /** \brief If \b set, this partially downloaded file will be - * removed when the download completes. - */ - std::string EraseFileName; - - /** \brief The compression-related file extensions that are being - * added to the downloaded file one by one if first fails (e.g., "gz bz2"). - */ - std::string CompressionExtensions; - - /** \brief The actual compression extension currently used */ - std::string CurrentCompressionExtension; - - /** \brief Do the changes needed to fetch via AptByHash (if needed) */ - void InitByHashIfNeeded(); - - /** \brief Get the full pathname of the final file for the current URI */ - virtual std::string GetFinalFilename() const APT_OVERRIDE; - - virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; - - public: - // Specialized action members - virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; - virtual void Done(std::string const &Message, HashStringList const &Hashes, - pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; - virtual std::string Custom600Headers() const APT_OVERRIDE; - virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; - virtual std::string GetMetaKey() const APT_OVERRIDE; - - pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, - IndexTarget const &Target) APT_NONNULL(2, 3); - virtual ~pkgAcqIndex(); - - private: - APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc, - std::string const &ShortDesc); -}; - /*}}}*/ /** \brief An item that is responsible for fetching a package file. {{{ * * If the package file already exists in the cache, nothing will be diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index c34a04d1a..018cf0052 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -992,6 +992,7 @@ ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : AddPattern("Release"); AddPattern("Contents-*"); AddPattern("Index"); + AddPattern("Index.*"); AddPattern("icons-*.tar"); AddPattern("icons-*.tar.*"); AddPattern("Components-*.yml"); diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 39d847203..59a3fd8af 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -35,6 +35,7 @@ wasmergeused() { fi testsuccess apt update "$@" + #apt update "$@" 2>&1 | tee rootdir/tmp/testsuccess.output msgtest 'No intermediate patch files' 'still exist' local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')" @@ -116,6 +117,34 @@ SHA256-Download: testsuccessequal "$(cat "${PKGFILE}-new") " aptcache show apt newstuff + msgmsg "Testcase: apply with compressed Index and one patch: $*" + cp "${PKGFILE}-new" aptarchive/Packages + compressfile 'aptarchive/Packages' + cat "$PATCHINDEX" | gzip > "${PATCHINDEX}.gz" + generatereleasefiles '+1hour' + signreleasefiles + find aptarchive -name 'Packages*' -type f -delete + rm "$PATCHINDEX" + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + wasmergeused "$@" + testnopackage oldstuff + testsuccessequal "$(cat "${PKGFILE}-new") +" aptcache show apt newstuff + + msgmsg "Testcase: apply with by-hash compressed Index and one patch: $*" + local NORMAL="$(readlink -f "./aptarchive/Packages.diff")" + local BYHASH="${NORMAL}/by-hash/SHA256" + mkdir -p "${BYHASH}" + find "${NORMAL}/" -maxdepth 1 -name "Index*" -exec mv '{}' "$BYHASH" \; + ln -s "${BYHASH}/Index.gz" "${BYHASH}/$(sha256sum "${BYHASH}/Index.gz" | cut -f1 -d' ')" + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + wasmergeused "$@" -o Acquire::By-Hash=force + testnopackage oldstuff + testsuccessequal "$(cat "${PKGFILE}-new") +" aptcache show apt newstuff + msgmsg "Testcase: SHA1-only patches are not used: $*" find aptarchive -name 'Packages*' -type f -delete cp "${PKGFILE}-new" aptarchive/Packages |