diff options
Diffstat (limited to 'apt-pkg')
| -rw-r--r-- | apt-pkg/acquire-item.cc | 176 | ||||
| -rw-r--r-- | apt-pkg/acquire-item.h | 100 | ||||
| -rw-r--r-- | apt-pkg/contrib/fileutl.cc | 2 | ||||
| -rw-r--r-- | apt-pkg/sourcelist.cc | 119 | ||||
| -rw-r--r-- | apt-pkg/sourcelist.h | 10 |
5 files changed, 377 insertions, 30 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b76921312..1185908f3 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -369,10 +369,10 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, return; } - if(Debug) - std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " - << CurrentPackagesFile << std::endl; - + if(Debug) + std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): " + << CurrentPackagesFile << std::endl; + QueueURI(Desc); } @@ -398,8 +398,8 @@ string pkgAcqDiffIndex::Custom600Headers() bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ { if(Debug) - std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile - << std::endl; + std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile + << std::endl; pkgTagSection Tags; string ServerSha1; @@ -426,16 +426,18 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ SHA1.AddFD(fd); string const local_sha1 = SHA1.Result(); - if(local_sha1 == ServerSha1) + if(local_sha1 == ServerSha1) { - // we have the same sha1 as the server + // we have the same sha1 as the server so we are done here if(Debug) std::clog << "Package file is up-to-date" << std::endl; - // set found to true, this will queue a pkgAcqIndexDiffs with - // a empty availabe_patches - found = true; - } - else + // list cleanup needs to know that this file as well as the already + // present index is ours, so we create an empty diff to save it for us + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedHash, ServerSha1, available_patches); + return true; + } + else { if(Debug) std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl; @@ -460,7 +462,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (available_patches.empty() == false) { // patching with too many files is rather slow compared to a fast download - unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 20); + unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); if (fileLimit != 0 && fileLimit < available_patches.size()) { if (Debug) @@ -496,14 +498,37 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ } // we have something, queue the next diff - if(found) + if(found) { // queue the diffs string::size_type const last_space = Description.rfind(" "); if(last_space != string::npos) Description.erase(last_space, Description.size()-last_space); - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + + /* decide if we should download patches one by one or in one go: + The first is good if the server merges patches, but many don't so client + based merging can be attempt in which case the second is better. + "bad things" will happen if patches are merged on the server, + but client side merging is attempt as well */ + bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); + if (pdiff_merge == true) + { + // reprepro adds this flag if it has merged patches on the server + std::string const precedence = Tags.FindS("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); + } + + if (pdiff_merge == false) + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedHash, ServerSha1, available_patches); + else + { + std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size()); + for(size_t i = 0; i < available_patches.size(); ++i) + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash, + available_patches[i], diffs); + } + Complete = false; Status = StatDone; Dequeue(); @@ -752,6 +777,123 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Has } } /*}}}*/ +// AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ +pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + string const &URI, string const &URIDesc, + string const &ShortDesc, HashString const &ExpectedHash, + DiffInfo const &patch, + std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches) + : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), + patch(patch),allPatches(allPatches), State(StateFetchDiff) +{ + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + Desc.URI = string(RealURI) + ".diff/" + patch.file + ".gz"; + Desc.Description = Description + " " + patch.file + string(".pdiff"); + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI + ".diff/" + patch.file); + + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; + + QueueURI(Desc); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + + // check if we are the first to fail, otherwise we are done here + State = StateDoneDiff; + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State == StateErrorDiff) + return; + + // first failure means we should fallback + State = StateErrorDiff; + std::clog << "Falling back to normal index file aquire" << std::endl; + new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, + ExpectedHash); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + + if (State == StateFetchDiff) + { + // rred expects the patch as $FinalFile.ed.$patchname.gz + Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz"); + + // check if this is the last completed diff + State = StateDoneDiff; + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State != StateDoneDiff) + { + if(Debug) + std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl; + return; + } + + // this is the last completed diff, so we are ready to apply now + State = StateApplyDiff; + + if(Debug) + std::clog << "Sending to rred method: " << FinalFile << std::endl; + + Local = true; + Desc.URI = "rred:" + FinalFile; + QueueURI(Desc); + Mode = "rred"; + return; + } + // success in download/apply all diffs, clean up + else if (State == StateApplyDiff) + { + // see if we really got the expected file + if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + { + RenameOnError(HashSumMismatch); + return; + } + + // move the result into place + if(Debug) + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << FinalFile << std::endl; + Rename(DestFile, FinalFile); + chmod(FinalFile.c_str(), 0644); + + // otherwise lists cleanup will eat the file + DestFile = FinalFile; + + // all set and done + Complete = true; + if(Debug) + std::clog << "allDone: " << DestFile << "\n" << std::endl; + } +} + /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 6b4f73708..5a1c7979c 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -429,7 +429,105 @@ class pkgAcqDiffIndex : public pkgAcquire::Item std::string ShortDesc, HashString ExpectedHash); }; /*}}}*/ -/** \brief An item that is responsible for fetching all the patches {{{ +/** \brief An item that is responsible for fetching client-merge patches {{{ + * that need to be applied to a given package index file. + * + * Instead of downloading and applying each patch one by one like its + * sister #pkgAcqIndexDiffs this class will download all patches at once + * and call rred with all the patches downloaded once. Rred will then + * merge and apply them in one go, which should be a lot faster – but is + * incompatible with server-based merges of patches like reprepro can do. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class pkgAcqIndexMergeDiffs : public pkgAcquire::Item +{ + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** \brief description of the item that is currently being + * downloaded. + */ + pkgAcquire::ItemDesc Desc; + + /** \brief URI of the package index file that is being + * reconstructed. + */ + std::string RealURI; + + /** \brief HashSum of the package index file that is being + * reconstructed. + */ + HashString ExpectedHash; + + /** \brief description of the file being downloaded. */ + std::string Description; + + /** \brief information about the current patch */ + struct DiffInfo const patch; + + /** \brief list of all download items for the patches */ + std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches; + + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff, + + /** \brief the work with this diff is done */ + StateDoneDiff, + + /** \brief something bad happened and fallback was triggered */ + StateErrorDiff + } State; + + public: + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); + + virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual std::string DescURI() {return RealURI + "Index";}; + + /** \brief Create an index merge-diff item. + * + * \param Owner The pkgAcquire object that owns this item. + * + * \param URI The URI of the package index file being + * reconstructed. + * + * \param URIDesc A long description of this item. + * + * \param ShortDesc A brief description of this item. + * + * \param ExpectedHash The expected md5sum of the completely + * reconstructed package index file; the index file will be tested + * against this value when it is entirely reconstructed. + * + * \param patch contains infos about the patch this item is supposed + * to download which were read from the index + * + * \param allPatches contains all related items so that each item can + * check if it was the last one to complete the download step + */ + pkgAcqIndexMergeDiffs(pkgAcquire *Owner,std::string const &URI,std::string const &URIDesc, + std::string const &ShortDesc, HashString const &ExpectedHash, + DiffInfo const &patch, std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches); +}; + /*}}}*/ +/** \brief An item that is responsible for fetching server-merge patches {{{ * that need to be applied to a given package index file. * * After downloading and applying a single patch, this item will diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index efbf7aaf4..ffb8b4b53 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -465,7 +465,7 @@ std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> c const char *C = Ent->d_name; for (; *C != 0; ++C) if (isalpha(*C) == 0 && isdigit(*C) == 0 - && *C != '_' && *C != '-') { + && *C != '_' && *C != '-' && *C != ':') { // no required extension -> dot is a bad character if (*C == '.' && Ext.empty() == false) continue; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0fd237cad..09d8287a0 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -17,6 +17,7 @@ #include <apt-pkg/configuration.h> #include <apt-pkg/metaindex.h> #include <apt-pkg/indexfile.h> +#include <apt-pkg/tagfile.h> #include <fstream> @@ -70,6 +71,53 @@ bool pkgSourceList::Type::FixupURI(string &URI) const return true; } /*}}}*/ +bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, + pkgTagSection &Tags, + int i, + FileFd &Fd) +{ + map<string, string> Options; + + string URI = Tags.FindS("Uri"); + if (!FixupURI(URI)) + { + _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + return false; + } + + // Define external/internal options + const char* option_deb822[] = { + "Architectures", "Architectures-Add", "Architectures-Delete", "Trusted", + }; + const char* option_internal[] = { + "arch", "arch+", "arch-", "trusted", + }; + for (unsigned int j=0; j < sizeof(option_deb822)/sizeof(char*); j++) + if (Tags.Exists(option_deb822[j])) + Options[option_internal[j]] = Tags.FindS(option_deb822[j]); + + // now create one item per suite/section + string Suite = Tags.FindS("Suite"); + Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); + string const Section = Tags.FindS("Section"); + + std::vector<std::string> list_dist = StringSplit(Suite, " "); + std::vector<std::string> list_section = StringSplit(Section, " "); + for (std::vector<std::string>::const_iterator I = list_dist.begin(); + I != list_dist.end(); I++) + { + for (std::vector<std::string>::const_iterator J = list_section.begin(); + J != list_section.end(); J++) + { + if (CreateItem(List, URI, (*I), (*J), Options) == false) + { + return false; + } + } + } + return true; +} + // Type::ParseLine - Parse a single line /*{{{*/ // --------------------------------------------------------------------- /* This is a generic one that is the 'usual' format for sources.list @@ -159,7 +207,6 @@ bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List, return true; } /*}}}*/ - // SourceList::pkgSourceList - Constructors /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -181,7 +228,6 @@ pkgSourceList::~pkgSourceList() delete *I; } /*}}}*/ - /*}}}*/ // SourceList::ReadMainList - Read the main source list from etc /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -216,7 +262,6 @@ bool pkgSourceList::ReadMainList() return Res; } /*}}}*/ -// CNC:2003-03-03 - Needed to preserve backwards compatibility. // SourceList::Reset - Clear the sourcelist contents /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -227,7 +272,6 @@ void pkgSourceList::Reset() SrcList.erase(SrcList.begin(),SrcList.end()); } /*}}}*/ -// CNC:2003-03-03 - Function moved to ReadAppend() and Reset(). // SourceList::Read - Parse the sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -242,16 +286,28 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { + if (_config->FindB("APT::Sources::Use-Deb822", true) == true) + { + int lines_parsed =ParseFileDeb822(File); + if (lines_parsed < 0) + return false; + else if (lines_parsed > 0) + return true; + // no lines parsed ... fall through and use old style parser + } + return ParseFileOldStyle(File); +} + +// SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ParseFileOldStyle(string File) +{ // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str()); - -#if 0 // Now Reset() does this. - for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) - delete *I; - SrcList.erase(SrcList.begin(),SrcList.end()); -#endif + // CNC:2003-12-10 - 300 is too short. char Buffer[1024]; @@ -298,6 +354,49 @@ bool pkgSourceList::ReadAppend(string File) return true; } /*}}}*/ +// SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* Returns: the number of stanzas parsed*/ +int pkgSourceList::ParseFileDeb822(string File) +{ + pkgTagSection Tags; + unsigned int i=0; + + // see if we can read the file + _error->PushToStack(); + FileFd Fd(File, FileFd::ReadOnly); + pkgTagFile Sources(&Fd); + if (_error->PendingError() == true) + { + _error->RevertToStack(); + return 0; + } + _error->MergeWithStack(); + + // read step by step + while (Sources.Step(Tags) == true) + { + if(!Tags.Exists("Type")) + continue; + + string const type = Tags.FindS("Type"); + Type *Parse = Type::GetType(type.c_str()); + if (Parse == 0) + { + _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + return -1; + } + + if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) + return -1; + + i++; + } + + // we are done, return the number of stanzas read + return i; +} + /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 02e27101a..0ccb4aa00 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -31,6 +31,7 @@ #include <vector> #include <map> #include <apt-pkg/pkgcache.h> +#include <apt-pkg/tagfile.h> #ifndef APT_8_CLEANER_HEADERS #include <apt-pkg/metaindex.h> @@ -60,6 +61,10 @@ class pkgSourceList const char *Label; bool FixupURI(std::string &URI) const; + virtual bool ParseStanza(std::vector<metaIndex *> &List, + pkgTagSection &Tags, + int stanza_n, + FileFd &Fd); virtual bool ParseLine(std::vector<metaIndex *> &List, const char *Buffer, unsigned long const &CurLine,std::string const &File) const; @@ -75,7 +80,10 @@ class pkgSourceList protected: std::vector<metaIndex *> SrcList; - + + int ParseFileDeb822(std::string File); + bool ParseFileOldStyle(std::string File); + public: bool ReadMainList(); |