From a537ce19f955f39ee62281bb12bc71a4c67bc635 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Dec 2013 08:01:00 +0100 Subject: first version with test --- apt-pkg/sourcelist.cc | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0fd237cad..1261ebb52 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -242,16 +243,49 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { + // try reading as deb822 + // FIXME: proper error handling so that we do not error for good old-style + // sources + FileFd Fd(File, FileFd::ReadOnly); + pkgTagFile Sources(&Fd); + if (_error->PendingError() == false) + { + pkgTagSection Tags; + map Options; + int i=0; + while (Sources.Step(Tags) == true) + { + if(!Tags.Exists("Type")) + continue; + string const type = Tags.FindS("Type"); + Type *Parse = Type::GetType(type.c_str()); + if (Parse == 0) + return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,File.c_str()); + + string URI = Tags.FindS("URL"); + if (!Parse->FixupURI(URI)) + return _error->Error(_("Malformed stanza %lu in source list %s (URI parse)"),i,File.c_str()); + string const Dist = Tags.FindS("Dist"); + string const Section = Tags.FindS("Section"); + // check if there are any options we support + const char* option_str[] = { + "arch", "arch+", "arch-", "trusted" }; + for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) + if (Tags.Exists(option_str[j])) + Options[option_str[j]] = Tags.FindS(option_str[j]); + Parse->CreateItem(SrcList, URI, Dist, Section, Options); + i++; + } + // we are done + if(i>0) + return true; + } + // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str()); - -#if 0 // Now Reset() does this. - for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) - delete *I; - SrcList.erase(SrcList.begin(),SrcList.end()); -#endif + // CNC:2003-12-10 - 300 is too short. char Buffer[1024]; -- cgit v1.2.3 From 300b15e3456aff88b3016a8bac90a0ba8911db8f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Dec 2013 08:36:10 +0100 Subject: fix section adding --- apt-pkg/sourcelist.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 1261ebb52..4883e2fab 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -273,7 +273,16 @@ bool pkgSourceList::ReadAppend(string File) for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) if (Tags.Exists(option_str[j])) Options[option_str[j]] = Tags.FindS(option_str[j]); - Parse->CreateItem(SrcList, URI, Dist, Section, Options); + + // now create one item per section + std::vector list; + if (Section.find(",")) + list = StringSplit(Section, ","); + else + list = StringSplit(Section, " "); + for (int i=0; i < list.size(); i++) + Parse->CreateItem(SrcList, URI, Dist, list[i], Options); + i++; } // we are done -- cgit v1.2.3 From 6f4478134e13070517c00960e7eb3793d142c0ea Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 07:59:18 +0100 Subject: refactor deb822 reading into its own function --- apt-pkg/sourcelist.cc | 49 +++++++++++++++++++++++++++++++++---------------- apt-pkg/sourcelist.h | 5 ++++- 2 files changed, 37 insertions(+), 17 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 4883e2fab..c28cf0127 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -238,21 +238,15 @@ bool pkgSourceList::Read(string File) return ReadAppend(File); } /*}}}*/ -// SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool pkgSourceList::ReadAppend(string File) + +// FIXME: move into pkgSourceList::Type::ParseFile() +bool pkgSourceList::ParseFileDeb822(FileFd &Fd) { - // try reading as deb822 - // FIXME: proper error handling so that we do not error for good old-style - // sources - FileFd Fd(File, FileFd::ReadOnly); - pkgTagFile Sources(&Fd); - if (_error->PendingError() == false) - { pkgTagSection Tags; map Options; - int i=0; + unsigned int i=0; + + pkgTagFile Sources(&Fd); while (Sources.Step(Tags) == true) { if(!Tags.Exists("Type")) @@ -260,11 +254,11 @@ bool pkgSourceList::ReadAppend(string File) string const type = Tags.FindS("Type"); Type *Parse = Type::GetType(type.c_str()); if (Parse == 0) - return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,File.c_str()); + return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); string URI = Tags.FindS("URL"); if (!Parse->FixupURI(URI)) - return _error->Error(_("Malformed stanza %lu in source list %s (URI parse)"),i,File.c_str()); + return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); string const Dist = Tags.FindS("Dist"); string const Section = Tags.FindS("Section"); // check if there are any options we support @@ -280,16 +274,39 @@ bool pkgSourceList::ReadAppend(string File) list = StringSplit(Section, ","); else list = StringSplit(Section, " "); - for (int i=0; i < list.size(); i++) - Parse->CreateItem(SrcList, URI, Dist, list[i], Options); + for (std::vector::const_iterator I = list.begin(); + I != list.end(); I++) + Parse->CreateItem(SrcList, URI, Dist, (*I), Options); i++; } + // we are done if(i>0) return true; + + return false; +} + + +// SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadAppend(string File) +{ + // *first* try reading as deb822 + // FIXME: proper error handling so that we do not error for good old-style + // sources + FileFd Fd(File, FileFd::ReadOnly); + if (_error->PendingError() == false) + { + if (ParseFileDeb822(Fd)) + return true; } + + // *then* read as old-style sources.list + // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 02e27101a..af3f4f5c6 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -75,7 +75,10 @@ class pkgSourceList protected: std::vector SrcList; - + + // FIXME: move int Type with the next ABI break + bool ParseFileDeb822(FileFd &Fd); + public: bool ReadMainList(); -- cgit v1.2.3 From 1fa78a8a3730633be662df07f7aec8f4c3dcc766 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 08:21:53 +0100 Subject: more refactor --- apt-pkg/sourcelist.cc | 133 +++++++++++++++++++++++++------------------------- apt-pkg/sourcelist.h | 4 +- 2 files changed, 69 insertions(+), 68 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index c28cf0127..714918bc1 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -160,7 +160,6 @@ bool pkgSourceList::Type::ParseLine(vector &List, return true; } /*}}}*/ - // SourceList::pkgSourceList - Constructors /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -182,7 +181,6 @@ pkgSourceList::~pkgSourceList() delete *I; } /*}}}*/ - /*}}}*/ // SourceList::ReadMainList - Read the main source list from etc /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -217,7 +215,6 @@ bool pkgSourceList::ReadMainList() return Res; } /*}}}*/ -// CNC:2003-03-03 - Needed to preserve backwards compatibility. // SourceList::Reset - Clear the sourcelist contents /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -228,7 +225,6 @@ void pkgSourceList::Reset() SrcList.erase(SrcList.begin(),SrcList.end()); } /*}}}*/ -// CNC:2003-03-03 - Function moved to ReadAppend() and Reset(). // SourceList::Read - Parse the sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -238,75 +234,21 @@ bool pkgSourceList::Read(string File) return ReadAppend(File); } /*}}}*/ - -// FIXME: move into pkgSourceList::Type::ParseFile() -bool pkgSourceList::ParseFileDeb822(FileFd &Fd) -{ - pkgTagSection Tags; - map Options; - unsigned int i=0; - - pkgTagFile Sources(&Fd); - while (Sources.Step(Tags) == true) - { - if(!Tags.Exists("Type")) - continue; - string const type = Tags.FindS("Type"); - Type *Parse = Type::GetType(type.c_str()); - if (Parse == 0) - return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); - - string URI = Tags.FindS("URL"); - if (!Parse->FixupURI(URI)) - return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - string const Dist = Tags.FindS("Dist"); - string const Section = Tags.FindS("Section"); - // check if there are any options we support - const char* option_str[] = { - "arch", "arch+", "arch-", "trusted" }; - for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) - if (Tags.Exists(option_str[j])) - Options[option_str[j]] = Tags.FindS(option_str[j]); - - // now create one item per section - std::vector list; - if (Section.find(",")) - list = StringSplit(Section, ","); - else - list = StringSplit(Section, " "); - for (std::vector::const_iterator I = list.begin(); - I != list.end(); I++) - Parse->CreateItem(SrcList, URI, Dist, (*I), Options); - - i++; - } - - // we are done - if(i>0) - return true; - - return false; -} - - // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ bool pkgSourceList::ReadAppend(string File) { - // *first* try reading as deb822 - // FIXME: proper error handling so that we do not error for good old-style - // sources - FileFd Fd(File, FileFd::ReadOnly); - if (_error->PendingError() == false) - { - if (ParseFileDeb822(Fd)) + if (ParseFileDeb822(File)) return true; - } - - - // *then* read as old-style sources.list + return ParseFileOldStyle(File); +} +// SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ParseFileOldStyle(string File) +{ // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) @@ -358,6 +300,65 @@ bool pkgSourceList::ReadAppend(string File) return true; } /*}}}*/ +// SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ParseFileDeb822(string File) +{ + // FIXME: proper error handling so that we do not error for good old-style + // sources + FileFd Fd(File, FileFd::ReadOnly); + if (_error->PendingError() == true) + { + return false; + } + + pkgTagSection Tags; + map Options; + unsigned int i=0; + + pkgTagFile Sources(&Fd); + while (Sources.Step(Tags) == true) + { + if(!Tags.Exists("Type")) + continue; + string const type = Tags.FindS("Type"); + Type *Parse = Type::GetType(type.c_str()); + if (Parse == 0) + return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + + string URI = Tags.FindS("URL"); + if (!Parse->FixupURI(URI)) + return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + string const Dist = Tags.FindS("Dist"); + string const Section = Tags.FindS("Section"); + // check if there are any options we support + const char* option_str[] = { + "arch", "arch+", "arch-", "trusted" }; + for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) + if (Tags.Exists(option_str[j])) + Options[option_str[j]] = Tags.FindS(option_str[j]); + + // now create one item per section + std::vector list; + if (Section.find(",")) + list = StringSplit(Section, ","); + else + list = StringSplit(Section, " "); + for (std::vector::const_iterator I = list.begin(); + I != list.end(); I++) + Parse->CreateItem(SrcList, URI, Dist, (*I), Options); + + i++; + } + + // we are done + if(i>0) + return true; + + return false; +} + /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index af3f4f5c6..5e0d585bb 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -76,8 +76,8 @@ class pkgSourceList std::vector SrcList; - // FIXME: move int Type with the next ABI break - bool ParseFileDeb822(FileFd &Fd); + bool ParseFileDeb822(std::string File); + bool ParseFileOldStyle(std::string File); public: -- cgit v1.2.3 From fce9f472046344d15d4f4df281a003d837cf4177 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 08:30:01 +0100 Subject: add APT::Sources::Use-Deb822 to allow disabling the deb822 parser --- apt-pkg/sourcelist.cc | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 714918bc1..0bbb2bd3f 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -239,7 +239,8 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { - if (ParseFileDeb822(File)) + if (_config->FindB("APT::Sources::Use-Deb822", true) == true) + if (ParseFileDeb822(File)) return true; return ParseFileOldStyle(File); } @@ -305,19 +306,23 @@ bool pkgSourceList::ParseFileOldStyle(string File) /* */ bool pkgSourceList::ParseFileDeb822(string File) { - // FIXME: proper error handling so that we do not error for good old-style - // sources + + pkgTagSection Tags; + map Options; + unsigned int i=0; + + // see if we can read the file + _error->PushToStack(); FileFd Fd(File, FileFd::ReadOnly); + pkgTagFile Sources(&Fd); if (_error->PendingError() == true) { + _error->RevertToStack(); return false; } - - pkgTagSection Tags; - map Options; - unsigned int i=0; + _error->MergeWithStack(); - pkgTagFile Sources(&Fd); + // read step by step while (Sources.Step(Tags) == true) { if(!Tags.Exists("Type")) -- cgit v1.2.3 From 42e19c826b9da6c21a6d286f31db51bc04c73d87 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 08:33:28 +0100 Subject: suppoer $(ARCH) in deb822 sources.list as well --- apt-pkg/sourcelist.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0bbb2bd3f..99cdbe030 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -327,6 +327,7 @@ bool pkgSourceList::ParseFileDeb822(string File) { if(!Tags.Exists("Type")) continue; + string const type = Tags.FindS("Type"); Type *Parse = Type::GetType(type.c_str()); if (Parse == 0) @@ -335,8 +336,10 @@ bool pkgSourceList::ParseFileDeb822(string File) string URI = Tags.FindS("URL"); if (!Parse->FixupURI(URI)) return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - string const Dist = Tags.FindS("Dist"); - string const Section = Tags.FindS("Section"); + + string Dist = Tags.FindS("Dist"); + Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); + // check if there are any options we support const char* option_str[] = { "arch", "arch+", "arch-", "trusted" }; @@ -345,6 +348,7 @@ bool pkgSourceList::ParseFileDeb822(string File) Options[option_str[j]] = Tags.FindS(option_str[j]); // now create one item per section + string const Section = Tags.FindS("Section"); std::vector list; if (Section.find(",")) list = StringSplit(Section, ","); -- cgit v1.2.3 From 5e1ed0889d8aac82ca18add8c6139b153225ae71 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 1 Dec 2013 21:52:36 +0100 Subject: query an empty pkgAcqIndexDiffs if index is up-to-date The previous code already did this, this is just being a hell of a lot more obvious, so that it isn't that easy to break in the future. Git-Dch: Ignore --- apt-pkg/acquire-item.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b76921312..009531c2e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -426,16 +426,18 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ SHA1.AddFD(fd); string const local_sha1 = SHA1.Result(); - if(local_sha1 == ServerSha1) + if(local_sha1 == ServerSha1) { - // we have the same sha1 as the server + // we have the same sha1 as the server so we are done here if(Debug) std::clog << "Package file is up-to-date" << std::endl; - // set found to true, this will queue a pkgAcqIndexDiffs with - // a empty availabe_patches - found = true; - } - else + // list cleanup needs to know that this file as well as the already + // present index is ours, so we create an empty diff to save it for us + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedHash, ServerSha1, available_patches); + return true; + } + else { if(Debug) std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl; -- cgit v1.2.3 From 9d39208af5c8c72d3886c70d603921cf427056ee Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 3 Dec 2013 20:53:04 +0100 Subject: allow ':' in GetListOfFilesInDir run-parts doesn't allow this char in valid filenames, but we tend to have files with this character in e.g. /var/lib/apt/lists/ Git-Dch: Ignore --- apt-pkg/contrib/fileutl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 7fbe4d604..130e990c3 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -465,7 +465,7 @@ std::vector GetListOfFilesInDir(string const &Dir, std::vector c const char *C = Ent->d_name; for (; *C != 0; ++C) if (isalpha(*C) == 0 && isdigit(*C) == 0 - && *C != '_' && *C != '-') { + && *C != '_' && *C != '-' && *C != ':') { // no required extension -> dot is a bad character if (*C == '.' && Ext.empty() == false) continue; -- cgit v1.2.3 From 47d2bc78adb49f3182f9a3d7a4baea363e772d64 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 6 Dec 2013 12:17:48 +0100 Subject: implement POC client-side merging of pdiffs via apt-file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idea of pdiffs is to avoid downloading the hole file by patching the existing index. This works very well, but becomes slow if a lot of patches needs to be applied to reconstruct an up-to-date index and in recent years more and more dinstall (or similar) runs are executed creating more and more pdiffs in the same amount of time, so pdiffs became less useful. The solution is simple: Reduce the amount of patches (which are very small) which need to be applied on top of the index we have available (which is usually pretty big). This can be done in two ways: Either merge the patches on the server-side so that the client has to download only one patch or the patches are all downloaded and merged on the client-side. The first needs a client who is doing one step at a time who can also skip patches if it needs (APT supports this for a long time now). The later is implemented by this commit, but depends on the server NOT merging the patches and the patches being in a strict order in which no patch is skipped. This is traditionally the case for dak, but other repository creators support merging – e.g. reprepro (which helpfully adds a flag indicating that the patches are merged). To support both or even mixes a client needs more information which isn't available for now. This POC uses the external diffindex-rred included in apt-file to do the heavy lifting of merging & applying all patches in one pass, hence to test this feature apt-file needs to be installed. --- apt-pkg/acquire-item.cc | 151 +++++++++++++++++++++++++++++++++++++++++++++++- apt-pkg/acquire-item.h | 100 +++++++++++++++++++++++++++++++- 2 files changed, 247 insertions(+), 4 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 009531c2e..b5b9577ef 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -498,14 +498,42 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ } // we have something, queue the next diff - if(found) + if(found) { // queue the diffs string::size_type const last_space = Description.rfind(" "); if(last_space != string::npos) Description.erase(last_space, Description.size()-last_space); - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + + /* decide if we should download patches one by one or in one go: + The first is good if the server merges patches, but many don't so client + based merging can be attempt in which case the second is better. + "bad things" will happen if patches are merged on the server, + but client side merging is attempt as well */ + bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); + if (pdiff_merge == true) + { + // this perl script is provided by apt-file + pdiff_merge = FileExists(_config->FindFile("Dir::Bin::rred", "/usr/bin/diffindex-rred")); + if (pdiff_merge == true) + { + // reprepro adds this flag if it has merged patches on the server + std::string const precedence = Tags.FindS("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); + } + } + + if (pdiff_merge == false) + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedHash, ServerSha1, available_patches); + else + { + std::vector *diffs = new std::vector(available_patches.size()); + for(size_t i = 0; i < available_patches.size(); ++i) + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash, + available_patches[i], diffs); + } + Complete = false; Status = StatDone; Dequeue(); @@ -754,6 +782,123 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Has } } /*}}}*/ +// AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ +pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + string const &URI, string const &URIDesc, + string const &ShortDesc, HashString const &ExpectedHash, + DiffInfo const &patch, + std::vector const * const allPatches) + : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), + patch(patch),allPatches(allPatches), State(StateFetchDiff) +{ + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + Desc.URI = string(RealURI) + ".diff/" + patch.file + ".gz"; + Desc.Description = Description + " " + patch.file + string(".pdiff"); + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI + ".diff/" + patch.file); + + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; + + QueueURI(Desc); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + + // check if we are the first to fail, otherwise we are done here + State = StateDoneDiff; + for (std::vector::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State == StateErrorDiff) + return; + + // first failure means we should fallback + State = StateErrorDiff; + std::clog << "Falling back to normal index file aquire" << std::endl; + new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, + ExpectedHash); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + + if (State == StateFetchDiff) + { + // rred expects the patch as $FinalFile.ed.$patchname.gz + Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz"); + + // check if this is the last completed diff + State = StateDoneDiff; + for (std::vector::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State != StateDoneDiff) + { + if(Debug) + std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl; + return; + } + + // this is the last completed diff, so we are ready to apply now + State = StateApplyDiff; + + if(Debug) + std::clog << "Sending to rred method: " << FinalFile << std::endl; + + Local = true; + Desc.URI = "rred:" + FinalFile; + QueueURI(Desc); + Mode = "rred"; + return; + } + // success in download/apply all diffs, clean up + else if (State == StateApplyDiff) + { + // see if we really got the expected file + if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + { + RenameOnError(HashSumMismatch); + return; + } + + // move the result into place + if(Debug) + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << FinalFile << std::endl; + Rename(DestFile, FinalFile); + chmod(FinalFile.c_str(), 0644); + + // otherwise lists cleanup will eat the file + DestFile = FinalFile; + + // all set and done + Complete = true; + if(Debug) + std::clog << "allDone: " << DestFile << "\n" << std::endl; + } +} + /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 6b4f73708..5a1c7979c 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -429,7 +429,105 @@ class pkgAcqDiffIndex : public pkgAcquire::Item std::string ShortDesc, HashString ExpectedHash); }; /*}}}*/ -/** \brief An item that is responsible for fetching all the patches {{{ +/** \brief An item that is responsible for fetching client-merge patches {{{ + * that need to be applied to a given package index file. + * + * Instead of downloading and applying each patch one by one like its + * sister #pkgAcqIndexDiffs this class will download all patches at once + * and call rred with all the patches downloaded once. Rred will then + * merge and apply them in one go, which should be a lot faster – but is + * incompatible with server-based merges of patches like reprepro can do. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class pkgAcqIndexMergeDiffs : public pkgAcquire::Item +{ + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** \brief description of the item that is currently being + * downloaded. + */ + pkgAcquire::ItemDesc Desc; + + /** \brief URI of the package index file that is being + * reconstructed. + */ + std::string RealURI; + + /** \brief HashSum of the package index file that is being + * reconstructed. + */ + HashString ExpectedHash; + + /** \brief description of the file being downloaded. */ + std::string Description; + + /** \brief information about the current patch */ + struct DiffInfo const patch; + + /** \brief list of all download items for the patches */ + std::vector const * const allPatches; + + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff, + + /** \brief the work with this diff is done */ + StateDoneDiff, + + /** \brief something bad happened and fallback was triggered */ + StateErrorDiff + } State; + + public: + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); + + virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual std::string DescURI() {return RealURI + "Index";}; + + /** \brief Create an index merge-diff item. + * + * \param Owner The pkgAcquire object that owns this item. + * + * \param URI The URI of the package index file being + * reconstructed. + * + * \param URIDesc A long description of this item. + * + * \param ShortDesc A brief description of this item. + * + * \param ExpectedHash The expected md5sum of the completely + * reconstructed package index file; the index file will be tested + * against this value when it is entirely reconstructed. + * + * \param patch contains infos about the patch this item is supposed + * to download which were read from the index + * + * \param allPatches contains all related items so that each item can + * check if it was the last one to complete the download step + */ + pkgAcqIndexMergeDiffs(pkgAcquire *Owner,std::string const &URI,std::string const &URIDesc, + std::string const &ShortDesc, HashString const &ExpectedHash, + DiffInfo const &patch, std::vector const * const allPatches); +}; + /*}}}*/ +/** \brief An item that is responsible for fetching server-merge patches {{{ * that need to be applied to a given package index file. * * After downloading and applying a single patch, this item will -- cgit v1.2.3 From d2d68aaf5bc2211e9c488f2603ccb4e5fd591a6d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 4 Jan 2014 15:39:04 +0100 Subject: improve tests --- apt-pkg/sourcelist.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 99cdbe030..35e13b6f5 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -342,7 +342,8 @@ bool pkgSourceList::ParseFileDeb822(string File) // check if there are any options we support const char* option_str[] = { - "arch", "arch+", "arch-", "trusted" }; + "arch", "arch+", "arch-", "trusted", + }; for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) if (Tags.Exists(option_str[j])) Options[option_str[j]] = Tags.FindS(option_str[j]); -- cgit v1.2.3 From 4194c9aee2766845618ef0431fd4803b0467aab7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 4 Jan 2014 16:23:32 +0100 Subject: improve error message --- apt-pkg/sourcelist.cc | 34 ++++++++++++++++++++++------------ apt-pkg/sourcelist.h | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 35e13b6f5..5e4a58e95 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -240,8 +240,14 @@ bool pkgSourceList::Read(string File) bool pkgSourceList::ReadAppend(string File) { if (_config->FindB("APT::Sources::Use-Deb822", true) == true) - if (ParseFileDeb822(File)) + { + int lines_parsed =ParseFileDeb822(File); + if (lines_parsed < 0) + return false; + else if (lines_parsed > 0) return true; + // no lines parsed ... fall through and use old style parser + } return ParseFileOldStyle(File); } @@ -303,10 +309,9 @@ bool pkgSourceList::ParseFileOldStyle(string File) /*}}}*/ // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ // --------------------------------------------------------------------- -/* */ -bool pkgSourceList::ParseFileDeb822(string File) +/* Returns: the number of stanzas parsed*/ +int pkgSourceList::ParseFileDeb822(string File) { - pkgTagSection Tags; map Options; unsigned int i=0; @@ -318,7 +323,7 @@ bool pkgSourceList::ParseFileDeb822(string File) if (_error->PendingError() == true) { _error->RevertToStack(); - return false; + return 0; } _error->MergeWithStack(); @@ -331,11 +336,19 @@ bool pkgSourceList::ParseFileDeb822(string File) string const type = Tags.FindS("Type"); Type *Parse = Type::GetType(type.c_str()); if (Parse == 0) - return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + { + _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + // true means we do not retry with old-style sources.list + return -1; + } string URI = Tags.FindS("URL"); if (!Parse->FixupURI(URI)) - return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + { + _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + // means we do not retry with old-style sources.list + return -1; + } string Dist = Tags.FindS("Dist"); Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); @@ -362,11 +375,8 @@ bool pkgSourceList::ParseFileDeb822(string File) i++; } - // we are done - if(i>0) - return true; - - return false; + // we are done, return the number of stanzas read + return i; } /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 5e0d585bb..d83c76d6a 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -76,7 +76,7 @@ class pkgSourceList std::vector SrcList; - bool ParseFileDeb822(std::string File); + int ParseFileDeb822(std::string File); bool ParseFileOldStyle(std::string File); public: -- cgit v1.2.3 From 1e4a2b763f2225d6de3d498263da2a1a12697667 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Wed, 15 Jan 2014 15:55:26 +0100 Subject: correct IndexDiff vs DiffIndex in Debug output --- apt-pkg/acquire-item.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b5b9577ef..73f5f4901 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -369,10 +369,10 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, return; } - if(Debug) - std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " - << CurrentPackagesFile << std::endl; - + if(Debug) + std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): " + << CurrentPackagesFile << std::endl; + QueueURI(Desc); } @@ -398,8 +398,8 @@ string pkgAcqDiffIndex::Custom600Headers() bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ { if(Debug) - std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile - << std::endl; + std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile + << std::endl; pkgTagSection Tags; string ServerSha1; -- cgit v1.2.3 From c3a17127ad383ec8e1f480ff2e1a4dccaa537a11 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Dec 2013 14:52:15 +0100 Subject: reenable unlimited pdiff files download In 51fc6def77edfb1f429a48e5169519e9e05a759b we limited the amount of pdiff to be downloaded per index to 20. This was a compromise between not letting it go overboard (becoming even slower) and not using bandwidth needlessly. Now that with the POC the speed reason is gone it makes sense again to download as much files as we possible can via pdiff to save bandwidth (and possibly even time). It also avoids problems with the limit in cases we were we deal with a server merged archieve as this limit assumes a strict patch progression. --- apt-pkg/acquire-item.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 73f5f4901..7f6443555 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -462,7 +462,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (available_patches.empty() == false) { // patching with too many files is rather slow compared to a fast download - unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 20); + unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); if (fileLimit != 0 && fileLimit < available_patches.size()) { if (Debug) -- cgit v1.2.3 From 50bd6fd3794dd1f61185302129dc6cd218d20b98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jan 2014 17:23:05 +0100 Subject: integrate Anthonys rred with POC for client-side merge Providing the benefits of both without the downsides :) (ABI breaks or external dependencies) For this Anthonys rred is equipped with: - magic-filename-pickup of patches rather than explicit messages - use of FileFd instead of FILE* to get on-the-fly uncompress of the gzip compressed pdiff patches The acquire code in turn stops checking for apt-file's helper as our own rred is now clever enough for our needs. --- apt-pkg/acquire-item.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7f6443555..1185908f3 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -513,14 +513,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); if (pdiff_merge == true) { - // this perl script is provided by apt-file - pdiff_merge = FileExists(_config->FindFile("Dir::Bin::rred", "/usr/bin/diffindex-rred")); - if (pdiff_merge == true) - { - // reprepro adds this flag if it has merged patches on the server - std::string const precedence = Tags.FindS("X-Patch-Precedence"); - pdiff_merge = (precedence != "merged"); - } + // reprepro adds this flag if it has merged patches on the server + std::string const precedence = Tags.FindS("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); } if (pdiff_merge == false) -- cgit v1.2.3 From db6594dfc508378b6d658aff2761da5406404238 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 16 Jan 2014 08:03:24 +0100 Subject: remove "," in components again --- apt-pkg/sourcelist.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 5e4a58e95..42ada7e18 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -363,11 +363,7 @@ int pkgSourceList::ParseFileDeb822(string File) // now create one item per section string const Section = Tags.FindS("Section"); - std::vector list; - if (Section.find(",")) - list = StringSplit(Section, ","); - else - list = StringSplit(Section, " "); + std::vector list = StringSplit(Section, " "); for (std::vector::const_iterator I = list.begin(); I != list.end(); I++) Parse->CreateItem(SrcList, URI, Dist, (*I), Options); -- cgit v1.2.3 From 7037aab52fc935298b033a4c7ba7ccb5b697622e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 16 Jan 2014 16:25:33 +0100 Subject: * refactor to have a new virtual ParseStanza Have a similar ParseStanza() to the current ParseLine(). Rename the Architectures options in deb822 to make it more user friendly --- apt-pkg/sourcelist.cc | 65 ++++++++++++++++++++++++++++++--------------------- apt-pkg/sourcelist.h | 5 ++++ 2 files changed, 44 insertions(+), 26 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 42ada7e18..fe0eace07 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -71,6 +71,44 @@ bool pkgSourceList::Type::FixupURI(string &URI) const return true; } /*}}}*/ +bool pkgSourceList::Type::ParseStanza(vector &List, + pkgTagSection &Tags, + int i, + FileFd &Fd) +{ + map Options; + + string URI = Tags.FindS("URL"); + if (!FixupURI(URI)) + { + _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + return false; + } + + string Dist = Tags.FindS("Dist"); + Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); + + // Define external/internal options + const char* option_deb822[] = { + "Architectures", "Architectures-Add", "Architectures-Delete", "Trusted", + }; + const char* option_internal[] = { + "arch", "arch+", "arch-", "trusted", + }; + for (unsigned int j=0; j < sizeof(option_deb822)/sizeof(char*); j++) + if (Tags.Exists(option_deb822[j])) + Options[option_internal[j]] = Tags.FindS(option_deb822[j]); + + // now create one item per section + string const Section = Tags.FindS("Section"); + std::vector list = StringSplit(Section, " "); + for (std::vector::const_iterator I = list.begin(); + I != list.end(); I++) + return CreateItem(List, URI, Dist, (*I), Options); + + return true; +} + // Type::ParseLine - Parse a single line /*{{{*/ // --------------------------------------------------------------------- /* This is a generic one that is the 'usual' format for sources.list @@ -313,7 +351,6 @@ bool pkgSourceList::ParseFileOldStyle(string File) int pkgSourceList::ParseFileDeb822(string File) { pkgTagSection Tags; - map Options; unsigned int i=0; // see if we can read the file @@ -338,35 +375,11 @@ int pkgSourceList::ParseFileDeb822(string File) if (Parse == 0) { _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); - // true means we do not retry with old-style sources.list return -1; } - string URI = Tags.FindS("URL"); - if (!Parse->FixupURI(URI)) - { - _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - // means we do not retry with old-style sources.list + if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) return -1; - } - - string Dist = Tags.FindS("Dist"); - Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); - - // check if there are any options we support - const char* option_str[] = { - "arch", "arch+", "arch-", "trusted", - }; - for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) - if (Tags.Exists(option_str[j])) - Options[option_str[j]] = Tags.FindS(option_str[j]); - - // now create one item per section - string const Section = Tags.FindS("Section"); - std::vector list = StringSplit(Section, " "); - for (std::vector::const_iterator I = list.begin(); - I != list.end(); I++) - Parse->CreateItem(SrcList, URI, Dist, (*I), Options); i++; } diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index d83c76d6a..0ccb4aa00 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -31,6 +31,7 @@ #include #include #include +#include #ifndef APT_8_CLEANER_HEADERS #include @@ -60,6 +61,10 @@ class pkgSourceList const char *Label; bool FixupURI(std::string &URI) const; + virtual bool ParseStanza(std::vector &List, + pkgTagSection &Tags, + int stanza_n, + FileFd &Fd); virtual bool ParseLine(std::vector &List, const char *Buffer, unsigned long const &CurLine,std::string const &File) const; -- cgit v1.2.3 From 796a0eff1acebe858632c344e77bfc3189b2244f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 16 Jan 2014 17:00:56 +0100 Subject: rename "distribution" in sources.list to "suite" --- apt-pkg/sourcelist.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index fe0eace07..ddebd206d 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -85,7 +85,7 @@ bool pkgSourceList::Type::ParseStanza(vector &List, return false; } - string Dist = Tags.FindS("Dist"); + string Dist = Tags.FindS("Suite"); Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); // Define external/internal options -- cgit v1.2.3 From a51fa92c521a6790446108a54ad1d9b6a16515ff Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 16 Jan 2014 17:03:07 +0100 Subject: rename URL to Uri in deb822-sources --- apt-pkg/sourcelist.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index ddebd206d..0cea8dc7f 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -78,7 +78,7 @@ bool pkgSourceList::Type::ParseStanza(vector &List, { map Options; - string URI = Tags.FindS("URL"); + string URI = Tags.FindS("Uri"); if (!FixupURI(URI)) { _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); -- cgit v1.2.3 From d73743ddae1228bcd409700d8d0ffbe26e2e6cd1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 16 Jan 2014 17:13:18 +0100 Subject: support multiple "Suite:" entries --- apt-pkg/sourcelist.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0cea8dc7f..09d8287a0 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -85,9 +85,6 @@ bool pkgSourceList::Type::ParseStanza(vector &List, return false; } - string Dist = Tags.FindS("Suite"); - Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); - // Define external/internal options const char* option_deb822[] = { "Architectures", "Architectures-Add", "Architectures-Delete", "Trusted", @@ -99,13 +96,25 @@ bool pkgSourceList::Type::ParseStanza(vector &List, if (Tags.Exists(option_deb822[j])) Options[option_internal[j]] = Tags.FindS(option_deb822[j]); - // now create one item per section + // now create one item per suite/section + string Suite = Tags.FindS("Suite"); + Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); string const Section = Tags.FindS("Section"); - std::vector list = StringSplit(Section, " "); - for (std::vector::const_iterator I = list.begin(); - I != list.end(); I++) - return CreateItem(List, URI, Dist, (*I), Options); - + + std::vector list_dist = StringSplit(Suite, " "); + std::vector list_section = StringSplit(Section, " "); + for (std::vector::const_iterator I = list_dist.begin(); + I != list_dist.end(); I++) + { + for (std::vector::const_iterator J = list_section.begin(); + J != list_section.end(); J++) + { + if (CreateItem(List, URI, (*I), (*J), Options) == false) + { + return false; + } + } + } return true; } -- cgit v1.2.3 From 866e9fadf892368fcb50e6a192bcdd350cfe8e5c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 17 Jan 2014 20:41:55 +0100 Subject: implement suggestion by donkult (thanks!) --- apt-pkg/sourcelist.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 09d8287a0..51b766095 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -78,7 +78,7 @@ bool pkgSourceList::Type::ParseStanza(vector &List, { map Options; - string URI = Tags.FindS("Uri"); + string URI = Tags.FindS("URI"); if (!FixupURI(URI)) { _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); @@ -87,7 +87,7 @@ bool pkgSourceList::Type::ParseStanza(vector &List, // Define external/internal options const char* option_deb822[] = { - "Architectures", "Architectures-Add", "Architectures-Delete", "Trusted", + "Architectures", "Architectures-Add", "Architectures-Remove", "Trusted", }; const char* option_internal[] = { "arch", "arch+", "arch-", "trusted", -- cgit v1.2.3 From 6c069a2247781754bcc8574687cb98b493c6ab8a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 18 Jan 2014 20:51:03 +0100 Subject: rename "Suite/Section" to plural --- apt-pkg/sourcelist.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 51b766095..4e580ba04 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -97,9 +97,9 @@ bool pkgSourceList::Type::ParseStanza(vector &List, Options[option_internal[j]] = Tags.FindS(option_deb822[j]); // now create one item per suite/section - string Suite = Tags.FindS("Suite"); + string Suite = Tags.FindS("Suites"); Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); - string const Section = Tags.FindS("Section"); + string const Section = Tags.FindS("Sections"); std::vector list_dist = StringSplit(Suite, " "); std::vector list_section = StringSplit(Section, " "); -- cgit v1.2.3 From 1700c3cfb902f3f61fa4d9269523467fcceaaf58 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 18 Jan 2014 23:42:51 +0100 Subject: do not ignore ioctl(TIOCSCTTY) errors Show a proper error message when a ioctl() in dpkgpm.cc fails. Also simply StartPtyMagic() a bit. --- apt-pkg/deb/dpkgpm.cc | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 14ce133cf..506ebf620 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1028,6 +1028,12 @@ bool pkgDPkgPM::Go(int StatusFd) void pkgDPkgPM::StartPtyMagic() { + if (_config->FindB("Dpkg::Use-Pty", true) == false) + { + d->master = d->slave = -1; + return; + } + // setup the pty and stuff struct winsize win; @@ -1036,10 +1042,9 @@ void pkgDPkgPM::StartPtyMagic() _error->PushToStack(); if (tcgetattr(STDOUT_FILENO, &d->tt) == 0) { - ioctl(1, TIOCGWINSZ, (char *)&win); - if (_config->FindB("Dpkg::Use-Pty", true) == false) + if (ioctl(1, TIOCGWINSZ, (char *)&win) < 0) { - d->master = d->slave = -1; + _error->Errno("ioctl", _("ioctl(TIOCGWINSZ) failed")); } else if (openpty(&d->master, &d->slave, NULL, &d->tt, &win) < 0) { _error->Errno("openpty", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); @@ -1393,12 +1398,17 @@ bool pkgDPkgPM::GoNoABIBreak(APT::Progress::PackageManager *progress) if(d->slave >= 0 && d->master >= 0) { setsid(); - ioctl(d->slave, TIOCSCTTY, 0); - close(d->master); - dup2(d->slave, 0); - dup2(d->slave, 1); - dup2(d->slave, 2); - close(d->slave); + int res = ioctl(d->slave, TIOCSCTTY, 0); + if (res < 0) { + std::cerr << "ioctl(TIOCSCTTY) failed for fd: " + << d->slave << std::endl; + } else { + close(d->master); + dup2(d->slave, 0); + dup2(d->slave, 1); + dup2(d->slave, 2); + close(d->slave); + } } close(fd[0]); // close the read end of the pipe -- cgit v1.2.3 From 7dd62ea93413a73b4ec394b16ff4e0367d226395 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 20 Jan 2014 07:59:11 +0100 Subject: add support for Enabled: no in deb822 sources.list --- apt-pkg/sourcelist.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 4e580ba04..5d41fb00e 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -84,6 +84,10 @@ bool pkgSourceList::Type::ParseStanza(vector &List, _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); return false; } + + string Enabled = Tags.FindS("Enabled"); + if (Enabled.size() > 0 && StringToBool(Enabled) == false) + return true; // Define external/internal options const char* option_deb822[] = { -- cgit v1.2.3 From 75c10df1533ede97e05fef3d1e2fc6a22fc4db00 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 20 Jan 2014 08:10:50 +0100 Subject: add support for multiple URIs in deb822 style sources.list --- apt-pkg/sourcelist.cc | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 5d41fb00e..339005149 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -78,13 +78,6 @@ bool pkgSourceList::Type::ParseStanza(vector &List, { map Options; - string URI = Tags.FindS("URI"); - if (!FixupURI(URI)) - { - _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - return false; - } - string Enabled = Tags.FindS("Enabled"); if (Enabled.size() > 0 && StringToBool(Enabled) == false) return true; @@ -104,20 +97,34 @@ bool pkgSourceList::Type::ParseStanza(vector &List, string Suite = Tags.FindS("Suites"); Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); string const Section = Tags.FindS("Sections"); + string URIS = Tags.FindS("URIs"); + std::vector list_uris = StringSplit(URIS, " "); std::vector list_dist = StringSplit(Suite, " "); std::vector list_section = StringSplit(Section, " "); - for (std::vector::const_iterator I = list_dist.begin(); - I != list_dist.end(); I++) + + for (std::vector::const_iterator U = list_uris.begin(); + U != list_uris.end(); U++) { - for (std::vector::const_iterator J = list_section.begin(); - J != list_section.end(); J++) + std::string URI = (*U); + if (!FixupURI(URI)) + { + _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + return false; + } + + for (std::vector::const_iterator I = list_dist.begin(); + I != list_dist.end(); I++) + { + for (std::vector::const_iterator J = list_section.begin(); + J != list_section.end(); J++) { if (CreateItem(List, URI, (*I), (*J), Options) == false) { return false; } } + } } return true; } -- cgit v1.2.3 From 7f316a3feab95370f1dd28c08c58bc3c140bf0a0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 20 Jan 2014 08:17:43 +0100 Subject: add support for multipl types in one line --- apt-pkg/sourcelist.cc | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 339005149..bbc514f5b 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -387,21 +387,26 @@ int pkgSourceList::ParseFileDeb822(string File) // read step by step while (Sources.Step(Tags) == true) { - if(!Tags.Exists("Type")) + if(!Tags.Exists("Types")) continue; - string const type = Tags.FindS("Type"); - Type *Parse = Type::GetType(type.c_str()); - if (Parse == 0) + string const types = Tags.FindS("Types"); + std::vector list_types = StringSplit(types, " "); + for (std::vector::const_iterator I = list_types.begin(); + I != list_types.end(); I++) { - _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); - return -1; - } + Type *Parse = Type::GetType((*I).c_str()); + if (Parse == 0) + { + _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str()); + return -1; + } - if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) - return -1; + if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) + return -1; - i++; + i++; + } } // we are done, return the number of stanzas read -- cgit v1.2.3 From 9e51c0b6a3a1a36336820eda11bcfd5534d9d80c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 22 Jan 2014 17:18:26 +0100 Subject: "apt show" show user friendly size info The size/installed-size is displayed via SizeToStr() and Size is rewriten to "Download-Size" to make clear what size is refered to here. --- apt-pkg/tagfile.cc | 84 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 41 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index bef3c76ba..b92b2c15a 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -567,52 +567,54 @@ bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[], } // Write all all of the tags, in order. - for (unsigned int I = 0; Order[I] != 0; I++) + if (Order != NULL) { - bool Rewritten = false; - - // See if this is a field that needs to be rewritten - for (unsigned int J = 0; Rewrite != 0 && Rewrite[J].Tag != 0; J++) + for (unsigned int I = 0; Order[I] != 0; I++) { - if (strcasecmp(Rewrite[J].Tag,Order[I]) == 0) - { - Visited[J] |= 2; - if (Rewrite[J].Rewrite != 0 && Rewrite[J].Rewrite[0] != 0) - { - if (isspace(Rewrite[J].Rewrite[0])) - fprintf(Output,"%s:%s\n",Rewrite[J].NewTag,Rewrite[J].Rewrite); - else - fprintf(Output,"%s: %s\n",Rewrite[J].NewTag,Rewrite[J].Rewrite); - } - - Rewritten = true; - break; - } - } + bool Rewritten = false; + + // See if this is a field that needs to be rewritten + for (unsigned int J = 0; Rewrite != 0 && Rewrite[J].Tag != 0; J++) + { + if (strcasecmp(Rewrite[J].Tag,Order[I]) == 0) + { + Visited[J] |= 2; + if (Rewrite[J].Rewrite != 0 && Rewrite[J].Rewrite[0] != 0) + { + if (isspace(Rewrite[J].Rewrite[0])) + fprintf(Output,"%s:%s\n",Rewrite[J].NewTag,Rewrite[J].Rewrite); + else + fprintf(Output,"%s: %s\n",Rewrite[J].NewTag,Rewrite[J].Rewrite); + } + Rewritten = true; + break; + } + } - // See if it is in the fragment - unsigned Pos; - if (Tags.Find(Order[I],Pos) == false) - continue; - Visited[Pos] |= 1; - - if (Rewritten == true) - continue; + // See if it is in the fragment + unsigned Pos; + if (Tags.Find(Order[I],Pos) == false) + continue; + Visited[Pos] |= 1; + + if (Rewritten == true) + continue; - /* Write out this element, taking a moment to rewrite the tag - in case of changes of case. */ - const char *Start; - const char *Stop; - Tags.Get(Start,Stop,Pos); + /* Write out this element, taking a moment to rewrite the tag + in case of changes of case. */ + const char *Start; + const char *Stop; + Tags.Get(Start,Stop,Pos); - if (fputs(Order[I],Output) < 0) - return _error->Errno("fputs","IO Error to output"); - Start += strlen(Order[I]); - if (fwrite(Start,Stop - Start,1,Output) != 1) - return _error->Errno("fwrite","IO Error to output"); - if (Stop[-1] != '\n') - fprintf(Output,"\n"); - } + if (fputs(Order[I],Output) < 0) + return _error->Errno("fputs","IO Error to output"); + Start += strlen(Order[I]); + if (fwrite(Start,Stop - Start,1,Output) != 1) + return _error->Errno("fwrite","IO Error to output"); + if (Stop[-1] != '\n') + fprintf(Output,"\n"); + } + } // Now write all the old tags that were missed. for (unsigned int I = 0; I != Tags.Count(); I++) -- cgit v1.2.3 From 655af1e137385aa41c5505edc350111e9f8762ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 24 Jan 2014 23:17:30 +0100 Subject: set APT::Sources::Use-Deb822 to default false for now --- apt-pkg/sourcelist.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index bbc514f5b..ac326d157 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -297,7 +297,7 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { - if (_config->FindB("APT::Sources::Use-Deb822", true) == true) + if (_config->FindB("APT::Sources::Use-Deb822", false) == true) { int lines_parsed =ParseFileDeb822(File); if (lines_parsed < 0) -- cgit v1.2.3 From 41e6bd080db8995bc7617569f0719bccc31e0da8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 25 Jan 2014 14:53:03 +0100 Subject: support " " in deb822 source options --- apt-pkg/sourcelist.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 4e580ba04..8462027c5 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -20,6 +20,7 @@ #include #include +#include #include /*}}}*/ @@ -94,7 +95,12 @@ bool pkgSourceList::Type::ParseStanza(vector &List, }; for (unsigned int j=0; j < sizeof(option_deb822)/sizeof(char*); j++) if (Tags.Exists(option_deb822[j])) - Options[option_internal[j]] = Tags.FindS(option_deb822[j]); + { + // for deb822 the " " is the delimiter, but the backend expects "," + std::string option = Tags.FindS(option_deb822[j]); + std::replace(option.begin(), option.end(), ' ', ','); + Options[option_internal[j]] = option; + } // now create one item per suite/section string Suite = Tags.FindS("Suites"); -- cgit v1.2.3