From a537ce19f955f39ee62281bb12bc71a4c67bc635 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Dec 2013 08:01:00 +0100 Subject: first version with test --- apt-pkg/sourcelist.cc | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0fd237cad..1261ebb52 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -242,16 +243,49 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { + // try reading as deb822 + // FIXME: proper error handling so that we do not error for good old-style + // sources + FileFd Fd(File, FileFd::ReadOnly); + pkgTagFile Sources(&Fd); + if (_error->PendingError() == false) + { + pkgTagSection Tags; + map Options; + int i=0; + while (Sources.Step(Tags) == true) + { + if(!Tags.Exists("Type")) + continue; + string const type = Tags.FindS("Type"); + Type *Parse = Type::GetType(type.c_str()); + if (Parse == 0) + return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,File.c_str()); + + string URI = Tags.FindS("URL"); + if (!Parse->FixupURI(URI)) + return _error->Error(_("Malformed stanza %lu in source list %s (URI parse)"),i,File.c_str()); + string const Dist = Tags.FindS("Dist"); + string const Section = Tags.FindS("Section"); + // check if there are any options we support + const char* option_str[] = { + "arch", "arch+", "arch-", "trusted" }; + for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) + if (Tags.Exists(option_str[j])) + Options[option_str[j]] = Tags.FindS(option_str[j]); + Parse->CreateItem(SrcList, URI, Dist, Section, Options); + i++; + } + // we are done + if(i>0) + return true; + } + // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str()); - -#if 0 // Now Reset() does this. - for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) - delete *I; - SrcList.erase(SrcList.begin(),SrcList.end()); -#endif + // CNC:2003-12-10 - 300 is too short. char Buffer[1024]; -- cgit v1.2.3 From 300b15e3456aff88b3016a8bac90a0ba8911db8f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Dec 2013 08:36:10 +0100 Subject: fix section adding --- apt-pkg/sourcelist.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 1261ebb52..4883e2fab 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -273,7 +273,16 @@ bool pkgSourceList::ReadAppend(string File) for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) if (Tags.Exists(option_str[j])) Options[option_str[j]] = Tags.FindS(option_str[j]); - Parse->CreateItem(SrcList, URI, Dist, Section, Options); + + // now create one item per section + std::vector list; + if (Section.find(",")) + list = StringSplit(Section, ","); + else + list = StringSplit(Section, " "); + for (int i=0; i < list.size(); i++) + Parse->CreateItem(SrcList, URI, Dist, list[i], Options); + i++; } // we are done -- cgit v1.2.3 From 6f4478134e13070517c00960e7eb3793d142c0ea Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 07:59:18 +0100 Subject: refactor deb822 reading into its own function --- apt-pkg/sourcelist.cc | 49 +++++++++++++++++++++++++++++++++---------------- apt-pkg/sourcelist.h | 5 ++++- 2 files changed, 37 insertions(+), 17 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 4883e2fab..c28cf0127 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -238,21 +238,15 @@ bool pkgSourceList::Read(string File) return ReadAppend(File); } /*}}}*/ -// SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool pkgSourceList::ReadAppend(string File) + +// FIXME: move into pkgSourceList::Type::ParseFile() +bool pkgSourceList::ParseFileDeb822(FileFd &Fd) { - // try reading as deb822 - // FIXME: proper error handling so that we do not error for good old-style - // sources - FileFd Fd(File, FileFd::ReadOnly); - pkgTagFile Sources(&Fd); - if (_error->PendingError() == false) - { pkgTagSection Tags; map Options; - int i=0; + unsigned int i=0; + + pkgTagFile Sources(&Fd); while (Sources.Step(Tags) == true) { if(!Tags.Exists("Type")) @@ -260,11 +254,11 @@ bool pkgSourceList::ReadAppend(string File) string const type = Tags.FindS("Type"); Type *Parse = Type::GetType(type.c_str()); if (Parse == 0) - return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,File.c_str()); + return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); string URI = Tags.FindS("URL"); if (!Parse->FixupURI(URI)) - return _error->Error(_("Malformed stanza %lu in source list %s (URI parse)"),i,File.c_str()); + return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); string const Dist = Tags.FindS("Dist"); string const Section = Tags.FindS("Section"); // check if there are any options we support @@ -280,16 +274,39 @@ bool pkgSourceList::ReadAppend(string File) list = StringSplit(Section, ","); else list = StringSplit(Section, " "); - for (int i=0; i < list.size(); i++) - Parse->CreateItem(SrcList, URI, Dist, list[i], Options); + for (std::vector::const_iterator I = list.begin(); + I != list.end(); I++) + Parse->CreateItem(SrcList, URI, Dist, (*I), Options); i++; } + // we are done if(i>0) return true; + + return false; +} + + +// SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadAppend(string File) +{ + // *first* try reading as deb822 + // FIXME: proper error handling so that we do not error for good old-style + // sources + FileFd Fd(File, FileFd::ReadOnly); + if (_error->PendingError() == false) + { + if (ParseFileDeb822(Fd)) + return true; } + + // *then* read as old-style sources.list + // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 02e27101a..af3f4f5c6 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -75,7 +75,10 @@ class pkgSourceList protected: std::vector SrcList; - + + // FIXME: move int Type with the next ABI break + bool ParseFileDeb822(FileFd &Fd); + public: bool ReadMainList(); -- cgit v1.2.3 From 1fa78a8a3730633be662df07f7aec8f4c3dcc766 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 08:21:53 +0100 Subject: more refactor --- apt-pkg/sourcelist.cc | 133 +++++++++++++++++++++++++------------------------- apt-pkg/sourcelist.h | 4 +- 2 files changed, 69 insertions(+), 68 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index c28cf0127..714918bc1 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -160,7 +160,6 @@ bool pkgSourceList::Type::ParseLine(vector &List, return true; } /*}}}*/ - // SourceList::pkgSourceList - Constructors /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -182,7 +181,6 @@ pkgSourceList::~pkgSourceList() delete *I; } /*}}}*/ - /*}}}*/ // SourceList::ReadMainList - Read the main source list from etc /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -217,7 +215,6 @@ bool pkgSourceList::ReadMainList() return Res; } /*}}}*/ -// CNC:2003-03-03 - Needed to preserve backwards compatibility. // SourceList::Reset - Clear the sourcelist contents /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -228,7 +225,6 @@ void pkgSourceList::Reset() SrcList.erase(SrcList.begin(),SrcList.end()); } /*}}}*/ -// CNC:2003-03-03 - Function moved to ReadAppend() and Reset(). // SourceList::Read - Parse the sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -238,75 +234,21 @@ bool pkgSourceList::Read(string File) return ReadAppend(File); } /*}}}*/ - -// FIXME: move into pkgSourceList::Type::ParseFile() -bool pkgSourceList::ParseFileDeb822(FileFd &Fd) -{ - pkgTagSection Tags; - map Options; - unsigned int i=0; - - pkgTagFile Sources(&Fd); - while (Sources.Step(Tags) == true) - { - if(!Tags.Exists("Type")) - continue; - string const type = Tags.FindS("Type"); - Type *Parse = Type::GetType(type.c_str()); - if (Parse == 0) - return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); - - string URI = Tags.FindS("URL"); - if (!Parse->FixupURI(URI)) - return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - string const Dist = Tags.FindS("Dist"); - string const Section = Tags.FindS("Section"); - // check if there are any options we support - const char* option_str[] = { - "arch", "arch+", "arch-", "trusted" }; - for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) - if (Tags.Exists(option_str[j])) - Options[option_str[j]] = Tags.FindS(option_str[j]); - - // now create one item per section - std::vector list; - if (Section.find(",")) - list = StringSplit(Section, ","); - else - list = StringSplit(Section, " "); - for (std::vector::const_iterator I = list.begin(); - I != list.end(); I++) - Parse->CreateItem(SrcList, URI, Dist, (*I), Options); - - i++; - } - - // we are done - if(i>0) - return true; - - return false; -} - - // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ bool pkgSourceList::ReadAppend(string File) { - // *first* try reading as deb822 - // FIXME: proper error handling so that we do not error for good old-style - // sources - FileFd Fd(File, FileFd::ReadOnly); - if (_error->PendingError() == false) - { - if (ParseFileDeb822(Fd)) + if (ParseFileDeb822(File)) return true; - } - - - // *then* read as old-style sources.list + return ParseFileOldStyle(File); +} +// SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ParseFileOldStyle(string File) +{ // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) @@ -358,6 +300,65 @@ bool pkgSourceList::ReadAppend(string File) return true; } /*}}}*/ +// SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ParseFileDeb822(string File) +{ + // FIXME: proper error handling so that we do not error for good old-style + // sources + FileFd Fd(File, FileFd::ReadOnly); + if (_error->PendingError() == true) + { + return false; + } + + pkgTagSection Tags; + map Options; + unsigned int i=0; + + pkgTagFile Sources(&Fd); + while (Sources.Step(Tags) == true) + { + if(!Tags.Exists("Type")) + continue; + string const type = Tags.FindS("Type"); + Type *Parse = Type::GetType(type.c_str()); + if (Parse == 0) + return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + + string URI = Tags.FindS("URL"); + if (!Parse->FixupURI(URI)) + return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + string const Dist = Tags.FindS("Dist"); + string const Section = Tags.FindS("Section"); + // check if there are any options we support + const char* option_str[] = { + "arch", "arch+", "arch-", "trusted" }; + for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) + if (Tags.Exists(option_str[j])) + Options[option_str[j]] = Tags.FindS(option_str[j]); + + // now create one item per section + std::vector list; + if (Section.find(",")) + list = StringSplit(Section, ","); + else + list = StringSplit(Section, " "); + for (std::vector::const_iterator I = list.begin(); + I != list.end(); I++) + Parse->CreateItem(SrcList, URI, Dist, (*I), Options); + + i++; + } + + // we are done + if(i>0) + return true; + + return false; +} + /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index af3f4f5c6..5e0d585bb 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -76,8 +76,8 @@ class pkgSourceList std::vector SrcList; - // FIXME: move int Type with the next ABI break - bool ParseFileDeb822(FileFd &Fd); + bool ParseFileDeb822(std::string File); + bool ParseFileOldStyle(std::string File); public: -- cgit v1.2.3 From fce9f472046344d15d4f4df281a003d837cf4177 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 08:30:01 +0100 Subject: add APT::Sources::Use-Deb822 to allow disabling the deb822 parser --- apt-pkg/sourcelist.cc | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 714918bc1..0bbb2bd3f 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -239,7 +239,8 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { - if (ParseFileDeb822(File)) + if (_config->FindB("APT::Sources::Use-Deb822", true) == true) + if (ParseFileDeb822(File)) return true; return ParseFileOldStyle(File); } @@ -305,19 +306,23 @@ bool pkgSourceList::ParseFileOldStyle(string File) /* */ bool pkgSourceList::ParseFileDeb822(string File) { - // FIXME: proper error handling so that we do not error for good old-style - // sources + + pkgTagSection Tags; + map Options; + unsigned int i=0; + + // see if we can read the file + _error->PushToStack(); FileFd Fd(File, FileFd::ReadOnly); + pkgTagFile Sources(&Fd); if (_error->PendingError() == true) { + _error->RevertToStack(); return false; } - - pkgTagSection Tags; - map Options; - unsigned int i=0; + _error->MergeWithStack(); - pkgTagFile Sources(&Fd); + // read step by step while (Sources.Step(Tags) == true) { if(!Tags.Exists("Type")) -- cgit v1.2.3 From 42e19c826b9da6c21a6d286f31db51bc04c73d87 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Dec 2013 08:33:28 +0100 Subject: suppoer $(ARCH) in deb822 sources.list as well --- apt-pkg/sourcelist.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0bbb2bd3f..99cdbe030 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -327,6 +327,7 @@ bool pkgSourceList::ParseFileDeb822(string File) { if(!Tags.Exists("Type")) continue; + string const type = Tags.FindS("Type"); Type *Parse = Type::GetType(type.c_str()); if (Parse == 0) @@ -335,8 +336,10 @@ bool pkgSourceList::ParseFileDeb822(string File) string URI = Tags.FindS("URL"); if (!Parse->FixupURI(URI)) return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - string const Dist = Tags.FindS("Dist"); - string const Section = Tags.FindS("Section"); + + string Dist = Tags.FindS("Dist"); + Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); + // check if there are any options we support const char* option_str[] = { "arch", "arch+", "arch-", "trusted" }; @@ -345,6 +348,7 @@ bool pkgSourceList::ParseFileDeb822(string File) Options[option_str[j]] = Tags.FindS(option_str[j]); // now create one item per section + string const Section = Tags.FindS("Section"); std::vector list; if (Section.find(",")) list = StringSplit(Section, ","); -- cgit v1.2.3 From d2d68aaf5bc2211e9c488f2603ccb4e5fd591a6d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 4 Jan 2014 15:39:04 +0100 Subject: improve tests --- apt-pkg/sourcelist.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 99cdbe030..35e13b6f5 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -342,7 +342,8 @@ bool pkgSourceList::ParseFileDeb822(string File) // check if there are any options we support const char* option_str[] = { - "arch", "arch+", "arch-", "trusted" }; + "arch", "arch+", "arch-", "trusted", + }; for (unsigned int j=0; j < sizeof(option_str)/sizeof(char*); j++) if (Tags.Exists(option_str[j])) Options[option_str[j]] = Tags.FindS(option_str[j]); -- cgit v1.2.3 From 4194c9aee2766845618ef0431fd4803b0467aab7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 4 Jan 2014 16:23:32 +0100 Subject: improve error message --- apt-pkg/sourcelist.cc | 34 ++++++++++++++++++++++------------ apt-pkg/sourcelist.h | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 35e13b6f5..5e4a58e95 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -240,8 +240,14 @@ bool pkgSourceList::Read(string File) bool pkgSourceList::ReadAppend(string File) { if (_config->FindB("APT::Sources::Use-Deb822", true) == true) - if (ParseFileDeb822(File)) + { + int lines_parsed =ParseFileDeb822(File); + if (lines_parsed < 0) + return false; + else if (lines_parsed > 0) return true; + // no lines parsed ... fall through and use old style parser + } return ParseFileOldStyle(File); } @@ -303,10 +309,9 @@ bool pkgSourceList::ParseFileOldStyle(string File) /*}}}*/ // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ // --------------------------------------------------------------------- -/* */ -bool pkgSourceList::ParseFileDeb822(string File) +/* Returns: the number of stanzas parsed*/ +int pkgSourceList::ParseFileDeb822(string File) { - pkgTagSection Tags; map Options; unsigned int i=0; @@ -318,7 +323,7 @@ bool pkgSourceList::ParseFileDeb822(string File) if (_error->PendingError() == true) { _error->RevertToStack(); - return false; + return 0; } _error->MergeWithStack(); @@ -331,11 +336,19 @@ bool pkgSourceList::ParseFileDeb822(string File) string const type = Tags.FindS("Type"); Type *Parse = Type::GetType(type.c_str()); if (Parse == 0) - return _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + { + _error->Error(_("Type '%s' is not known on stanza %u in source list %s"),type.c_str(),i,Fd.Name().c_str()); + // true means we do not retry with old-style sources.list + return -1; + } string URI = Tags.FindS("URL"); if (!Parse->FixupURI(URI)) - return _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + { + _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); + // means we do not retry with old-style sources.list + return -1; + } string Dist = Tags.FindS("Dist"); Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); @@ -362,11 +375,8 @@ bool pkgSourceList::ParseFileDeb822(string File) i++; } - // we are done - if(i>0) - return true; - - return false; + // we are done, return the number of stanzas read + return i; } /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 5e0d585bb..d83c76d6a 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -76,7 +76,7 @@ class pkgSourceList std::vector SrcList; - bool ParseFileDeb822(std::string File); + int ParseFileDeb822(std::string File); bool ParseFileOldStyle(std::string File); public: -- cgit v1.2.3