From 81460e32961bb0b9922bf8a1a27d87705d8c3e51 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 21 Jun 2015 23:12:24 +0200 Subject: bring back deb822 sources.list entries as .sources Having two different formats in the same file is very dirty and causes external tools to fail hard trying to parse them. It is probably not a good idea for them to parse them in the first place, but they do and we shouldn't break them if there is a better way. So we solve this issue for now by giving our deb822 format a new filename extension ".sources" which unsupporting applications are likely to ignore an can begin gradually moving forward rather than waiting for the unknown applications to catch up. Currently and for the forseeable future apt is going to support both with the same feature set as documented in the manpage, with the longtime plan of adopting the 'new' format as default, but that is a long way to go and might get going more from having an easier time setting options than from us pushing it explicitely. --- apt-pkg/policy.cc | 19 +---- apt-pkg/sourcelist.cc | 228 +++++++++++++++++++++++++------------------------- apt-pkg/sourcelist.h | 16 ++-- apt-pkg/tagfile.cc | 8 ++ apt-pkg/tagfile.h | 8 ++ 5 files changed, 140 insertions(+), 139 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc index cd48e040c..170da7c63 100644 --- a/apt-pkg/policy.cc +++ b/apt-pkg/policy.cc @@ -396,21 +396,6 @@ APT_PURE signed short pkgPolicy::GetPriority(pkgCache::PkgFileIterator const &Fi return PFPriority[File->ID]; } /*}}}*/ -// PreferenceSection class - Overriding the default TrimRecord method /*{{{*/ -// --------------------------------------------------------------------- -/* The preference file is a user generated file so the parser should - therefore be a bit more friendly by allowing comments and new lines - all over the place rather than forcing a special format */ -class PreferenceSection : public pkgTagSection -{ - void TrimRecord(bool /*BeforeRecord*/, const char* &End) - { - for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++) - if (Stop[0] == '#') - Stop = (const char*) memchr(Stop,'\n',End-Stop); - } -}; - /*}}}*/ // ReadPinDir - Load the pin files from this dir into a Policy /*{{{*/ // --------------------------------------------------------------------- /* This will load each pin file in the given dir into a Policy. If the @@ -455,8 +440,8 @@ bool ReadPinFile(pkgPolicy &Plcy,string File) pkgTagFile TF(&Fd); if (_error->PendingError() == true) return false; - - PreferenceSection Tags; + + pkgUserTagSection Tags; while (TF.Step(Tags) == true) { // can happen when there are only comments in a record diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 6ef99863d..69f7ac043 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -82,15 +82,15 @@ bool pkgSourceList::Type::FixupURI(string &URI) const return true; } /*}}}*/ -bool pkgSourceList::Type::ParseStanza(vector &List, +bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ pkgTagSection &Tags, - int i, + unsigned int const i, FileFd &Fd) { map Options; string Enabled = Tags.FindS("Enabled"); - if (Enabled.size() > 0 && StringToBool(Enabled) == false) + if (Enabled.empty() == false && StringToBool(Enabled) == false) return true; std::map mapping; @@ -115,46 +115,63 @@ bool pkgSourceList::Type::ParseStanza(vector &List, // now create one item per suite/section string Suite = Tags.FindS("Suites"); Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); - string const Section = Tags.FindS("Sections"); - string URIS = Tags.FindS("URIs"); + string const Component = Tags.FindS("Components"); + string const URIS = Tags.FindS("URIs"); + + std::vector const list_uris = VectorizeString(URIS, ' '); + std::vector const list_suite = VectorizeString(Suite, ' '); + std::vector const list_comp = VectorizeString(Component, ' '); + + if (list_uris.empty()) + // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI"); - std::vector list_uris = StringSplit(URIS, " "); - std::vector list_dist = StringSplit(Suite, " "); - std::vector list_section = StringSplit(Section, " "); - for (std::vector::const_iterator U = list_uris.begin(); U != list_uris.end(); ++U) { - std::string URI = (*U); - if (!FixupURI(URI)) - { - _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - return false; - } + std::string URI = *U; + if (U->empty() || FixupURI(URI) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse"); + + if (list_suite.empty()) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite"); - for (std::vector::const_iterator I = list_dist.begin(); - I != list_dist.end(); ++I) + for (std::vector::const_iterator S = list_suite.begin(); + S != list_suite.end(); ++S) { - for (std::vector::const_iterator J = list_section.begin(); - J != list_section.end(); ++J) - { - if (CreateItem(List, URI, (*I), (*J), Options) == false) - { - return false; - } - } + if (S->empty() == false && (*S)[S->size() - 1] == '/') + { + if (list_comp.empty() == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component"); + if (CreateItem(List, URI, *S, "", Options) == false) + return false; + } + else + { + if (list_comp.empty()) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component"); + + for (std::vector::const_iterator C = list_comp.begin(); + C != list_comp.end(); ++C) + { + if (CreateItem(List, URI, *S, *C, Options) == false) + { + return false; + } + } + } } } return true; } - + /*}}}*/ // Type::ParseLine - Parse a single line /*{{{*/ // --------------------------------------------------------------------- /* This is a generic one that is the 'usual' format for sources.list Weird types may override this. */ bool pkgSourceList::Type::ParseLine(vector &List, const char *Buffer, - unsigned long const &CurLine, + unsigned int const CurLine, string const &File) const { for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces @@ -171,10 +188,10 @@ bool pkgSourceList::Type::ParseLine(vector &List, // get one option, e.g. option1=value1 string option; if (ParseQuoteWord(Buffer,option) == false) - return _error->Error(_("Malformed line %lu in source list %s ([option] unparseable)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable"); if (option.length() < 3) - return _error->Error(_("Malformed line %lu in source list %s ([option] too short)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short"); // accept options even if the last has no space before the ]-end marker if (option.at(option.length()-1) == ']') @@ -185,16 +202,16 @@ bool pkgSourceList::Type::ParseLine(vector &List, size_t const needle = option.find('='); if (needle == string::npos) - return _error->Error(_("Malformed line %lu in source list %s ([%s] is not an assignment)"),CurLine,File.c_str(), option.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment"); string const key = string(option, 0, needle); string const value = string(option, needle + 1, option.length()); if (key.empty() == true) - return _error->Error(_("Malformed line %lu in source list %s ([%s] has no key)"),CurLine,File.c_str(), option.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key"); if (value.empty() == true) - return _error->Error(_("Malformed line %lu in source list %s ([%s] key %s has no value)"),CurLine,File.c_str(),option.c_str(),key.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value"); Options[key] = value; } @@ -207,33 +224,33 @@ bool pkgSourceList::Type::ParseLine(vector &List, string Section; if (ParseQuoteWord(Buffer,URI) == false) - return _error->Error(_("Malformed line %lu in source list %s (URI)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI"); if (ParseQuoteWord(Buffer,Dist) == false) - return _error->Error(_("Malformed line %lu in source list %s (dist)"),CurLine,File.c_str()); - + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite"); + if (FixupURI(URI) == false) - return _error->Error(_("Malformed line %lu in source list %s (URI parse)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse"); // Check for an absolute dists specification. if (Dist.empty() == false && Dist[Dist.size() - 1] == '/') { if (ParseQuoteWord(Buffer,Section) == true) - return _error->Error(_("Malformed line %lu in source list %s (absolute dist)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component"); Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); return CreateItem(List, URI, Dist, Section, Options); } - + // Grab the rest of the dists if (ParseQuoteWord(Buffer,Section) == false) - return _error->Error(_("Malformed line %lu in source list %s (dist parse)"),CurLine,File.c_str()); - + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component"); + do { if (CreateItem(List, URI, Dist, Section, Options) == false) return false; } while (ParseQuoteWord(Buffer,Section) == true); - + return true; } /*}}}*/ @@ -242,11 +259,6 @@ bool pkgSourceList::Type::ParseLine(vector &List, /* */ pkgSourceList::pkgSourceList() : d(NULL) { -} - -pkgSourceList::pkgSourceList(string File) : d(NULL) -{ - Read(File); } /*}}}*/ // SourceList::~pkgSourceList - Destructor /*{{{*/ @@ -305,7 +317,7 @@ void pkgSourceList::Reset() // SourceList::Read - Parse the sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::Read(string File) +bool pkgSourceList::Read(string const &File) { Reset(); return ReadAppend(File); @@ -314,71 +326,63 @@ bool pkgSourceList::Read(string File) // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::ReadAppend(string File) +bool pkgSourceList::ReadAppend(string const &File) { - if (_config->FindB("APT::Sources::Use-Deb822", false) == true) - { - int lines_parsed =ParseFileDeb822(File); - if (lines_parsed < 0) - return false; - else if (lines_parsed > 0) - return true; - // no lines parsed ... fall through and use old style parser - } - return ParseFileOldStyle(File); + if (flExtension(File) == "sources") + return ParseFileDeb822(File); + else + return ParseFileOldStyle(File); } // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::ParseFileOldStyle(string File) +bool pkgSourceList::ParseFileOldStyle(std::string const &File) { // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (F.fail() == true) return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str()); - // CNC:2003-12-10 - 300 is too short. - char Buffer[1024]; - - int CurLine = 0; - while (F.eof() == false) + std::string Buffer; + for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine) { - F.getline(Buffer,sizeof(Buffer)); - CurLine++; - _strtabexpand(Buffer,sizeof(Buffer)); - if (F.fail() && !F.eof()) - return _error->Error(_("Line %u too long in source list %s."), - CurLine,File.c_str()); - - - char *I; - // CNC:2003-02-20 - Do not break if '#' is inside []. - for (I = Buffer; *I != 0 && *I != '#'; I++) - if (*I == '[') - { - char *b_end = strchr(I + 1, ']'); - if (b_end != NULL) - I = b_end; - } - *I = 0; - - const char *C = _strstrip(Buffer); - - // Comment or blank - if (C[0] == '#' || C[0] == 0) + // remove comments + size_t curpos = 0; + while ((curpos = Buffer.find('#', curpos)) != std::string::npos) + { + size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '['); + size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']'); + if (openbrackets > closedbrackets) + { + // a # in an option, unlikely, but oh well, it was supported so stick to it + ++curpos; + continue; + } + Buffer.erase(curpos); + break; + } + // remove spaces before/after + curpos = Buffer.find_first_not_of(" \t\r"); + if (curpos != 0) + Buffer.erase(0, curpos); + curpos = Buffer.find_last_not_of(" \t\r"); + if (curpos != std::string::npos) + Buffer.erase(curpos + 1); + + if (Buffer.empty()) continue; - + // Grok it - string LineType; - if (ParseQuoteWord(C,LineType) == false) + std::string const LineType = Buffer.substr(0, Buffer.find(' ')); + if (LineType.empty() || LineType == Buffer) return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str()); Type *Parse = Type::GetType(LineType.c_str()); if (Parse == 0) return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str()); - - if (Parse->ParseLine(SrcList, C, CurLine, File) == false) + + if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false) return false; } return true; @@ -387,30 +391,25 @@ bool pkgSourceList::ParseFileOldStyle(string File) // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ // --------------------------------------------------------------------- /* Returns: the number of stanzas parsed*/ -int pkgSourceList::ParseFileDeb822(string File) +bool pkgSourceList::ParseFileDeb822(string const &File) { - pkgTagSection Tags; - unsigned int i=0; + pkgUserTagSection Tags; + unsigned int i = 1; // see if we can read the file - _error->PushToStack(); FileFd Fd(File, FileFd::ReadOnly); pkgTagFile Sources(&Fd); if (_error->PendingError() == true) - { - _error->RevertToStack(); - return 0; - } - _error->MergeWithStack(); - + return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str()); + // read step by step while (Sources.Step(Tags) == true) { - if(!Tags.Exists("Types")) - continue; + if(Tags.Exists("Types") == false) + return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str()); string const types = Tags.FindS("Types"); - std::vector list_types = StringSplit(types, " "); + std::vector const list_types = VectorizeString(types, ' '); for (std::vector::const_iterator I = list_types.begin(); I != list_types.end(); ++I) { @@ -418,18 +417,16 @@ int pkgSourceList::ParseFileDeb822(string File) if (Parse == 0) { _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str()); - return -1; + return false; } - + if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) - return -1; + return false; - i++; + ++i; } } - - // we are done, return the number of stanzas read - return i; + return true; } /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ @@ -471,9 +468,12 @@ bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const // Based on ReadConfigDir() /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::ReadSourceDir(string Dir) +bool pkgSourceList::ReadSourceDir(string const &Dir) { - vector const List = GetListOfFilesInDir(Dir, "list", true); + std::vector ext; + ext.push_back("list"); + ext.push_back("sources"); + std::vector const List = GetListOfFilesInDir(Dir, ext, true); // Read the files for (vector::const_iterator I = List.begin(); I != List.end(); ++I) diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 4f42b3e91..079f3cb3d 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -72,11 +72,11 @@ class pkgSourceList bool FixupURI(std::string &URI) const; virtual bool ParseStanza(std::vector &List, pkgTagSection &Tags, - int stanza_n, + unsigned int const stanza_n, FileFd &Fd); virtual bool ParseLine(std::vector &List, const char *Buffer, - unsigned long const &CurLine,std::string const &File) const; + unsigned int const CurLine,std::string const &File) const; virtual bool CreateItem(std::vector &List,std::string const &URI, std::string const &Dist,std::string const &Section, std::map const &Options) const = 0; @@ -90,18 +90,19 @@ class pkgSourceList std::vector SrcList; - int ParseFileDeb822(std::string File); - bool ParseFileOldStyle(std::string File); + private: + APT_HIDDEN bool ParseFileDeb822(std::string const &File); + APT_HIDDEN bool ParseFileOldStyle(std::string const &File); public: bool ReadMainList(); - bool Read(std::string File); + bool Read(std::string const &File); // CNC:2003-03-03 void Reset(); - bool ReadAppend(std::string File); - bool ReadSourceDir(std::string Dir); + bool ReadAppend(std::string const &File); + bool ReadSourceDir(std::string const &Dir); // List accessors inline const_iterator begin() const {return SrcList.begin();}; @@ -117,7 +118,6 @@ class pkgSourceList time_t GetLastModifiedTime(); pkgSourceList(); - explicit pkgSourceList(std::string File); virtual ~pkgSourceList(); }; diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 6d7d8185b..cc63b213f 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -775,6 +775,14 @@ bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::v } /*}}}*/ +void pkgUserTagSection::TrimRecord(bool /*BeforeRecord*/, const char* &End)/*{{{*/ +{ + for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++) + if (Stop[0] == '#') + Stop = (const char*) memchr(Stop,'\n',End-Stop); +} + /*}}}*/ + #include "tagfile-order.c" // TFRewrite - Rewrite a control record /*{{{*/ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index d0d0c7a84..81fff89f0 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -142,6 +142,14 @@ class pkgTagSection bool Write(FileFd &File, char const * const * const Order = NULL, std::vector const &Rewrite = std::vector()) const; }; + +/* For user generated file the parser should be a bit more relaxed in exchange + for being a bit slower to allow comments and new lines all over the place */ +class pkgUserTagSection : public pkgTagSection +{ + virtual void TrimRecord(bool BeforeRecord, const char* &End); +}; + class pkgTagFilePrivate; class pkgTagFile { -- cgit v1.2.3