From 435704925edae0801103eb12b9f2bbce89d74769 Mon Sep 17 00:00:00 2001 From: Daniel Burrows Date: Mon, 31 Mar 2008 21:54:01 -0700 Subject: When a line in a config file is too long, say which config file it is. --- apt-pkg/contrib/configuration.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index da57c2054..a523b513e 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -512,7 +512,8 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, CurLine++; // This should be made to work instead, but this is better than looping if (F.fail() && !F.eof()) - return _error->Error(_("Line %d too long (max %lu)"), CurLine, sizeof(Buffer)); + return _error->Error(_("%s: Line %d too long (max %lu)"), + FName.c_str(), CurLine, sizeof(Buffer)); _strtabexpand(Buffer,sizeof(Buffer)); _strstrip(Buffer); -- cgit v1.2.3 From 3f807f6c39dd6056fca04264ae896990a83c7244 Mon Sep 17 00:00:00 2001 From: Daniel Burrows Date: Tue, 1 Apr 2008 20:30:58 -0700 Subject: Lift the 1024-character limit on configuration file lines. To do this, I changed the configuration parser so that instead of reading the line into a buffer and munging it in-place, it builds a string containing the current line (via successive getline() calls) and then walks down that string. Instead of skipping comments and whitespace by erasing them from the buffer, the new code just skips over them. --- apt-pkg/contrib/configuration.cc | 166 +++++++++++++++++++++++++++------------ 1 file changed, 117 insertions(+), 49 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index a523b513e..3140be446 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -495,8 +495,7 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, ifstream F(FName.c_str(),ios::in); if (!F != 0) return _error->Errno("ifstream::ifstream",_("Opening configuration file %s"),FName.c_str()); - - char Buffer[1024]; + string LineBuffer; string Stack[100]; unsigned int StackPos = 0; @@ -508,24 +507,58 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, bool InComment = false; while (F.eof() == false) { - F.getline(Buffer,sizeof(Buffer)); + // The raw input line. + std::string Input; + // The input line with comments stripped. + std::string Fragment; + do + { + char *Buffer = new char[1024]; + + F.clear(); + F.getline(Buffer,sizeof(Buffer) / 2); + + Input += Buffer; + } + while (F.fail() && !F.eof()); + + { + // Allocate enough space to expand an entire line of tabs + // below. + const int BufferSize = Input.size() * 8 + 1; + char *Buffer = new char[BufferSize]; + try + { + memcpy(Buffer, Input.c_str(), Input.size() + 1); + + _strtabexpand(Buffer, BufferSize); + _strstrip(Buffer); + Input = Buffer; + } + catch(...) + { + delete[] Buffer; + throw; + } + delete[] Buffer; + } CurLine++; - // This should be made to work instead, but this is better than looping - if (F.fail() && !F.eof()) - return _error->Error(_("%s: Line %d too long (max %lu)"), - FName.c_str(), CurLine, sizeof(Buffer)); - _strtabexpand(Buffer,sizeof(Buffer)); - _strstrip(Buffer); + // The first meaningful character in the current fragment; will + // be adjusted below as we remove bytes from the front. + std::string::const_iterator Start = Input.begin(); + // The last meaningful character in the current fragment. + std::string::const_iterator End = Input.end(); // Multi line comment if (InComment == true) { - for (const char *I = Buffer; *I != 0; I++) + for (std::string::const_iterator I = Start; + I != End; ++I) { - if (*I == '*' && I[1] == '/') + if (*I == '*' && I + 1 != End && I[1] == '/') { - memmove(Buffer,I+2,strlen(I+2) + 1); + Start = I + 2; InComment = false; break; } @@ -536,76 +569,94 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, // Discard single line comments bool InQuote = false; - for (char *I = Buffer; *I != 0; I++) + for (std::string::const_iterator I = Start; + I != End; ++I) { if (*I == '"') InQuote = !InQuote; if (InQuote == true) continue; - if (*I == '/' && I[1] == '/') + if (*I == '/' && I + 1 != End && I[1] == '/') { - *I = 0; + End = I; break; } } - // Look for multi line comments + // Look for multi line comments and build up the + // fragment. + Fragment.reserve(End - Start); InQuote = false; - for (char *I = Buffer; *I != 0; I++) + for (std::string::const_iterator I = Start; + I != End; ++I) { if (*I == '"') InQuote = !InQuote; if (InQuote == true) - continue; - - if (*I == '/' && I[1] == '*') + Fragment.push_back(*I); + else if (*I == '/' && I + 1 != End && I[1] == '*') { InComment = true; - for (char *J = Buffer; *J != 0; J++) + for (std::string::const_iterator J = I; + J != End; ++J) { - if (*J == '*' && J[1] == '/') + if (*J == '*' && J + 1 != End && J[1] == '/') { - memmove(I,J+2,strlen(J+2) + 1); + // Pretend we just finished walking over the + // comment, and don't add anything to the output + // fragment. + I = J + 1; InComment = false; break; } } if (InComment == true) - { - *I = 0; - break; - } + break; } + else + Fragment.push_back(*I); } - - // Blank - if (Buffer[0] == 0) + + // Skip blank lines. + if (Fragment.empty()) continue; - // We now have a valid line fragment + // We now have a valid line fragment. Walk down it and + // interpret it. InQuote = false; - for (char *I = Buffer; *I != 0;) + Start = Fragment.begin(); + End = Fragment.end(); + for (std::string::const_iterator I = Start; + I != End; ++I) { + if(Start > End) + { + _error->Error("Why is Start > End?"); + } + if (*I == '"') InQuote = !InQuote; if (InQuote == false && (*I == '{' || *I == ';' || *I == '}')) { // Put the last fragment into the buffer - char *Start = Buffer; - char *Stop = I; - for (; Start != I && isspace(*Start) != 0; Start++); - for (; Stop != Start && isspace(Stop[-1]) != 0; Stop--); - if (LineBuffer.empty() == false && Stop - Start != 0) + std::string::const_iterator FirstNonWhitespace = Start; + std::string::const_iterator LastNonWhitespace = I; + for (; FirstNonWhitespace != I && isspace(*FirstNonWhitespace) != 0; FirstNonWhitespace++) + ; + for (; LastNonWhitespace != FirstNonWhitespace && isspace(LastNonWhitespace[-1]) != 0; LastNonWhitespace--) + ; + if (LineBuffer.empty() == false && LastNonWhitespace - FirstNonWhitespace != 0) LineBuffer += ' '; - LineBuffer += string(Start,Stop - Start); - - // Remove the fragment + LineBuffer += string(FirstNonWhitespace, LastNonWhitespace); + + // Drop this from the input string, saving the character + // that terminated the construct we just closed. (i.e., a + // brace or a semicolon) char TermChar = *I; - memmove(Buffer,I + 1,strlen(I + 1) + 1); - I = Buffer; + Start = I + 1; // Syntax Error if (TermChar == '{' && LineBuffer.empty() == true) @@ -727,15 +778,32 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, } } - else - I++; } - // Store the fragment - const char *Stripd = _strstrip(Buffer); - if (*Stripd != 0 && LineBuffer.empty() == false) - LineBuffer += " "; - LineBuffer += Stripd; + // Store the remaining text, if any, in the current line buffer. + + // NB: could change this to use string-based operations; I'm + // using strstrip now to ensure backwards compatibility. + // -- dburrows 2008-04-01 + { + char *Buffer = new char[End - Start + 1]; + try + { + std::copy(Start, End, Buffer); + Buffer[End - Start] = '\0'; + + const char *Stripd = _strstrip(Buffer); + if (*Stripd != 0 && LineBuffer.empty() == false) + LineBuffer += " "; + LineBuffer += Stripd; + } + catch(...) + { + delete[] Buffer; + throw; + } + delete[] Buffer; + } } if (LineBuffer.empty() == false) -- cgit v1.2.3 From 8a27f5da85830fe500077891c5db1c77ae1773a6 Mon Sep 17 00:00:00 2001 From: Daniel Burrows Date: Tue, 1 Apr 2008 20:49:58 -0700 Subject: Improve some comments in the configuration parser. --- apt-pkg/contrib/configuration.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index 3140be446..6f617e914 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -511,6 +511,8 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, std::string Input; // The input line with comments stripped. std::string Fragment; + + // Grab the next line of F and place it in Input. do { char *Buffer = new char[1024]; @@ -522,9 +524,9 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, } while (F.fail() && !F.eof()); + // Expand tabs in the input line and remove leading and trailing + // whitespace. { - // Allocate enough space to expand an entire line of tabs - // below. const int BufferSize = Input.size() * 8 + 1; char *Buffer = new char[BufferSize]; try @@ -544,6 +546,9 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, } CurLine++; + // Now strip comments; if the whole line is contained in a + // comment, skip this line. + // The first meaningful character in the current fragment; will // be adjusted below as we remove bytes from the front. std::string::const_iterator Start = Input.begin(); @@ -623,8 +628,7 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, if (Fragment.empty()) continue; - // We now have a valid line fragment. Walk down it and - // interpret it. + // The line has actual content; interpret what it means. InQuote = false; Start = Fragment.begin(); End = Fragment.end(); -- cgit v1.2.3 From faa2bd8278cf8118e852dd3a1a8da93b642b60ac Mon Sep 17 00:00:00 2001 From: Daniel Burrows Date: Tue, 1 Apr 2008 20:54:16 -0700 Subject: Remove a debugging message that I left in by accident (if it ever triggers the program is about to crash anyway). --- apt-pkg/contrib/configuration.cc | 5 ----- 1 file changed, 5 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index 6f617e914..843e91f2a 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -635,11 +635,6 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, for (std::string::const_iterator I = Start; I != End; ++I) { - if(Start > End) - { - _error->Error("Why is Start > End?"); - } - if (*I == '"') InQuote = !InQuote; -- cgit v1.2.3 From a4dcbf1ce5fea58b54f37c5db9d8ba2c35b78eb4 Mon Sep 17 00:00:00 2001 From: Daniel Burrows Date: Tue, 1 Apr 2008 20:55:09 -0700 Subject: Improve the names used for the non-whitespace iterators. --- apt-pkg/contrib/configuration.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index 843e91f2a..7326b84ea 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -641,15 +641,15 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, if (InQuote == false && (*I == '{' || *I == ';' || *I == '}')) { // Put the last fragment into the buffer - std::string::const_iterator FirstNonWhitespace = Start; - std::string::const_iterator LastNonWhitespace = I; - for (; FirstNonWhitespace != I && isspace(*FirstNonWhitespace) != 0; FirstNonWhitespace++) + std::string::const_iterator NonWhitespaceStart = Start; + std::string::const_iterator NonWhitespaceStop = I; + for (; NonWhitespaceStart != I && isspace(*NonWhitespaceStart) != 0; NonWhitespaceStart++) ; - for (; LastNonWhitespace != FirstNonWhitespace && isspace(LastNonWhitespace[-1]) != 0; LastNonWhitespace--) + for (; NonWhitespaceStop != NonWhitespaceStart && isspace(NonWhitespaceStop[-1]) != 0; NonWhitespaceStop--) ; - if (LineBuffer.empty() == false && LastNonWhitespace - FirstNonWhitespace != 0) + if (LineBuffer.empty() == false && NonWhitespaceStop - NonWhitespaceStart != 0) LineBuffer += ' '; - LineBuffer += string(FirstNonWhitespace, LastNonWhitespace); + LineBuffer += string(NonWhitespaceStart, NonWhitespaceStop); // Drop this from the input string, saving the character // that terminated the construct we just closed. (i.e., a -- cgit v1.2.3 From ffc36991800aa50902d91b818c4422893e34a2c2 Mon Sep 17 00:00:00 2001 From: Daniel Burrows Date: Wed, 2 Apr 2008 09:06:49 -0700 Subject: Fix ReadMessages to handle messages that are longer than 64000 bytes. There's a bug in this code that needs to be fixed, but it existed prior to this change. --- apt-pkg/contrib/strutl.cc | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index a04c266ba..eacc7077a 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -658,11 +658,24 @@ string TimeRFC1123(time_t Date) // --------------------------------------------------------------------- /* This pulls full messages from the input FD into the message buffer. It assumes that messages will not pause during transit so no - fancy buffering is used. */ + fancy buffering is used. + + In particular: this reads blocks from the input until it believes + that it's run out of input text. Each block is terminated by a + double newline ('\n' followed by '\n'). As noted below, there is a + bug in this code: it assumes that all the blocks have been read if + it doesn't see additional text in the buffer after the last one is + parsed, which will cause it to lose blocks if the last block + coincides with the end of the buffer. + */ bool ReadMessages(int Fd, vector &List) { char Buffer[64000]; char *End = Buffer; + // Represents any left-over from the previous iteration of the + // parse loop. (i.e., if a message is split across the end + // of the buffer, it goes here) + string PartialMessage; while (1) { @@ -690,6 +703,7 @@ bool ReadMessages(int Fd, vector &List) // Pull the message out string Message(Buffer,I-Buffer); + PartialMessage += Message; // Fix up the buffer for (; I < End && *I == '\n'; I++); @@ -697,10 +711,32 @@ bool ReadMessages(int Fd, vector &List) memmove(Buffer,I,End-Buffer); I = Buffer; - List.push_back(Message); + List.push_back(PartialMessage); + PartialMessage.clear(); } - if (End == Buffer) - return true; + if (End != Buffer) + { + // If there's text left in the buffer, store it + // in PartialMessage and throw the rest of the buffer + // away. This allows us to handle messages that + // are longer than the static buffer size. + PartialMessage += string(Buffer, End); + End = Buffer; + } + else + { + // BUG ALERT: if a message block happens to end at a + // multiple of 64000 characters, this will cause it to + // terminate early, leading to a badly formed block and + // probably crashing the method. However, this is the only + // way we have to find the end of the message block. I have + // an idea of how to fix this, but it will require changes + // to the protocol (essentially to mark the beginning and + // end of the block). + // + // -- dburrows 2008-04-02 + return true; + } if (WaitFd(Fd) == false) return false; -- cgit v1.2.3