add optional support for comments in pkgTagFile

APT usually deals with perfectly formatted files generated automatically
be other programs – and as it has to parse multiple MBs of such files it
tries to be fast rather than forgiving.

This was always a problem if we reused this parser for files with a
deb822 syntax which are mostly written by hand however, like
apt_preferences or the deb822-style sources as these can include stray
newlines and more importantly comments all over the place.

As a stopgap we had pkgUserTagSection which deals at least with comments
before and after a given stanza, but comments in between weren't really
supported and now that we support parsing debian/control for e.g.
build-dep we face the full comment problem e.g. with comments inbetween
multi-line fields (like Build-Depends).

We can't easily deal with this on the pkgTagSection level as the interface
gives access to 'raw' char-pointers for performance reasons so we would
need to optionally add a buffer here on which we could remove comments
to hand out pointers into this buffer instead. The interface is quite
large already and supports writing stanzas as well, which does not
support comments at all either. So while in future it might make sense
to have a parser setup which deals with and keeps comments in this
commit we opt for the simpler solution for now: We officially declare
that pkgTagSection does not support comments and instead expect the
caller to deal with them, which in our case is pkgTagFile:

pkgTagFile is extended with an additional mode which can deal with
comments by dropping them from the buffer which will later form the
input of pkgTagSection. The actual implementation is slightly more
complex than this sentence suggests at first on one hand to have good
performance and on the other to allow jumping directly to stanzas with
offsets collected in a previous run (like our cache generation does it
for example).

1 files changed, 61 insertions, 0 deletions

diff --git a/test/libapt/tagfile_test.cc b/test/libapt/tagfile_test.cc
index 064e91b79..24fbd389e 100644
--- a/test/libapt/tagfile_test.cc
+++ b/test/libapt/tagfile_test.cc
@@ -40,6 +40,10 @@ TEST(TagFileTest,SingleField)
    EXPECT_EQ(0, section.Count());
    EXPECT_FALSE(section.Exists("FieldA-12345678"));
    EXPECT_FALSE(section.Exists("FieldB-12345678"));
+
+   createTemporaryFile("emptyfile", fd, NULL, NULL);
+   ASSERT_FALSE(tfile.Step(section));
+   EXPECT_EQ(0, section.Count());
 }
 
 TEST(TagFileTest,MultipleSections)
@@ -222,3 +226,60 @@ TEST(TagFileTest, SpacesEverywhere)
    // overridden values are still present, but not really accessible
    EXPECT_EQ(12, section.Count());
 }
+
+TEST(TagFileTest, Comments)
+{
+   FileFd fd;
+   createTemporaryFile("commentfile", fd, NULL, "# Leading comments should be ignored.\n"
+"\n"
+"Source: foo\n"
+"#Package: foo\n"
+"Section: bar\n"
+"#Section: overriden\n"
+"Priority: optional\n"
+"Build-Depends: debhelper,\n"
+"# apt-utils, (temporarily disabled)\n"
+" apt\n"
+"\n"
+"# Comments in the middle shouldn't result in extra blank paragraphs either.\n"
+"\n"
+"# Ditto.\n"
+"\n"
+"# A comment at the top of a paragraph should be ignored.\n"
+"Package: foo\n"
+"Architecture: any\n"
+"Description: An awesome package\n"
+"  # This should still appear in the result.\n"
+"# this one shouldn't\n"
+"  Blah, blah, blah. # but this again.\n"
+"# A comment at the end of a paragraph should be ignored.\n"
+"\n"
+"# Trailing comments shouldn't cause extra blank paragraphs."
+	 );
+
+   pkgTagFile tfile(&fd, pkgTagFile::SUPPORT_COMMENTS, 1);
+   pkgTagSection section;
+   EXPECT_TRUE(tfile.Step(section));
+   EXPECT_FALSE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("Source"));
+   EXPECT_EQ("foo", section.FindS("Source"));
+   EXPECT_TRUE(section.Exists("Section"));
+   EXPECT_EQ("bar", section.FindS("Section"));
+   EXPECT_TRUE(section.Exists("Priority"));
+   EXPECT_EQ("optional", section.FindS("Priority"));
+   EXPECT_TRUE(section.Exists("Build-Depends"));
+   EXPECT_EQ("debhelper,\n apt", section.FindS("Build-Depends"));
+
+   EXPECT_TRUE(tfile.Step(section));
+   EXPECT_FALSE(section.Exists("Source"));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_EQ("foo", section.FindS("Package"));
+   EXPECT_FALSE(section.Exists("Section"));
+   EXPECT_TRUE(section.Exists("Architecture"));
+   EXPECT_EQ("any", section.FindS("Architecture"));
+   EXPECT_FALSE(section.Exists("Build-Depends"));
+   EXPECT_TRUE(section.Exists("Description"));
+   EXPECT_EQ("An awesome package\n  # This should still appear in the result.\n  Blah, blah, blah. # but this again.", section.FindS("Description"));
+
+   EXPECT_FALSE(tfile.Step(section));
+}