Fix the newly introduced method GetListOfFilesInDir to not accept every

file if no extension is enforced (= restore old behaviour). (Closes: #565213)

This commit includes also:
* apt-pkg/policy.cc:
  - accept also partfiles with "pref" file extension as valid
* apt-pkg/contrib/configuration.cc:
  - accept also partfiles with "conf" file extension as valid
* doc/apt.conf.5.xml:
  - reorder description and split out syntax
  - add partfile name convention (Closes: #558348)
* doc/apt_preferences.conf.5.xml:
  - describe partfile name convention also here

And a lovely test application of course.

10 files changed, 267 insertions, 22 deletions

diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index 80b089fac..7588b041c 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -832,7 +832,7 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
 bool ReadConfigDir(Configuration &Conf,const string &Dir,
 		   bool const &AsSectional, unsigned const &Depth)
 {
-   vector<string> const List = GetListOfFilesInDir(Dir, "", true);
+   vector<string> const List = GetListOfFilesInDir(Dir, "conf", true, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index cce8a4512..da32983f1 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -202,8 +202,37 @@ bool FileExists(string File)
 /* If an extension is given only files with this extension are included
    in the returned vector, otherwise every "normal" file is included. */
 std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
-					bool const &SortList) 
+					bool const &SortList)
 {
+   return GetListOfFilesInDir(Dir, Ext, SortList, false);
+}
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList, bool const &AllowNoExt)
+{
+   std::vector<string> ext;
+   ext.reserve(2);
+   if (Ext.empty() == false)
+      ext.push_back(Ext);
+   if (AllowNoExt == true && ext.empty() == false)
+      ext.push_back("");
+   return GetListOfFilesInDir(Dir, ext, SortList);
+}
+std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
+					bool const &SortList)
+{
+   // Attention debuggers: need to be set with the environment config file!
+   bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false);
+   if (Debug == true)
+   {
+      std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl;
+      if (Ext.empty() == true)
+	 std::clog << "\tNO extension" << std::endl;
+      else
+	 for (std::vector<string>::const_iterator e = Ext.begin();
+	      e != Ext.end(); ++e)
+	    std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl;
+   }
+
    std::vector<string> List;
    DIR *D = opendir(Dir.c_str());
    if (D == 0) 
@@ -214,28 +243,73 @@ std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
 
    for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) 
    {
+      // skip "hidden" files
       if (Ent->d_name[0] == '.')
 	 continue;
 
-      if (Ext.empty() == false && flExtension(Ent->d_name) != Ext)
-	 continue;
+      // check for accepted extension:
+      // no extension given -> periods are bad as hell!
+      // extensions given -> "" extension allows no extension
+      if (Ext.empty() == false)
+      {
+	 string d_ext = flExtension(Ent->d_name);
+	 if (d_ext == Ent->d_name) // no extension
+	 {
+	    if (std::find(Ext.begin(), Ext.end(), "") == Ext.end())
+	    {
+	       if (Debug == true)
+		  std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl;
+	       continue;
+	    }
+	 }
+	 else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end())
+	 {
+	    if (Debug == true)
+	       std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl;
+	    continue;
+	 }
+      }
 
-      // Skip bad file names ala run-parts
+      // Skip bad filenames ala run-parts
       const char *C = Ent->d_name;
       for (; *C != 0; ++C)
 	 if (isalpha(*C) == 0 && isdigit(*C) == 0
-	     && *C != '_' && *C != '-' && *C != '.')
+	     && *C != '_' && *C != '-') {
+	    // no required extension -> dot is a bad character
+	    if (*C == '.' && Ext.empty() == false)
+	       continue;
 	    break;
+	 }
 
+      // we don't reach the end of the name -> bad character included
       if (*C != 0)
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → bad character »"
+	       << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl;
 	 continue;
+      }
+
+      // skip filenames which end with a period. These are never valid
+      if (*(C - 1) == '.')
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl;
+	 continue;
+      }
 
       // Make sure it is a file and not something else
       string const File = flCombine(Dir,Ent->d_name);
       struct stat St;
       if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0)
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → stat says not a good file" << std::endl;
 	 continue;
+      }
 
+      if (Debug == true)
+	 std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl;
       List.push_back(File);
    }
    closedir(D);
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index 2807c29d9..85a94898c 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -82,8 +82,13 @@ bool RunScripts(const char *Cnf);
 bool CopyFile(FileFd &From,FileFd &To);
 int GetLock(string File,bool Errors = true);
 bool FileExists(string File);
+// FIXME: next ABI-Break: merge the two method-headers
 std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
 					bool const &SortList);
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList, bool const &AllowNoExt);
+std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
+					bool const &SortList);
 string SafeGetCWD();
 void SetCloseExec(int Fd,bool Close);
 void SetNonBlock(int Fd,bool Block);
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 393181b6d..f9901bc9a 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -280,7 +280,7 @@ bool ReadPinDir(pkgPolicy &Plcy,string Dir)
       return true;
    }
 
-   vector<string> const List = GetListOfFilesInDir(Dir, "", true);
+   vector<string> const List = GetListOfFilesInDir(Dir, "pref", true, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)
diff --git a/debian/changelog b/debian/changelog
index ef6f846cb..caafde70c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -20,6 +20,24 @@ apt (0.7.26) UNRELEASED; urgency=low
 
  -- Michael Vogt <mvo@debian.org>  Thu, 10 Dec 2009 22:02:38 +0100
 
+apt (0.7.25.2) UNRELEASED; urgency=low
+
+  * apt-pkg/contrib/fileutl.cc:
+    - Fix the newly introduced method GetListOfFilesInDir to not
+      accept every file if no extension is enforced
+      (= restore old behaviour). (Closes: #565213)
+  * apt-pkg/policy.cc:
+    - accept also partfiles with "pref" file extension as valid
+  * apt-pkg/contrib/configuration.cc:
+    - accept also partfiles with "conf" file extension as valid
+  * doc/apt.conf.5.xml:
+    - reorder description and split out syntax
+    - add partfile name convention (Closes: #558348)
+  * doc/apt_preferences.conf.5.xml:
+    - describe partfile name convention also here
+
+ -- David Kalnischkies <kalnischkies@gmail.com>  Sat, 16 Jan 2010 21:06:38 +0100
+
 apt (0.7.25.1) unstable; urgency=low
 
   [ Christian Perrier ]
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index e568baa35..c13ad4867 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -21,7 +21,7 @@
    &apt-email;
    &apt-product;
    <!-- The last update date -->
-   <date>18 September 2009</date>
+   <date>16 January 2010</date>
  </refentryinfo>
  
  <refmeta>
@@ -37,16 +37,27 @@
  </refnamediv>
  
  <refsect1><title>Description</title>
-   <para><filename>apt.conf</filename> is the main configuration file for the APT suite of
-   tools, all tools make use of the configuration file and a common command line
-   parser to provide a uniform environment. When an APT tool starts up it will
-   read the configuration specified by the <envar>APT_CONFIG</envar> environment 
-   variable (if any) and then read the files in <literal>Dir::Etc::Parts</literal> 
-   then read the main configuration file specified by 
-   <literal>Dir::Etc::main</literal> then finally apply the
-   command line options to override the configuration directives, possibly 
-   loading even more config files.</para>
-
+ <para><filename>apt.conf</filename> is the main configuration file for
+   the APT suite of tools, but by far not the only place changes to options
+   can be made. All tools therefore share the configuration files and also
+   use a common command line parser to provide a uniform environment.</para>
+   <orderedlist>
+      <para>When an APT tool starts up it will read the configuration files
+      in the following order:</para>
+      <listitem><para>the file specified by the <envar>APT_CONFIG</envar>
+	 environment variable (if any)</para></listitem>
+      <listitem><para>all files in <literal>Dir::Etc::Parts</literal> in
+	 alphanumeric ascending order which have no or "<literal>conf</literal>"
+	 as filename extension and which only contain alphanumeric,
+	 hyphen (-), underscore (_) and period (.) characters -
+	 otherwise they will be silently ignored.</para></listitem>
+      <listitem><para>the main configuration file specified by
+	 <literal>Dir::Etc::main</literal></para></listitem>
+      <listitem><para>the command line options are applied to override the
+	 configuration directives or to load even more configuration files.</para></listitem>
+   </orderedlist>
+   </refsect1>
+   <refsect1><title>Syntax</title>
    <para>The configuration file is organized in a tree with options organized into
    functional groups. Option specification is given with a double colon
    notation, for instance <literal>APT::Get::Assume-Yes</literal> is an option within 
diff --git a/doc/apt_preferences.5.xml b/doc/apt_preferences.5.xml
index 159d61f2b..9a4791c08 100644
--- a/doc/apt_preferences.5.xml
+++ b/doc/apt_preferences.5.xml
@@ -53,6 +53,13 @@ earliest in the &sources-list; file.
 The APT preferences file does not affect the choice of instance, only
 the choice of version.</para>
 
+<para>Note that the files in the <filename>/etc/apt/preferences.d</filename>
+directory are parsed in alphanumeric ascending order and need to obey the
+following naming convention: The files have no or "<literal>pref</literal>"
+as filename extension and which only contain alphanumeric,  hyphen (-),
+underscore (_) and period (.) characters - otherwise they will be silently
+ignored.</para>
+
 <refsect2><title>APT's Default Priority Assignments</title>
 
 <para>If there is no preferences file or if there is no entry in the file
diff --git a/test/libapt/getlistoffilesindir_test.cc b/test/libapt/getlistoffilesindir_test.cc
new file mode 100644
index 000000000..ed8d2dad6
--- /dev/null
+++ b/test/libapt/getlistoffilesindir_test.cc
@@ -0,0 +1,82 @@
+#include <apt-pkg/fileutl.h>
+
+#include "assert.h"
+#include <string>
+#include <vector>
+
+#include <stdio.h>
+#include <iostream>
+
+// simple helper to quickly output a vector of strings
+void dumpVector(std::vector<std::string> vec) {
+	for (std::vector<std::string>::const_iterator v = vec.begin();
+	     v != vec.end(); v++)
+		std::cout << *v << std::endl;
+}
+
+#define P(x)	string(argv[1]).append("/").append(x)
+
+int main(int argc,char *argv[])
+{
+	if (argc != 2) {
+		std::cout << "One parameter expected - given " << argc << std::endl;
+		return 100;
+	}
+
+	// Files with no extension
+	std::vector<std::string> files = GetListOfFilesInDir(argv[1], "", true);
+	equals(files.size(), 2);
+	equals(files[0], P("01yet-anothernormalfile"));
+	equals(files[1], P("anormalfile"));
+
+	// Files with no extension - should be the same as above
+	files = GetListOfFilesInDir(argv[1], "", true, true);
+	equals(files.size(), 2);
+	equals(files[0], P("01yet-anothernormalfile"));
+	equals(files[1], P("anormalfile"));
+
+	// Files with impossible extension
+	files = GetListOfFilesInDir(argv[1], "impossible", true);
+	equals(files.size(), 0);
+
+	// Files with impossible or no extension
+	files = GetListOfFilesInDir(argv[1], "impossible", true, true);
+	equals(files.size(), 2);
+	equals(files[0], P("01yet-anothernormalfile"));
+	equals(files[1], P("anormalfile"));
+
+	// Files with list extension - nothing more
+	files = GetListOfFilesInDir(argv[1], "list", true);
+	equals(files.size(), 4);
+	equals(files[0], P("01yet-anotherapt.list"));
+	equals(files[1], P("anormalapt.list"));
+	equals(files[2], P("linkedfile.list"));
+	equals(files[3], P("multi.dot.list"));
+
+	// Files with conf or no extension
+	files = GetListOfFilesInDir(argv[1], "conf", true, true);
+	equals(files.size(), 5);
+	equals(files[0], P("01yet-anotherapt.conf"));
+	equals(files[1], P("01yet-anothernormalfile"));
+	equals(files[2], P("anormalapt.conf"));
+	equals(files[3], P("anormalfile"));
+	equals(files[4], P("multi.dot.conf"));
+
+	// Files with disabled extension - nothing more
+	files = GetListOfFilesInDir(argv[1], "disabled", true);
+	equals(files.size(), 3);
+	equals(files[0], P("disabledfile.conf.disabled"));
+	equals(files[1], P("disabledfile.disabled"));
+	equals(files[2], P("disabledfile.list.disabled"));
+
+	// Files with disabled or no extension
+	files = GetListOfFilesInDir(argv[1], "disabled", true, true);
+	equals(files.size(), 5);
+	equals(files[0], P("01yet-anothernormalfile"));
+	equals(files[1], P("anormalfile"));
+	equals(files[2], P("disabledfile.conf.disabled"));
+	equals(files[3], P("disabledfile.disabled"));
+	equals(files[4], P("disabledfile.list.disabled"));
+
+	return 0;
+}
diff --git a/test/libapt/makefile b/test/libapt/makefile
index 5712c025a..08f581e6d 100644
--- a/test/libapt/makefile
+++ b/test/libapt/makefile
@@ -17,3 +17,9 @@ PROGRAM = ParseDepends${BASENAME}
 SLIBS = -lapt-pkg
 SOURCE = parsedepends_test.cc
 include $(PROGRAM_H)
+
+# Program for testing GetListOfFilesInDir
+PROGRAM = GetListOfFilesInDir${BASENAME}
+SLIBS = -lapt-pkg
+SOURCE = getlistoffilesindir_test.cc
+include $(PROGRAM_H)
diff --git a/test/libapt/run-tests.sh b/test/libapt/run-tests.sh
index 365bbe215..1fcfb6861 100755
--- a/test/libapt/run-tests.sh
+++ b/test/libapt/run-tests.sh
@@ -1,10 +1,52 @@
 #!/bin/sh
+set -e
+
 echo "Compiling the tests ..."
 make
 echo "Running all testcases ..."
-PATH=$(pwd)/../../build/bin
-for testapp in $(/bin/ls ${PATH}/*_libapt_test)
+LDPATH=$(pwd)/../../build/bin
+EXT="_libapt_test"
+for testapp in $(ls ${LDPATH}/*$EXT)
 do
-	echo -n "Testing with \033[1;35m$(/usr/bin/basename ${testapp})\033[0m ... "
-	LD_LIBRARY_PATH=${PATH} ${testapp} && echo "\033[1;32mOKAY\033[0m" || echo "\033[1;31mFAILED\033[0m"
+	name=$(basename ${testapp})
+	tmppath=""
+
+	if [ $name = "GetListOfFilesInDir${EXT}" ]; then
+		# TODO: very-low: move env creation to the actual test-app
+		echo "Prepare Testarea for \033[1;35m$name\033[0m ..."
+		tmppath=$(mktemp -d)
+		touch "${tmppath}/anormalfile" \
+			"${tmppath}/01yet-anothernormalfile" \
+			"${tmppath}/anormalapt.conf" \
+			"${tmppath}/01yet-anotherapt.conf" \
+			"${tmppath}/anormalapt.list" \
+			"${tmppath}/01yet-anotherapt.list" \
+			"${tmppath}/wrongextension.wron" \
+			"${tmppath}/wrong-extension.wron" \
+			"${tmppath}/strangefile." \
+			"${tmppath}/s.t.r.a.n.g.e.f.i.l.e" \
+			"${tmppath}/.hiddenfile" \
+			"${tmppath}/.hiddenfile.conf" \
+			"${tmppath}/.hiddenfile.list" \
+			"${tmppath}/multi..dot" \
+			"${tmppath}/multi.dot.conf" \
+			"${tmppath}/multi.dot.list" \
+			"${tmppath}/disabledfile.disabled" \
+			"${tmppath}/disabledfile.conf.disabled" \
+			"${tmppath}/disabledfile.list.disabled" \
+			"${tmppath}/invälid.conf" \
+			"${tmppath}/invalíd" \
+			"${tmppath}/01invalíd"
+		ln -s "${tmppath}/anormalfile" "${tmppath}/linkedfile.list"
+		ln -s "${tmppath}/non-existing-file" "${tmppath}/brokenlink.list"
+	fi
+
+	echo -n "Testing with \033[1;35m${name}\033[0m ... "
+	LD_LIBRARY_PATH=${LDPATH} ${testapp} ${tmppath} && echo "\033[1;32mOKAY\033[0m" || echo "\033[1;31mFAILED\033[0m"
+
+	if [ -n "$tmppath" -a -d "$tmppath" ]; then
+		echo "Cleanup Testarea after \033[1;35m$name\033[0m ..."
+		rm -rf "$tmppath"
+	fi
+
 done