* apt-pkg/contrib/fileutl.cc:

  - Fix the newly introduced method GetListOfFilesInDir to not
    accept every file if no extension is enforced
    (= restore old behaviour). (Closes: #565213)
* apt-pkg/policy.cc:
  - accept also partfiles with "pref" file extension as valid
* apt-pkg/contrib/configuration.cc:
  - accept also partfiles with "conf" file extension as valid
* doc/apt.conf.5.xml:
  - reorder description and split out syntax
  - add partfile name convention (Closes: #558348)
* doc/apt_preferences.conf.5.xml:
  - describe partfile name convention also here
* apt-pkg/deb/dpkgpm.cc:
  - don't segfault if term.log file can't be opened.
    Thanks Sam Brightman for the patch! (Closes: #475770)
* doc/*:
  - replace the per language addendum with a global addendum
  - add a explanation why translations include (maybe) english
    parts to the new global addendum (Closes: #561636)
* apt-pkg/contrib/strutl.cc:
  - fix malloc asseration fail with ja_JP.eucJP locale in
    apt-cache search. Thanks Kusanagi Kouichi! (Closes: #548884)

8 files changed, 151 insertions, 30 deletions

diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index aaa669ffc..8e3b84499 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -832,7 +832,7 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional,
 bool ReadConfigDir(Configuration &Conf,const string &Dir,
 		   bool AsSectional, unsigned Depth)
 {
-   vector<string> const List = GetListOfFilesInDir(Dir, "", true);
+   vector<string> const List = GetListOfFilesInDir(Dir, "conf", true, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)
diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h
index a3be6a575..86aa9eca3 100644
--- a/apt-pkg/contrib/error.h
+++ b/apt-pkg/contrib/error.h
@@ -53,6 +53,8 @@
     
 #include <string>
 
+#include <system.h>
+
 using std::string;
 
 class GlobalError
@@ -71,13 +73,13 @@ class GlobalError
    public:
 
    // Call to generate an error from a library call.
-   bool Errno(const char *Function,const char *Description,...) APT_MFORMAT2;
-   bool WarningE(const char *Function,const char *Description,...) APT_MFORMAT2;
+   bool Errno(const char *Function,const char *Description,...) APT_MFORMAT2 __cold;
+   bool WarningE(const char *Function,const char *Description,...) APT_MFORMAT2 __cold;
 
    /* A warning should be considered less severe than an error, and may be
       ignored by the client. */
-   bool Error(const char *Description,...) APT_MFORMAT1;
-   bool Warning(const char *Description,...) APT_MFORMAT1;
+   bool Error(const char *Description,...) APT_MFORMAT1 __cold;
+   bool Warning(const char *Description,...) APT_MFORMAT1 __cold;
 
    // Simple accessors
    inline bool PendingError() {return PendingFlag;};
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index cce8a4512..da32983f1 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -202,8 +202,37 @@ bool FileExists(string File)
 /* If an extension is given only files with this extension are included
    in the returned vector, otherwise every "normal" file is included. */
 std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
-					bool const &SortList) 
+					bool const &SortList)
 {
+   return GetListOfFilesInDir(Dir, Ext, SortList, false);
+}
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList, bool const &AllowNoExt)
+{
+   std::vector<string> ext;
+   ext.reserve(2);
+   if (Ext.empty() == false)
+      ext.push_back(Ext);
+   if (AllowNoExt == true && ext.empty() == false)
+      ext.push_back("");
+   return GetListOfFilesInDir(Dir, ext, SortList);
+}
+std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
+					bool const &SortList)
+{
+   // Attention debuggers: need to be set with the environment config file!
+   bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false);
+   if (Debug == true)
+   {
+      std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl;
+      if (Ext.empty() == true)
+	 std::clog << "\tNO extension" << std::endl;
+      else
+	 for (std::vector<string>::const_iterator e = Ext.begin();
+	      e != Ext.end(); ++e)
+	    std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl;
+   }
+
    std::vector<string> List;
    DIR *D = opendir(Dir.c_str());
    if (D == 0) 
@@ -214,28 +243,73 @@ std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
 
    for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) 
    {
+      // skip "hidden" files
       if (Ent->d_name[0] == '.')
 	 continue;
 
-      if (Ext.empty() == false && flExtension(Ent->d_name) != Ext)
-	 continue;
+      // check for accepted extension:
+      // no extension given -> periods are bad as hell!
+      // extensions given -> "" extension allows no extension
+      if (Ext.empty() == false)
+      {
+	 string d_ext = flExtension(Ent->d_name);
+	 if (d_ext == Ent->d_name) // no extension
+	 {
+	    if (std::find(Ext.begin(), Ext.end(), "") == Ext.end())
+	    {
+	       if (Debug == true)
+		  std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl;
+	       continue;
+	    }
+	 }
+	 else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end())
+	 {
+	    if (Debug == true)
+	       std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl;
+	    continue;
+	 }
+      }
 
-      // Skip bad file names ala run-parts
+      // Skip bad filenames ala run-parts
       const char *C = Ent->d_name;
       for (; *C != 0; ++C)
 	 if (isalpha(*C) == 0 && isdigit(*C) == 0
-	     && *C != '_' && *C != '-' && *C != '.')
+	     && *C != '_' && *C != '-') {
+	    // no required extension -> dot is a bad character
+	    if (*C == '.' && Ext.empty() == false)
+	       continue;
 	    break;
+	 }
 
+      // we don't reach the end of the name -> bad character included
       if (*C != 0)
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → bad character »"
+	       << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl;
 	 continue;
+      }
+
+      // skip filenames which end with a period. These are never valid
+      if (*(C - 1) == '.')
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl;
+	 continue;
+      }
 
       // Make sure it is a file and not something else
       string const File = flCombine(Dir,Ent->d_name);
       struct stat St;
       if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0)
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → stat says not a good file" << std::endl;
 	 continue;
+      }
 
+      if (Debug == true)
+	 std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl;
       List.push_back(File);
    }
    closedir(D);
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index 2807c29d9..85a94898c 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -82,8 +82,13 @@ bool RunScripts(const char *Cnf);
 bool CopyFile(FileFd &From,FileFd &To);
 int GetLock(string File,bool Errors = true);
 bool FileExists(string File);
+// FIXME: next ABI-Break: merge the two method-headers
 std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
 					bool const &SortList);
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList, bool const &AllowNoExt);
+std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
+					bool const &SortList);
 string SafeGetCWD();
 void SetCloseExec(int Fd,bool Close);
 void SetNonBlock(int Fd,bool Block);
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index 4c05f2df8..2913fbf44 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -43,9 +43,10 @@ bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest)
 {
   iconv_t cd;
   const char *inbuf;
-  char *inptr, *outbuf, *outptr;
-  size_t insize, outsize;
-  
+  char *inptr, *outbuf;
+  size_t insize, bufsize;
+  dest->clear();
+
   cd = iconv_open(codeset, "UTF-8");
   if (cd == (iconv_t)(-1)) {
      // Something went wrong
@@ -55,33 +56,49 @@ bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest)
      else
 	perror("iconv_open");
      
-     // Clean the destination string
-     *dest = "";
-     
      return false;
   }
 
-  insize = outsize = orig.size();
+  insize = bufsize = orig.size();
   inbuf = orig.data();
   inptr = (char *)inbuf;
-  outbuf = new char[insize+1];
-  outptr = outbuf;
+  outbuf = new char[bufsize];
+  size_t lastError = -1;
 
   while (insize != 0)
   {
+     char *outptr = outbuf;
+     size_t outsize = bufsize;
      size_t const err = iconv(cd, &inptr, &insize, &outptr, &outsize);
+     dest->append(outbuf, outptr - outbuf);
      if (err == (size_t)(-1))
      {
-	insize--;
-	outsize++;
-	inptr++;
-	*outptr = '?';
-	outptr++;
+	switch (errno)
+	{
+	case EILSEQ:
+	   insize--;
+	   inptr++;
+	   // replace a series of unknown multibytes with a single "?"
+	   if (lastError != insize) {
+	      lastError = insize - 1;
+	      dest->append("?");
+	   }
+	   break;
+	case EINVAL:
+	   insize = 0;
+	   break;
+	case E2BIG:
+	   if (outptr == outbuf)
+	   {
+	      bufsize *= 2;
+	      delete[] outbuf;
+	      outbuf = new char[bufsize];
+	   }
+	   break;
+	}
      }
   }
 
-  *outptr = '\0';
-  *dest = outbuf;
   delete[] outbuf;
   
   iconv_close(cd);
diff --git a/apt-pkg/contrib/system.h b/apt-pkg/contrib/system.h
index 7ec3d7feb..b57093b93 100644
--- a/apt-pkg/contrib/system.h
+++ b/apt-pkg/contrib/system.h
@@ -55,4 +55,26 @@
 #define CLRFLAG(v,f)	((v) &=~FLAG(f))
 #define	CHKFLAG(v,f)	((v) &  FLAG(f) ? true : false)
 
+// some nice optional GNUC features
+#if __GNUC__ >= 3
+        #define __must_check    __attribute__ ((warn_unused_result))
+        #define __deprecated    __attribute__ ((deprecated))
+        /* likely() and unlikely() can be used to mark boolean expressions
+           as (not) likely true which will help the compiler to optimise */
+        #define likely(x)       __builtin_expect (!!(x), 1)
+        #define unlikely(x)     __builtin_expect (!!(x), 0)
+#else
+        #define __must_check    /* no warn_unused_result */
+        #define __deprecated    /* no deprecated */
+        #define likely(x)       (x)
+        #define unlikely(x)     (x)
+#endif
+
+// cold functions are unlikely() to be called
+#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4
+        #define __cold  __attribute__ ((__cold__))
+#else
+        #define __cold  /* no cold marker */
+#endif
+
 #endif
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index d1a275a47..565f01b84 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -561,15 +561,16 @@ bool pkgDPkgPM::OpenLog()
    if (!logfile_name.empty())
    {
       term_out = fopen(logfile_name.c_str(),"a");
+      if (term_out == NULL)
+	 return _error->WarningE(_("Could not open file '%s'"), logfile_name.c_str());
+
       chmod(logfile_name.c_str(), 0600);
       // output current time
       char outstr[200];
       time_t t = time(NULL);
       struct tm *tmp = localtime(&t);
       strftime(outstr, sizeof(outstr), "%F  %T", tmp);
-      fprintf(term_out, "\nLog started: ");
-      fprintf(term_out, "%s", outstr);
-      fprintf(term_out, "\n");
+      fprintf(term_out, "\nLog started: %s\n", outstr);
    }
    return true;
 }
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 393181b6d..f9901bc9a 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -280,7 +280,7 @@ bool ReadPinDir(pkgPolicy &Plcy,string Dir)
       return true;
    }
 
-   vector<string> const List = GetListOfFilesInDir(Dir, "", true);
+   vector<string> const List = GetListOfFilesInDir(Dir, "pref", true, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)