* doc/apt.conf.5.xml:

  - add and document APT::Cache-{Start,Grow,Limit} options for mmap control

7 files changed, 65 insertions, 31 deletions

diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index 62d42e4da..8f7791a8a 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -843,19 +843,21 @@ bool FileFd::Close()
    bool Res = true;
    if ((Flags & AutoClose) == AutoClose)
       if (iFd >= 0 && close(iFd) != 0)
-	 Res &= _error->Errno("close",_("Problem closing the file"));
-   iFd = -1;
+	 Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());
 
-   if ((Flags & Replace) == Replace) {
+   if ((Flags & Replace) == Replace && iFd >= 0) {
       if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)
-	 Res &= _error->Errno("rename",_("Problem renaming the file"));
+	 Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str());
+
       FileName = TemporaryFileName; // for the unlink() below.
    }
-	    
+
+   iFd = -1;
+
    if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
        FileName.empty() == false)
       if (unlink(FileName.c_str()) != 0)
-	 Res &= _error->WarningE("unlnk",_("Problem unlinking the file"));
+	 Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str());
 
 
    return Res;
diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc
index aa184b130..69fb61fca 100644
--- a/apt-pkg/contrib/mmap.cc
+++ b/apt-pkg/contrib/mmap.cc
@@ -398,6 +398,8 @@ bool DynamicMMap::Grow() {
 	if (Limit != 0 && WorkSpace >= Limit)
 		return _error->Error(_("Unable to increase the size of the MMap as the "
 		                       "limit of %lu bytes is already reached."), Limit);
+	if (GrowFactor <= 0)
+		return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user."));
 
 	unsigned long const newSize = WorkSpace + GrowFactor;
 
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 18bad6727..1175d5129 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -36,7 +36,7 @@
 #include <stdio.h>
 									/*}}}*/
 typedef vector<pkgIndexFile *>::iterator FileIterator;
-template <typename Iter> std::set<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
+template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap(6);
 
 // CacheGenerator::pkgCacheGenerator - Constructor			/*{{{*/
 // ---------------------------------------------------------------------
@@ -113,25 +113,25 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
       if (UniqHash[i] != 0)
 	 UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 
-   for (std::set<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
+   for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
-   for (std::set<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
+   for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
-   for (std::set<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
+   for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
-   for (std::set<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
+   for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
-   for (std::set<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
+   for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
-   for (std::set<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
+   for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
-   for (std::set<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
+   for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
       (*i)->ReOwn(Cache, oldMap, newMap);
 }									/*}}}*/
@@ -1094,6 +1094,18 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    return true;
 }
 									/*}}}*/
+DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
+   unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
+   unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
+   unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
+   Flags |= MMap::Moveable;
+   if (_config->FindB("APT::Cache-Fallback", false) == true)
+      Flags |= MMap::Fallback;
+   if (CacheF != NULL)
+      return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
+   else
+      return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
+}
 // CacheGenerator::MakeStatusCache - Construct the status cache		/*{{{*/
 // ---------------------------------------------------------------------
 /* This makes sure that the status cache (the cache that has all 
@@ -1109,7 +1121,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 			MMap **OutMap,bool AllowMem)
 {
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
-   unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
    
    vector<pkgIndexFile *> Files;
    for (vector<metaIndex *>::const_iterator i = List.begin();
@@ -1181,7 +1192,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       unlink(CacheFile.c_str());
       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
       fchmod(CacheF->Fd(),0644);
-      Map = new DynamicMMap(*CacheF,MMap::Public | MMap::Moveable, MapSize);
+      Map = CreateDynamicMMap(CacheF, MMap::Public);
       if (_error->PendingError() == true)
 	 return false;
       if (Debug == true)
@@ -1190,7 +1201,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    else
    {
       // Just build it in memory..
-      Map = new DynamicMMap(MMap::Moveable, MapSize);
+      Map = CreateDynamicMMap(NULL);
       if (Debug == true)
 	 std::clog << "Open memory Map (not filebased)" << std::endl;
    }
@@ -1297,13 +1308,12 @@ __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutM
    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
 {
-   unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
    vector<pkgIndexFile *> Files;
    unsigned long EndOfSource = Files.size();
    if (_system->AddStatusFiles(Files) == false)
       return false;
-   
-   SPtr<DynamicMMap> Map = new DynamicMMap(MMap::Moveable, MapSize);
+
+   SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
    unsigned long CurrentSize = 0;
    unsigned long TotalSize = 0;
    
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index 3bee1f958..ff198833a 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -23,7 +23,7 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/md5.h>
 
-#include <set>
+#include <vector>
 
 class pkgSourceList;
 class OpProgress;
@@ -46,16 +46,14 @@ class pkgCacheGenerator							/*{{{*/
    friend class ListParser;
 
    template<typename Iter> class Dynamic {
-      Iter *I;
-
       public:
-      static std::set<Iter*> toReMap;
-      Dynamic(Iter &It) : I(&It) {
-	 toReMap.insert(I);
+      static std::vector<Iter*> toReMap;
+      Dynamic(Iter &I) {
+	 toReMap.push_back(&I);
       }
 
       ~Dynamic() {
-	 toReMap.erase(I);
+	 toReMap.pop_back();
       }
    };
 
@@ -101,6 +99,7 @@ class pkgCacheGenerator							/*{{{*/
    static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
 			MMap **OutMap = 0,bool AllowMem = false);
    static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap);
+   static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0);
 
    void ReMap(void const * const oldMap, void const * const newMap);
 
diff --git a/debian/changelog b/debian/changelog
index eebf4fd01..d2491606c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+apt (0.7.26~exp9) UNRELEASEDexperimental; urgency=low
+
+  [ David Kalnischkies ]
+  * doc/apt.conf.5.xml:
+    - add and document APT::Cache-{Start,Grow,Limit} options for mmap control
+
+ -- Michael Vogt <mvo@debian.org>  Fri, 09 Jul 2010 21:51:55 +0200
+
 apt (0.7.26~exp8) experimental; urgency=low
 
   [ David Kalnischkies ]
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index 39e2c8e6b..7b7290794 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -199,9 +199,20 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
      anything that those packages depend on.</para></listitem>
      </varlistentry>
 
-     <varlistentry><term>Cache-Limit</term>
-     <listitem><para>APT uses a fixed size memory mapped cache file to store the 'available'
-     information. This sets the size of that cache (in bytes).</para></listitem>
+     <varlistentry><term>Cache-Start, Cache-Grow and Cache-Limit</term>
+     <listitem><para>APT uses since version 0.7.26 a resizable memory mapped cache file to store the 'available'
+     information. <literal>Cache-Start</literal> acts as a hint to which size the Cache will grow
+     and is therefore the amount of memory APT will request at startup. The default value is
+     20971520 bytes (~20 MB). Note that these amount of space need to be available for APT
+     otherwise it will likely fail ungracefully, so for memory restricted devices these value should
+     be lowered while on systems with a lot of configured sources this might be increased.
+     <literal>Cache-Grow</literal> defines in byte with the default of 1048576 (~1 MB) how much
+     the Cache size will be increased in the event the space defined by <literal>Cache-Start</literal>
+     is not enough. These value will be applied again and again until either the cache is big
+     enough to store all information or the size of the cache reaches the <literal>Cache-Limit</literal>.
+     The default of <literal>Cache-Limit</literal> is 0 which stands for no limit.
+     If <literal>Cache-Grow</literal> is set to 0 the automatic grow of the cache is disabled.
+     </para></listitem>
      </varlistentry>
 
      <varlistentry><term>Build-Essential</term>
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index fdec32c2c..26fb53fec 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -95,7 +95,9 @@ APT
   Clean-Installed "true";
   Immediate-Configure "true";      // DO NOT turn this off, see the man page
   Force-LoopBreak "false";         // DO NOT turn this on, see the man page
-  Cache-Limit "4194304";
+  Cache-Start "20971520";
+  Cache-Grow "1048576";
+  Cache-Limit "0";
   Default-Release "";
 
   // consider Recommends, Suggests as important dependencies that should