make http size check work

7 files changed, 181 insertions, 55 deletions

diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index ba71ee225..ebf99a8f8 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -781,33 +781,14 @@ static bool SimpleGenRelease(CommandLine &CmdL)
 }
 
 									/*}}}*/
-// Generate - Full generate, using a config file			/*{{{*/
+// DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 // ---------------------------------------------------------------------
-/* */
-static bool Generate(CommandLine &CmdL)
+static bool DoGeneratePackagesAndSources(Configuration &Setup,
+					 vector<PackageMap> &PkgList,
+					 struct CacheDB::Stats &SrcStats,
+					 struct CacheDB::Stats &Stats,
+					 CommandLine &CmdL)
 {
-   struct CacheDB::Stats SrcStats;
-   if (CmdL.FileSize() < 2)
-      return ShowHelp(CmdL);
-
-   struct timeval StartTime;
-   gettimeofday(&StartTime,0);   
-   struct CacheDB::Stats Stats;
-   
-   // Read the configuration file.
-   Configuration Setup;
-   if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
-      return false;
-
-   vector<PackageMap> PkgList;
-   LoadTree(PkgList,Setup);
-   LoadBinDir(PkgList,Setup);
-
-   // Sort by cache DB to improve IO locality.
-   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
-   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
-		
-   // Generate packages
    if (CmdL.FileSize() <= 2)
    {
       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
@@ -876,9 +857,16 @@ static bool Generate(CommandLine &CmdL)
       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 	 delete I->TransWriter;
 
-   if (_config->FindB("APT::FTPArchive::Contents",true) == false)
-      return true;
-   
+   return true;
+}
+
+                                                                        /*}}}*/
+// DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
+// ---------------------------------------------------------------------
+static bool DoGenerateContents(Configuration &Setup,
+			       vector<PackageMap> &PkgList,
+			       CommandLine &CmdL)
+{
    c1out << "Packages done, Starting contents." << endl;
 
    // Sort the contents file list by date
@@ -935,17 +923,62 @@ static bool Generate(CommandLine &CmdL)
 	 break;
       }      
    }
+
+   return true;
+}
+
+									/*}}}*/
+// Generate - Full generate, using a config file			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static bool Generate(CommandLine &CmdL)
+{
+   struct CacheDB::Stats SrcStats;
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+
+   struct timeval StartTime;
+   gettimeofday(&StartTime,0);
+   struct CacheDB::Stats Stats;
    
+   // Read the configuration file.
+   Configuration Setup;
+   if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
+      return false;
+
+   vector<PackageMap> PkgList;
+   LoadTree(PkgList,Setup);
+   LoadBinDir(PkgList,Setup);
+
+   // Sort by cache DB to improve IO locality.
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
+
+   // Generate packages
+   if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
+   {
+      if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
+         return false;
+   } else {
+      c1out << "Skipping Packages/Sources generation" << endl;
+   }
+
+   // do Contents if needed
+   if (_config->FindB("APT::FTPArchive::Contents", true) == true)
+      if (DoGenerateContents(Setup, PkgList, CmdL) == false)
+         return false;
+
    struct timeval NewTime;
-   gettimeofday(&NewTime,0);   
-   double Delta = NewTime.tv_sec - StartTime.tv_sec + 
+   gettimeofday(&NewTime,0);
+   double Delta = NewTime.tv_sec - StartTime.tv_sec +
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
-   c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages 
+   c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
          << " archives. Took " << TimeToStr((long)Delta) << endl;
-   
+
    return true;
 }
-									/*}}}*/
+
+                                                                        /*}}}*/
 // Clean - Clean out the databases					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index be54ea62c..da45eb8d2 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -34,6 +34,19 @@
 #include <apti18n.h>
 									/*}}}*/
 
+CacheDB::CacheDB(std::string const &DB)
+   : Dbp(0), Fd(NULL), DebFile(0)
+{
+   TmpKey[0]='\0';
+   ReadyDB(DB);
+}
+
+CacheDB::~CacheDB()
+{
+   ReadyDB();
+   delete DebFile;
+}
+
 // CacheDB::ReadyDB - Ready the DB2					/*{{{*/
 // ---------------------------------------------------------------------
 /* This opens the DB2 file for caching package information */
@@ -88,7 +101,7 @@ bool CacheDB::ReadyDB(std::string const &DB)
           return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
       }
    }
-   
+
    DBFile = DB;
    DBLoaded = true;
    return true;
@@ -177,6 +190,45 @@ bool CacheDB::GetFileStat(bool const &doStat)
    return true;
 }
 									/*}}}*/
+// CacheDB::GetCurStatCompatOldFormat           			/*{{{*/
+// ---------------------------------------------------------------------
+/* Read the old (32bit FileSize) StateStore format from disk */
+bool CacheDB::GetCurStatCompatOldFormat()
+{
+   InitQueryStats();
+   Data.data = &CurStatOldFormat;
+   Data.flags = DB_DBT_USERMEM;
+   Data.ulen = sizeof(CurStatOldFormat);
+   if (Get() == false)
+   {
+      CurStat.Flags = 0;
+   } else {
+      CurStat.Flags = CurStatOldFormat.Flags;
+      CurStat.mtime = CurStatOldFormat.mtime;
+      CurStat.FileSize = CurStatOldFormat.FileSize;
+      memcpy(CurStat.MD5, CurStatOldFormat.MD5, sizeof(CurStat.MD5));
+      memcpy(CurStat.SHA1, CurStatOldFormat.SHA1, sizeof(CurStat.SHA1));
+      memcpy(CurStat.SHA256, CurStatOldFormat.SHA256, sizeof(CurStat.SHA256));
+   }
+   return true;
+}
+									/*}}}*/
+// CacheDB::GetCurStatCompatOldFormat           			/*{{{*/
+// ---------------------------------------------------------------------
+/* Read the new (64bit FileSize) StateStore format from disk */
+bool CacheDB::GetCurStatCompatNewFormat()
+{
+   InitQueryStats();
+   Data.data = &CurStat;
+   Data.flags = DB_DBT_USERMEM;
+   Data.ulen = sizeof(CurStat);
+   if (Get() == false)
+   {
+      CurStat.Flags = 0;
+   }
+   return true;
+}
+									/*}}}*/
 // CacheDB::GetCurStat - Set the CurStat variable.			/*{{{*/
 // ---------------------------------------------------------------------
 /* Sets the CurStat variable.  Either to 0 if no database is used
@@ -187,19 +239,29 @@ bool CacheDB::GetCurStat()
    
    if (DBLoaded)
    {
-      /* First see if there is anything about it
-         in the database */
-      
-      /* Get the flags (and mtime) */
+      // do a first query to just get the size of the data on disk
       InitQueryStats();
-      // Ensure alignment of the returned structure
       Data.data = &CurStat;
-      Data.ulen = sizeof(CurStat);
       Data.flags = DB_DBT_USERMEM;
-      if (Get() == false)
+      Data.ulen = 0;
+      Get();
+
+      if (Data.size == 0)
+      {
+         // nothing needs to be done, we just have not data for this deb
+      }
+      // check if the record is written in the old format (32bit filesize)
+      else if(Data.size == sizeof(CurStatOldFormat))
       {
-	 CurStat.Flags = 0;
-      }      
+         GetCurStatCompatOldFormat();
+      }
+      else if(Data.size == sizeof(CurStat))
+      {
+         GetCurStatCompatNewFormat();
+      } else {
+         return _error->Error("Cache record size mismatch (%ul)", Data.size);
+      }
+
       CurStat.Flags = ntohl(CurStat.Flags);
       CurStat.FileSize = ntohl(CurStat.FileSize);
    }      
diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h
index 14dba5578..613963f6f 100644
--- a/ftparchive/cachedb.h
+++ b/ftparchive/cachedb.h
@@ -86,8 +86,12 @@ class CacheDB
    bool OpenDebFile();
    void CloseDebFile();
 
-   bool GetFileStat(bool const &doStat = false);
+   // GetCurStat needs some compat code, see lp #1274466)
+   bool GetCurStatCompatOldFormat();
+   bool GetCurStatCompatNewFormat();
    bool GetCurStat();
+
+   bool GetFileStat(bool const &doStat = false);
    bool LoadControl();
    bool LoadContents(bool const &GenOnly);
    bool LoadSource();
@@ -95,10 +99,24 @@ class CacheDB
 
    // Stat info stored in the DB, Fixed types since it is written to disk.
    enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2),
-                  FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), 
-                  FlSHA512=(1<<6), FlSource=(1<<7),
+                  FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5),
+                  FlSHA512=(1<<6), FlSource=(1<<7)
    };
 
+   // the on-disk format changed (FileSize increased to 64bit) in 
+   // commit 650faab0 which will lead to corruption with old caches
+   struct StatStoreOldFormat
+   {
+      uint32_t Flags;
+      uint32_t mtime;
+      uint32_t FileSize;
+      uint8_t  MD5[16];
+      uint8_t  SHA1[20];
+      uint8_t  SHA256[32];
+   } CurStatOldFormat;
+
+   // WARNING: this struct is read/written to the DB so do not change the
+   //          layout of the fields (see lp #1274466), only append to it
    struct StatStore
    {
       uint32_t Flags;
@@ -150,7 +168,7 @@ class CacheDB
 		SHA512Bytes(0),Packages(0), Misses(0), DeLinkBytes(0) {};
    } Stats;
    
-   bool ReadyDB(std::string const &DB);
+   bool ReadyDB(std::string const &DB = "");
    inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;};
    inline bool Loaded() {return DBLoaded == true;};
    
@@ -171,8 +189,8 @@ class CacheDB
    
    bool Clean();
    
-   CacheDB(std::string const &DB) : Dbp(0), Fd(NULL), DebFile(0) {TmpKey[0]='\0'; ReadyDB(DB);};
-   ~CacheDB() {ReadyDB(std::string()); delete DebFile;};
+   CacheDB(std::string const &DB);
+   ~CacheDB();
 };
     
 #endif
diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc
index 7a1fb779e..8c4181eda 100644
--- a/ftparchive/contents.cc
+++ b/ftparchive/contents.cc
@@ -302,7 +302,18 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
    DoPrint(Out,Top->BTreeRight,Buf);  
 }
 									/*}}}*/
-
+// ContentsExtract Constructor						/*{{{*/
+ContentsExtract::ContentsExtract()
+   : Data(0), MaxSize(0), CurSize(0)
+{
+}
+									/*}}}*/
+// ContentsExtract Destructor						/*{{{*/
+ContentsExtract::~ContentsExtract()
+{
+   free(Data);
+}
+									/*}}}*/
 // ContentsExtract::Read - Read the archive				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/ftparchive/contents.h b/ftparchive/contents.h
index dbbb83350..f58e3278e 100644
--- a/ftparchive/contents.h
+++ b/ftparchive/contents.h
@@ -85,8 +85,8 @@ class ContentsExtract : public pkgDirStream
    bool TakeContents(const void *Data,unsigned long long Length);
    void Add(GenContents &Contents,std::string const &Package);
    
-   ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {};
-   virtual ~ContentsExtract() {delete [] Data;};
+   ContentsExtract();
+   virtual ~ContentsExtract();
 };
 
 #endif
diff --git a/ftparchive/sources.h b/ftparchive/sources.h
index 91e0b1376..9ada15728 100644
--- a/ftparchive/sources.h
+++ b/ftparchive/sources.h
@@ -17,7 +17,7 @@ class DscExtract
    bool TakeDsc(const void *Data, unsigned long Size);
    bool Read(std::string FileName);
    
-   DscExtract() : Data(0), Length(0) {
+   DscExtract() : Data(0), Length(0), IsClearSigned(false) {
      Data = new char[maxSize];
    };
    ~DscExtract() { 
diff --git a/ftparchive/writer.h b/ftparchive/writer.h
index c62d4addf..226996475 100644
--- a/ftparchive/writer.h
+++ b/ftparchive/writer.h
@@ -126,8 +126,10 @@ class PackagesWriter : public FTWScanner
       {return Over.ReadExtraOverride(File);};
    virtual bool DoPackage(string FileName);
 
-   PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides=string(),
-		  string const &Arch=string());
+   PackagesWriter(string const &DB,
+                  string const &Overrides,
+                  string const &ExtOverrides = "",
+		  string const &Arch = "");
    virtual ~PackagesWriter() {};
 };