diff options
Diffstat (limited to 'ftparchive')
| -rw-r--r-- | ftparchive/apt-ftparchive.cc | 52 | ||||
| -rw-r--r-- | ftparchive/cachedb.cc | 376 | ||||
| -rw-r--r-- | ftparchive/cachedb.h | 69 | ||||
| -rw-r--r-- | ftparchive/contents.cc | 13 | ||||
| -rw-r--r-- | ftparchive/contents.h | 4 | ||||
| -rw-r--r-- | ftparchive/makefile | 2 | ||||
| -rw-r--r-- | ftparchive/override.cc | 4 | ||||
| -rw-r--r-- | ftparchive/sources.cc | 47 | ||||
| -rw-r--r-- | ftparchive/sources.h | 32 | ||||
| -rw-r--r-- | ftparchive/writer.cc | 107 | ||||
| -rw-r--r-- | ftparchive/writer.h | 8 |
11 files changed, 505 insertions, 209 deletions
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 692f19e25..ba71ee225 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -62,6 +62,7 @@ struct PackageMap // Stuff for the Package File string PkgFile; string BinCacheDB; + string SrcCacheDB; string BinOverride; string ExtraOverride; @@ -106,6 +107,12 @@ struct PackageMap inline bool operator() (const PackageMap &x,const PackageMap &y) {return x.BinCacheDB < y.BinCacheDB;}; }; + + struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool> + { + inline bool operator() (const PackageMap &x,const PackageMap &y) + {return x.SrcCacheDB < y.SrcCacheDB;}; + }; void GetGeneral(Configuration &Setup,Configuration &Block); bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats); @@ -232,11 +239,14 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) gettimeofday(&NewTime,0); double Delta = NewTime.tv_sec - StartTime.tv_sec + (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; - + c0out << Packages.Stats.Packages << " files " << /* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */ SizeToStr(Packages.Stats.Bytes) << "B " << TimeToStr((long)Delta) << endl; + + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl; Stats.Add(Packages.Stats); Stats.DeLinkBytes = Packages.Stats.DeLinkBytes; @@ -263,7 +273,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) SrcDone = true; // Create a package writer object. - SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"), + SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB), flCombine(OverrideDir,BinOverride), flCombine(OverrideDir,SrcOverride), flCombine(OverrideDir,SrcExtraOverride)); @@ -323,6 +333,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) c0out << Sources.Stats.Packages << " pkgs in " << TimeToStr((long)Delta) << endl; + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Sources.Stats.Misses << endl; + Stats.Add(Sources.Stats); Stats.DeLinkBytes = Sources.Stats.DeLinkBytes; @@ -435,6 +448,9 @@ bool PackageMap::GenContents(Configuration &Setup, double Delta = NewTime.tv_sec - StartTime.tv_sec + (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl; + c0out << Contents.Stats.Packages << " files " << SizeToStr(Contents.Stats.Bytes) << "B " << TimeToStr((long)Delta) << endl; @@ -465,6 +481,8 @@ static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup) string DContentsH = Setup.Find("TreeDefault::Contents::Header",""); string DBCache = Setup.Find("TreeDefault::BinCacheDB", "packages-$(ARCH).db"); + string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB", + "sources-$(SECTION).db"); string DSources = Setup.Find("TreeDefault::Sources", "$(DIST)/$(SECTION)/source/Sources"); string DFLFile = Setup.Find("TreeDefault::FileList", ""); @@ -524,6 +542,7 @@ static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup) Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars); Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars); Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars); + Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars); } else { @@ -573,6 +592,7 @@ static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup) Itm.PkgFile = Block.Find("Packages"); Itm.SrcFile = Block.Find("Sources"); Itm.BinCacheDB = Block.Find("BinCacheDB"); + Itm.SrcCacheDB = Block.Find("SrcCacheDB"); Itm.BinOverride = Block.Find("BinOverride"); Itm.ExtraOverride = Block.Find("ExtraOverride"); Itm.SrcExtraOverride = Block.Find("SrcExtraOverride"); @@ -670,6 +690,10 @@ static bool SimpleGenPackages(CommandLine &CmdL) if (Packages.RecursiveScan(CmdL.FileList[1]) == false) return false; + // Give some stats if asked for + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl; + return true; } /*}}}*/ @@ -726,6 +750,10 @@ static bool SimpleGenSources(CommandLine &CmdL) if (Sources.RecursiveScan(CmdL.FileList[1]) == false) return false; + // Give some stats if asked for + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl; + return true; } /*}}}*/ @@ -777,6 +805,7 @@ static bool Generate(CommandLine &CmdL) // Sort by cache DB to improve IO locality. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); + stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare()); // Generate packages if (CmdL.FileSize() <= 2) @@ -936,20 +965,33 @@ static bool Clean(CommandLine &CmdL) // Sort by cache DB to improve IO locality. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); + stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare()); string CacheDir = Setup.FindDir("Dir::CacheDir"); for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ) { - c0out << I->BinCacheDB << endl; + if(I->BinCacheDB != "") + c0out << I->BinCacheDB << endl; + if(I->SrcCacheDB != "") + c0out << I->SrcCacheDB << endl; CacheDB DB(flCombine(CacheDir,I->BinCacheDB)); + CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB)); if (DB.Clean() == false) _error->DumpErrors(); + if (DB_SRC.Clean() == false) + _error->DumpErrors(); string CacheDB = I->BinCacheDB; - for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I); + string SrcCacheDB = I->SrcCacheDB; + while(I != PkgList.end() && + I->BinCacheDB == CacheDB && + I->SrcCacheDB == SrcCacheDB) + ++I; + } - + + return true; } /*}}}*/ diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 523c6b5fa..0901492f7 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -20,6 +20,7 @@ #include <apt-pkg/configuration.h> #include <apt-pkg/fileutl.h> #include <apt-pkg/debfile.h> +#include <apt-pkg/gpgv.h> #include <netinet/in.h> // htonl, etc #include <ctype.h> @@ -31,6 +32,19 @@ #include <apti18n.h> /*}}}*/ +CacheDB::CacheDB(std::string const &DB) + : Dbp(0), Fd(NULL), DebFile(0) +{ + TmpKey[0]='\0'; + ReadyDB(DB); +}; + +CacheDB::~CacheDB() +{ + ReadyDB(); + delete DebFile; +}; + // CacheDB::ReadyDB - Ready the DB2 /*{{{*/ // --------------------------------------------------------------------- /* This opens the DB2 file for caching package information */ @@ -85,7 +99,7 @@ bool CacheDB::ReadyDB(std::string const &DB) return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err)); } } - + DBFile = DB; DBLoaded = true; return true; @@ -96,14 +110,54 @@ bool CacheDB::ReadyDB(std::string const &DB) /* */ bool CacheDB::OpenFile() { - Fd = new FileFd(FileName,FileFd::ReadOnly); - if (_error->PendingError() == true) - { - delete Fd; - Fd = NULL; - return false; - } - return true; + // always close existing file first + CloseFile(); + + // open a new file + Fd = new FileFd(FileName,FileFd::ReadOnly); + if (_error->PendingError() == true) + { + CloseFile(); + return false; + } + return true; +} + /*}}}*/ +// CacheDB::CloseFile - Close the file /*{{{*/ +void CacheDB::CloseFile() +{ + if(Fd != NULL) + { + delete Fd; + Fd = NULL; + } +} + /*}}}*/ +// CacheDB::OpenDebFile - Open a debfile /*{{{*/ +bool CacheDB::OpenDebFile() +{ + // always close existing file first + CloseDebFile(); + + // first open the fd, then pass it to the debDebFile + if(OpenFile() == false) + return false; + DebFile = new debDebFile(*Fd); + if (_error->PendingError() == true) + return false; + return true; +} + /*}}}*/ +// CacheDB::CloseDebFile - Close a debfile again /*{{{*/ +void CacheDB::CloseDebFile() +{ + CloseFile(); + + if(DebFile != NULL) + { + delete DebFile; + DebFile = NULL; + } } /*}}}*/ // CacheDB::GetFileStat - Get stats from the file /*{{{*/ @@ -112,29 +166,65 @@ bool CacheDB::OpenFile() * to look at the file, also get the mtime from the file. */ bool CacheDB::GetFileStat(bool const &doStat) { - if ((CurStat.Flags & FlSize) == FlSize && doStat == false) - { - /* Already worked out the file size */ - } - else - { - /* Get it from the file. */ - if (Fd == NULL && OpenFile() == false) - { - return false; - } - // Stat the file - struct stat St; - if (fstat(Fd->Fd(),&St) != 0) - { - return _error->Errno("fstat", - _("Failed to stat %s"),FileName.c_str()); - } - CurStat.FileSize = St.st_size; - CurStat.mtime = htonl(St.st_mtime); - CurStat.Flags |= FlSize; - } - return true; + if ((CurStat.Flags & FlSize) == FlSize && doStat == false) + return true; + + /* Get it from the file. */ + if (OpenFile() == false) + return false; + + // Stat the file + struct stat St; + if (fstat(Fd->Fd(),&St) != 0) + { + CloseFile(); + return _error->Errno("fstat", + _("Failed to stat %s"),FileName.c_str()); + } + CurStat.FileSize = St.st_size; + CurStat.mtime = htonl(St.st_mtime); + CurStat.Flags |= FlSize; + + return true; +} + /*}}}*/ +// CacheDB::GetCurStatCompatOldFormat /*{{{*/ +// --------------------------------------------------------------------- +/* Read the old (32bit FileSize) StateStore format from disk */ +bool CacheDB::GetCurStatCompatOldFormat() +{ + InitQueryStats(); + Data.data = &CurStatOldFormat; + Data.flags = DB_DBT_USERMEM; + Data.ulen = sizeof(CurStatOldFormat); + if (Get() == false) + { + CurStat.Flags = 0; + } else { + CurStat.Flags = CurStatOldFormat.Flags; + CurStat.mtime = CurStatOldFormat.mtime; + CurStat.FileSize = CurStatOldFormat.FileSize; + memcpy(CurStat.MD5, CurStatOldFormat.MD5, sizeof(CurStat.MD5)); + memcpy(CurStat.SHA1, CurStatOldFormat.SHA1, sizeof(CurStat.SHA1)); + memcpy(CurStat.SHA256, CurStatOldFormat.SHA256, sizeof(CurStat.SHA256)); + } + return true; +} + /*}}}*/ +// CacheDB::GetCurStatCompatOldFormat /*{{{*/ +// --------------------------------------------------------------------- +/* Read the new (64bit FileSize) StateStore format from disk */ +bool CacheDB::GetCurStatCompatNewFormat() +{ + InitQueryStats(); + Data.data = &CurStat; + Data.flags = DB_DBT_USERMEM; + Data.ulen = sizeof(CurStat); + if (Get() == false) + { + CurStat.Flags = 0; + } + return true; } /*}}}*/ // CacheDB::GetCurStat - Set the CurStat variable. /*{{{*/ @@ -145,79 +235,110 @@ bool CacheDB::GetCurStat() { memset(&CurStat,0,sizeof(CurStat)); - if (DBLoaded) - { - /* First see if there is anything about it - in the database */ - - /* Get the flags (and mtime) */ - InitQuery("st"); - // Ensure alignment of the returned structure - Data.data = &CurStat; - Data.ulen = sizeof(CurStat); - Data.flags = DB_DBT_USERMEM; - if (Get() == false) + if (DBLoaded) + { + // do a first query to just get the size of the data on disk + InitQueryStats(); + Data.data = &CurStat; + Data.flags = DB_DBT_USERMEM; + Data.ulen = 0; + Get(); + + if (Data.size == 0) + { + // nothing needs to be done, we just have not data for this deb + } + // check if the record is written in the old format (32bit filesize) + else if(Data.size == sizeof(CurStatOldFormat)) + { + GetCurStatCompatOldFormat(); + } + else if(Data.size == sizeof(CurStat)) { - CurStat.Flags = 0; - } - CurStat.Flags = ntohl(CurStat.Flags); - CurStat.FileSize = ntohl(CurStat.FileSize); + GetCurStatCompatNewFormat(); + } else { + return _error->Error("Cache record size mismatch (%ul)", Data.size); + } + + CurStat.Flags = ntohl(CurStat.Flags); + CurStat.FileSize = ntohl(CurStat.FileSize); } - return true; + return true; } /*}}}*/ // CacheDB::GetFileInfo - Get all the info about the file /*{{{*/ // --------------------------------------------------------------------- -bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents, - bool const &GenContentsOnly, bool const &DoMD5, bool const &DoSHA1, - bool const &DoSHA256, bool const &DoSHA512, +bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, + bool const &DoContents, + bool const &GenContentsOnly, + bool const &DoSource, + bool const &DoMD5, bool const &DoSHA1, + bool const &DoSHA256, bool const &DoSHA512, bool const &checkMtime) { - this->FileName = FileName; + bool result = true; + this->FileName = FileName; - if (GetCurStat() == false) - { - return false; - } + if (GetCurStat() == false) + return false; OldStat = CurStat; - if (GetFileStat(checkMtime) == false) - { - delete Fd; - Fd = NULL; - return false; - } - - /* if mtime changed, update CurStat from disk */ - if (checkMtime == true && OldStat.mtime != CurStat.mtime) - CurStat.Flags = FlSize; - - Stats.Bytes += CurStat.FileSize; - Stats.Packages++; - - if ((DoControl && LoadControl() == false) - || (DoContents && LoadContents(GenContentsOnly) == false) - || (DoMD5 && GetMD5(false) == false) - || (DoSHA1 && GetSHA1(false) == false) - || (DoSHA256 && GetSHA256(false) == false) - || (DoSHA512 && GetSHA512(false) == false) - ) - { - delete Fd; - Fd = NULL; - delete DebFile; - DebFile = NULL; - return false; - } - - delete Fd; - Fd = NULL; - delete DebFile; - DebFile = NULL; + if (GetFileStat(checkMtime) == false) + return false; - return true; + /* if mtime changed, update CurStat from disk */ + if (checkMtime == true && OldStat.mtime != CurStat.mtime) + CurStat.Flags = FlSize; + + Stats.Bytes += CurStat.FileSize; + Stats.Packages++; + + if ((DoControl && LoadControl() == false) + || (DoContents && LoadContents(GenContentsOnly) == false) + || (DoSource && LoadSource() == false) + || (DoMD5 && GetMD5(false) == false) + || (DoSHA1 && GetSHA1(false) == false) + || (DoSHA256 && GetSHA256(false) == false) + || (DoSHA512 && GetSHA512(false) == false) ) + { + result = false; + } + + return result; } /*}}}*/ + +bool CacheDB::LoadSource() +{ + // Try to read the control information out of the DB. + if ((CurStat.Flags & FlSource) == FlSource) + { + // Lookup the control information + InitQuerySource(); + if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true) + { + return true; + } + CurStat.Flags &= ~FlSource; + } + if (OpenFile() == false) + return false; + + Stats.Misses++; + if (Dsc.Read(FileName) == false) + return false; + + if (Dsc.Data == 0) + return _error->Error(_("Failed to read .dsc")); + + // Write back the control information + InitQuerySource(); + if (Put(Dsc.Data, Dsc.Length) == true) + CurStat.Flags |= FlSource; + + return true; +} + // CacheDB::LoadControl - Load Control information /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -227,23 +348,14 @@ bool CacheDB::LoadControl() if ((CurStat.Flags & FlControl) == FlControl) { // Lookup the control information - InitQuery("cl"); + InitQueryControl(); if (Get() == true && Control.TakeControl(Data.data,Data.size) == true) return true; CurStat.Flags &= ~FlControl; } - if (Fd == NULL && OpenFile() == false) - { + if(OpenDebFile() == false) return false; - } - // Create a deb instance to read the archive - if (DebFile == 0) - { - DebFile = new debDebFile(*Fd); - if (_error->PendingError() == true) - return false; - } Stats.Misses++; if (Control.Read(*DebFile) == false) @@ -253,7 +365,7 @@ bool CacheDB::LoadControl() return _error->Error(_("Archive has no control record")); // Write back the control information - InitQuery("cl"); + InitQueryControl(); if (Put(Control.Control,Control.Length) == true) CurStat.Flags |= FlControl; return true; @@ -271,7 +383,7 @@ bool CacheDB::LoadContents(bool const &GenOnly) return true; // Lookup the contents information - InitQuery("cn"); + InitQueryContent(); if (Get() == true) { if (Contents.TakeContents(Data.data,Data.size) == true) @@ -281,23 +393,15 @@ bool CacheDB::LoadContents(bool const &GenOnly) CurStat.Flags &= ~FlContents; } - if (Fd == NULL && OpenFile() == false) - { + if(OpenDebFile() == false) return false; - } - // Create a deb instance to read the archive - if (DebFile == 0) - { - DebFile = new debDebFile(*Fd); - if (_error->PendingError() == true) - return false; - } + Stats.Misses++; if (Contents.Read(*DebFile) == false) return false; // Write back the control information - InitQuery("cn"); + InitQueryContent(); if (Put(Contents.Data,Contents.CurSize) == true) CurStat.Flags |= FlContents; return true; @@ -347,14 +451,13 @@ bool CacheDB::GetMD5(bool const &GenOnly) MD5Res = bytes2hex(CurStat.MD5, sizeof(CurStat.MD5)); return true; - } + } Stats.MD5Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + MD5Summation MD5; if (Fd->Seek(0) == false || MD5.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -382,10 +485,9 @@ bool CacheDB::GetSHA1(bool const &GenOnly) Stats.SHA1Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + SHA1Summation SHA1; if (Fd->Seek(0) == false || SHA1.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -413,10 +515,9 @@ bool CacheDB::GetSHA256(bool const &GenOnly) Stats.SHA256Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + SHA256Summation SHA256; if (Fd->Seek(0) == false || SHA256.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -444,10 +545,9 @@ bool CacheDB::GetSHA512(bool const &GenOnly) Stats.SHA512Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + SHA512Summation SHA512; if (Fd->Seek(0) == false || SHA512.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -467,11 +567,11 @@ bool CacheDB::Finish() if (CurStat.Flags == OldStat.Flags && CurStat.mtime == OldStat.mtime) return true; - + // Write the stat information CurStat.Flags = htonl(CurStat.Flags); CurStat.FileSize = htonl(CurStat.FileSize); - InitQuery("st"); + InitQueryStats(); Put(&CurStat,sizeof(CurStat)); CurStat.Flags = ntohl(CurStat.Flags); CurStat.FileSize = ntohl(CurStat.FileSize); @@ -504,16 +604,24 @@ bool CacheDB::Clean() { if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 || stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 || + stringcmp(Colon + 1, (char *)Key.data+Key.size,"cs") == 0 || stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0) { - if (FileExists(std::string((const char *)Key.data,Colon)) == true) - continue; + std::string FileName = std::string((const char *)Key.data,Colon); + if (FileExists(FileName) == true) { + continue; + } } } - Cursor->c_del(Cursor,0); } - Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL); + int res = Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL); + if (res < 0) + _error->Warning("compact failed with result %i", res); + + if(_config->FindB("Debug::APT::FTPArchive::Clean", false) == true) + Dbp->stat_print(Dbp, 0); + return true; } diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 49b9a0ef5..29d710d2c 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -22,9 +22,11 @@ #include <stdio.h> #include "contents.h" +#include "sources.h" class FileFd; + class CacheDB { protected: @@ -39,7 +41,7 @@ class CacheDB std::string DBFile; // Generate a key for the DB of a given type - inline void InitQuery(const char *Type) + void _InitQuery(const char *Type) { memset(&Key,0,sizeof(Key)); memset(&Data,0,sizeof(Data)); @@ -47,6 +49,19 @@ class CacheDB Key.size = snprintf(TmpKey,sizeof(TmpKey),"%s:%s",FileName.c_str(), Type); } + void InitQueryStats() { + _InitQuery("st"); + } + void InitQuerySource() { + _InitQuery("cs"); + } + void InitQueryControl() { + _InitQuery("cl"); + } + void InitQueryContent() { + _InitQuery("cn"); + } + inline bool Get() { return Dbp->get(Dbp,0,&Key,&Data,0) == 0; @@ -65,10 +80,20 @@ class CacheDB return true; } bool OpenFile(); - bool GetFileStat(bool const &doStat = false); + void CloseFile(); + + bool OpenDebFile(); + void CloseDebFile(); + + // GetCurStat needs some compat code, see lp #1274466) + bool GetCurStatCompatOldFormat(); + bool GetCurStatCompatNewFormat(); bool GetCurStat(); + + bool GetFileStat(bool const &doStat = false); bool LoadControl(); bool LoadContents(bool const &GenOnly); + bool LoadSource(); bool GetMD5(bool const &GenOnly); bool GetSHA1(bool const &GenOnly); bool GetSHA256(bool const &GenOnly); @@ -77,8 +102,23 @@ class CacheDB // Stat info stored in the DB, Fixed types since it is written to disk. enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2), FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), - FlSHA512=(1<<6)}; + FlSHA512=(1<<6), FlSource=(1<<7), + }; + + // the on-disk format changed (FileSize increased to 64bit) in + // commit 650faab0 which will lead to corruption with old caches + struct StatStoreOldFormat + { + uint32_t Flags; + uint32_t mtime; + uint32_t FileSize; + uint8_t MD5[16]; + uint8_t SHA1[20]; + uint8_t SHA256[32]; + } CurStatOldFormat; + // WARNING: this struct is read/written to the DB so do not change the + // layout of the fields (see lp #1274466), only append to it struct StatStore { uint32_t Flags; @@ -101,6 +141,8 @@ class CacheDB // Data collection helpers debDebFile::MemControlExtract Control; ContentsExtract Contents; + DscExtract Dsc; + std::string MD5Res; std::string SHA1Res; std::string SHA256Res; @@ -132,21 +174,32 @@ class CacheDB SHA512Bytes(0),Packages(0), Misses(0), DeLinkBytes(0) {}; } Stats; - bool ReadyDB(std::string const &DB); + bool ReadyDB(std::string const &DB = ""); inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;}; inline bool Loaded() {return DBLoaded == true;}; inline unsigned long long GetFileSize(void) {return CurStat.FileSize;} bool SetFile(std::string const &FileName,struct stat St,FileFd *Fd); - bool GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents, bool const &GenContentsOnly, - bool const &DoMD5, bool const &DoSHA1, bool const &DoSHA256, bool const &DoSHA512, bool const &checkMtime = false); + + // terrible old overloaded interface + bool GetFileInfo(std::string const &FileName, + bool const &DoControl, + bool const &DoContents, + bool const &GenContentsOnly, + bool const &DoSource, + bool const &DoMD5, + bool const &DoSHA1, + bool const &DoSHA256, + bool const &DoSHA512, + bool const &checkMtime = false); + bool Finish(); bool Clean(); - CacheDB(std::string const &DB) : Dbp(0), Fd(NULL), DebFile(0) {TmpKey[0]='\0'; ReadyDB(DB);}; - ~CacheDB() {ReadyDB(std::string()); delete DebFile;}; + CacheDB(std::string const &DB); + ~CacheDB(); }; #endif diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc index 7a1fb779e..91dd2b8bd 100644 --- a/ftparchive/contents.cc +++ b/ftparchive/contents.cc @@ -302,7 +302,18 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf) DoPrint(Out,Top->BTreeRight,Buf); } /*}}}*/ - +// ContentsExtract Constructor /*{{{*/ +ContentsExtract::ContentsExtract() + : Data(0), MaxSize(0), CurSize(0) +{ +}; + /*}}}*/ +// ContentsExtract Destructor /*{{{*/ +ContentsExtract::~ContentsExtract() +{ + free(Data); +}; + /*}}}*/ // ContentsExtract::Read - Read the archive /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/ftparchive/contents.h b/ftparchive/contents.h index dbbb83350..f58e3278e 100644 --- a/ftparchive/contents.h +++ b/ftparchive/contents.h @@ -85,8 +85,8 @@ class ContentsExtract : public pkgDirStream bool TakeContents(const void *Data,unsigned long long Length); void Add(GenContents &Contents,std::string const &Package); - ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {}; - virtual ~ContentsExtract() {delete [] Data;}; + ContentsExtract(); + virtual ~ContentsExtract(); }; #endif diff --git a/ftparchive/makefile b/ftparchive/makefile index c53ecff72..d1ffe182a 100644 --- a/ftparchive/makefile +++ b/ftparchive/makefile @@ -12,7 +12,7 @@ PROGRAM=apt-ftparchive SLIBS = -lapt-pkg -lapt-inst $(BDBLIB) $(INTLLIBS) LIB_MAKES = apt-pkg/makefile apt-inst/makefile SOURCE = apt-ftparchive.cc cachedb.cc writer.cc contents.cc override.cc \ - multicompress.cc + multicompress.cc sources.cc include $(PROGRAM_H) else PROGRAM=apt-ftparchive diff --git a/ftparchive/override.cc b/ftparchive/override.cc index b4cd49b6c..8a0c5bab1 100644 --- a/ftparchive/override.cc +++ b/ftparchive/override.cc @@ -37,7 +37,7 @@ bool Override::ReadOverride(string const &File,bool const &Source) if (F == 0) return _error->Errno("fopen",_("Unable to open %s"),File.c_str()); - char Line[500]; + char Line[1000]; unsigned long long Counter = 0; while (fgets(Line,sizeof(Line),F) != 0) { @@ -141,7 +141,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &/*Source*/) if (F == 0) return _error->Errno("fopen",_("Unable to open %s"),File.c_str()); - char Line[500]; + char Line[1000]; unsigned long long Counter = 0; while (fgets(Line,sizeof(Line),F) != 0) { diff --git a/ftparchive/sources.cc b/ftparchive/sources.cc new file mode 100644 index 000000000..d0878a70a --- /dev/null +++ b/ftparchive/sources.cc @@ -0,0 +1,47 @@ +#include <string> +#include <iostream> + +// for memcpy +#include <cstring> + +#include <apt-pkg/error.h> +#include <apt-pkg/gpgv.h> + +#include "sources.h" + +bool DscExtract::TakeDsc(const void *newData, unsigned long newSize) +{ + if(newSize > maxSize) + return _error->Error("DSC data is too large %lu!", newSize); + + if (newSize == 0) + { + Length = 0; + return true; + } + memcpy(Data, newData, newSize); + Length = newSize; + + return true; +} + +bool DscExtract::Read(std::string FileName) +{ + FileFd F; + if (OpenMaybeClearSignedFile(FileName, F) == false) + return false; + + unsigned long long const FSize = F.FileSize(); + if(FSize > maxSize) + return _error->Error("DSC file '%s' is too large!",FileName.c_str()); + + if (F.Read(Data, FSize) == false) + return false; + Length = FSize; + + IsClearSigned = (FileName != F.Name()); + + return true; +} + + diff --git a/ftparchive/sources.h b/ftparchive/sources.h new file mode 100644 index 000000000..91e0b1376 --- /dev/null +++ b/ftparchive/sources.h @@ -0,0 +1,32 @@ +#ifndef SOURCES_H +#define SOURCES_H + +#include <apt-pkg/tagfile.h> + +class DscExtract +{ + public: + //FIXME: do we really need to enforce a maximum size of the dsc file? + static const int maxSize = 128*1024; + + char *Data; + pkgTagSection Section; + unsigned long Length; + bool IsClearSigned; + + bool TakeDsc(const void *Data, unsigned long Size); + bool Read(std::string FileName); + + DscExtract() : Data(0), Length(0) { + Data = new char[maxSize]; + }; + ~DscExtract() { + if(Data != NULL) { + delete [] Data; + Data = NULL; + } + }; +}; + + +#endif diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 153c4fb42..7c1c9cc03 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -385,10 +385,14 @@ bool FTWScanner::SetExts(string const &Vals) bool PackagesWriter::DoPackage(string FileName) { // Pull all the data we need form the DB - if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) - == false) + if (Db.GetFileInfo(FileName, + true, /* DoControl */ + DoContents, + true, /* GenContentsOnly */ + false, /* DoSource */ + DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false) { - return false; + return false; } unsigned long long FileSize = Db.GetFileSize(); @@ -614,59 +618,36 @@ SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string c /* */ bool SourcesWriter::DoPackage(string FileName) { - // Open the archive - FileFd F; - if (OpenMaybeClearSignedFile(FileName, F) == false) - return false; - - unsigned long long const FSize = F.FileSize(); - //FIXME: do we really need to enforce a maximum size of the dsc file? - if (FSize > 128*1024) - return _error->Error("DSC file '%s' is too large!",FileName.c_str()); - - if (BufSize < FSize + 2) + // Pull all the data we need form the DB + if (Db.GetFileInfo(FileName, + false, /* DoControl */ + false, /* DoContents */ + false, /* GenContentsOnly */ + true, /* DoSource */ + DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false) { - BufSize = FSize + 2; - Buffer = (char *)realloc(Buffer , BufSize); - } - - if (F.Read(Buffer, FSize) == false) return false; + } - // Stat the file for later (F might be clearsigned, so not F.FileSize()) - struct stat St; - if (stat(FileName.c_str(), &St) != 0) - return _error->Errno("fstat","Failed to stat %s",FileName.c_str()); + // we need to perform a "write" here (this is what finish is doing) + // because the call to Db.GetFileInfo() in the loop will change + // the "db cursor" + Db.Finish(); - // Hash the file - char *Start = Buffer; - char *BlkEnd = Buffer + FSize; - - Hashes DscHashes; - if (FSize == (unsigned long long) St.st_size) - { - if (DoMD5 == true) - DscHashes.MD5.Add((unsigned char *)Start,BlkEnd - Start); - if (DoSHA1 == true) - DscHashes.SHA1.Add((unsigned char *)Start,BlkEnd - Start); - if (DoSHA256 == true) - DscHashes.SHA256.Add((unsigned char *)Start,BlkEnd - Start); - if (DoSHA512 == true) - DscHashes.SHA512.Add((unsigned char *)Start,BlkEnd - Start); - } - else - { - FileFd DscFile(FileName, FileFd::ReadOnly); - DscHashes.AddFD(DscFile, St.st_size, DoMD5, DoSHA1, DoSHA256, DoSHA512); - } + // read stuff + char *Start = Db.Dsc.Data; + char *BlkEnd = Db.Dsc.Data + Db.Dsc.Length; // Add extra \n to the end, just in case (as in clearsigned they are missing) *BlkEnd++ = '\n'; *BlkEnd++ = '\n'; pkgTagSection Tags; - if (Tags.Scan(Start,BlkEnd - Start) == false || Tags.Exists("Source") == false) + if (Tags.Scan(Start,BlkEnd - Start) == false) return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str()); + + if (Tags.Exists("Source") == false) + return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str()); Tags.Trim(); // Lookup the overide information, finding first the best priority. @@ -714,6 +695,10 @@ bool SourcesWriter::DoPackage(string FileName) OverItem = auto_ptr<Override::Item>(new Override::Item); } + struct stat St; + if (stat(FileName.c_str(), &St) != 0) + return _error->Errno("fstat","Failed to stat %s",FileName.c_str()); + auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source"))); // const auto_ptr<Override::Item> autoSOverItem(SOverItem); if (SOverItem.get() == 0) @@ -732,23 +717,23 @@ bool SourcesWriter::DoPackage(string FileName) string const strippedName = flNotDir(FileName); std::ostringstream ostreamFiles; if (DoMD5 == true && Tags.Exists("Files")) - ostreamFiles << "\n " << string(DscHashes.MD5.Result()) << " " << St.st_size << " " + ostreamFiles << "\n " << Db.MD5Res.c_str() << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Files"); string const Files = ostreamFiles.str(); std::ostringstream ostreamSha1; if (DoSHA1 == true && Tags.Exists("Checksums-Sha1")) - ostreamSha1 << "\n " << string(DscHashes.SHA1.Result()) << " " << St.st_size << " " + ostreamSha1 << "\n " << string(Db.SHA1Res.c_str()) << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Checksums-Sha1"); std::ostringstream ostreamSha256; if (DoSHA256 == true && Tags.Exists("Checksums-Sha256")) - ostreamSha256 << "\n " << string(DscHashes.SHA256.Result()) << " " << St.st_size << " " + ostreamSha256 << "\n " << string(Db.SHA256Res.c_str()) << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Checksums-Sha256"); std::ostringstream ostreamSha512; if (DoSHA512 == true && Tags.Exists("Checksums-Sha512")) - ostreamSha512 << "\n " << string(DscHashes.SHA512.Result()) << " " << St.st_size << " " + ostreamSha512 << "\n " << string(Db.SHA512Res.c_str()) << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Checksums-Sha512"); // Strip the DirStrip prefix from the FileName and add the PathPrefix @@ -785,8 +770,13 @@ bool SourcesWriter::DoPackage(string FileName) (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) || (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))) { - if (Db.GetFileInfo(OriginalPath, false, false, false, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) - == false) + if (Db.GetFileInfo(OriginalPath, + false, /* DoControl */ + false, /* DoContents */ + false, /* GenContentsOnly */ + false, /* DoSource */ + DoMD5, DoSHA1, DoSHA256, DoSHA512, + DoAlwaysStat) == false) { return _error->Error("Error getting file info"); } @@ -802,6 +792,9 @@ bool SourcesWriter::DoPackage(string FileName) if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")) ostreamSha512 << "\n " << string(Db.SHA512Res) << " " << Db.GetFileSize() << " " << ParseJnk; + + // write back the GetFileInfo() stats data + Db.Finish(); } // Perform the delinking operation @@ -872,7 +865,7 @@ bool SourcesWriter::DoPackage(string FileName) Stats.Packages++; - return Db.Finish(); + return true; } /*}}}*/ @@ -893,7 +886,15 @@ ContentsWriter::ContentsWriter(string const &DB, string const &Arch) : determine what the package name is. */ bool ContentsWriter::DoPackage(string FileName, string Package) { - if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false)) + if (!Db.GetFileInfo(FileName, + Package.empty(), /* DoControl */ + true, /* DoContents */ + false, /* GenContentsOnly */ + false, /* DoSource */ + false, /* DoMD5 */ + false, /* DoSHA1 */ + false, /* DoSHA256 */ + false)) /* DoSHA512 */ { return false; } diff --git a/ftparchive/writer.h b/ftparchive/writer.h index 86884dcfc..d8a10e0bb 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -127,8 +127,10 @@ class PackagesWriter : public FTWScanner {return Over.ReadExtraOverride(File);}; virtual bool DoPackage(string FileName); - PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides=string(), - string const &Arch=string()); + PackagesWriter(string const &DB, + string const &Overrides, + string const &ExtOverrides = "", + string const &Arch = ""); virtual ~PackagesWriter() {}; }; @@ -174,7 +176,7 @@ class SourcesWriter : public FTWScanner string PathPrefix; string DirStrip; FILE *Output; - struct CacheDB::Stats Stats; + struct CacheDB::Stats &Stats; virtual bool DoPackage(string FileName); |