From 21ea1dbb50176a89e7f456f9b31220ff3097fdf2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 27 May 2014 16:25:43 +0200 Subject: use free() instead of delete() when realloc is used ContentsExtract::~ContentsExtract() needs to use free() because Data got allocated via realloc() Reported-By: clang -fsanitize=address -fno-omit-frame-pointer --- ftparchive/cachedb.cc | 13 +++++++++++++ ftparchive/cachedb.h | 6 +++--- ftparchive/contents.cc | 13 ++++++++++++- ftparchive/contents.h | 4 ++-- ftparchive/writer.h | 6 ++++-- 5 files changed, 34 insertions(+), 8 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index e56deae1e..12eac20d8 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -32,6 +32,19 @@ #include /*}}}*/ +CacheDB::CacheDB(std::string const &DB) + : Dbp(0), Fd(NULL), DebFile(0) +{ + TmpKey[0]='\0'; + ReadyDB(DB); +}; + +CacheDB::~CacheDB() +{ + ReadyDB(); + delete DebFile; +}; + // CacheDB::ReadyDB - Ready the DB2 /*{{{*/ // --------------------------------------------------------------------- /* This opens the DB2 file for caching package information */ diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 54a274944..edb8594bf 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -156,7 +156,7 @@ class CacheDB SHA512Bytes(0),Packages(0), Misses(0), DeLinkBytes(0) {}; } Stats; - bool ReadyDB(std::string const &DB); + bool ReadyDB(std::string const &DB = ""); inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;}; inline bool Loaded() {return DBLoaded == true;}; @@ -180,8 +180,8 @@ class CacheDB bool Clean(); - CacheDB(std::string const &DB) : Dbp(0), Fd(NULL), DebFile(0) {TmpKey[0]='\0'; ReadyDB(DB);}; - ~CacheDB() {ReadyDB(std::string()); delete DebFile;}; + CacheDB(std::string const &DB); + ~CacheDB(); }; #endif diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc index 7a1fb779e..91dd2b8bd 100644 --- a/ftparchive/contents.cc +++ b/ftparchive/contents.cc @@ -302,7 +302,18 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf) DoPrint(Out,Top->BTreeRight,Buf); } /*}}}*/ - +// ContentsExtract Constructor /*{{{*/ +ContentsExtract::ContentsExtract() + : Data(0), MaxSize(0), CurSize(0) +{ +}; + /*}}}*/ +// ContentsExtract Destructor /*{{{*/ +ContentsExtract::~ContentsExtract() +{ + free(Data); +}; + /*}}}*/ // ContentsExtract::Read - Read the archive /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/ftparchive/contents.h b/ftparchive/contents.h index dbbb83350..f58e3278e 100644 --- a/ftparchive/contents.h +++ b/ftparchive/contents.h @@ -85,8 +85,8 @@ class ContentsExtract : public pkgDirStream bool TakeContents(const void *Data,unsigned long long Length); void Add(GenContents &Contents,std::string const &Package); - ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {}; - virtual ~ContentsExtract() {delete [] Data;}; + ContentsExtract(); + virtual ~ContentsExtract(); }; #endif diff --git a/ftparchive/writer.h b/ftparchive/writer.h index b1a653e7d..d8a10e0bb 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -127,8 +127,10 @@ class PackagesWriter : public FTWScanner {return Over.ReadExtraOverride(File);}; virtual bool DoPackage(string FileName); - PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides=string(), - string const &Arch=string()); + PackagesWriter(string const &DB, + string const &Overrides, + string const &ExtOverrides = "", + string const &Arch = ""); virtual ~PackagesWriter() {}; }; -- cgit v1.2.3 From 243b2a381f4a12939d91084ecf100ee6d3dcb007 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 4 Jun 2014 12:39:36 +0200 Subject: Add compat mode for old (32bit FileSize) CacheDB (LP: #1274466) --- ftparchive/cachedb.cc | 69 +++++++++++++++++++++++++++++++++++++++++++-------- ftparchive/cachedb.h | 20 ++++++++++++++- 2 files changed, 78 insertions(+), 11 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 12eac20d8..0901492f7 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -99,7 +99,7 @@ bool CacheDB::ReadyDB(std::string const &DB) return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err)); } } - + DBFile = DB; DBLoaded = true; return true; @@ -185,6 +185,45 @@ bool CacheDB::GetFileStat(bool const &doStat) CurStat.mtime = htonl(St.st_mtime); CurStat.Flags |= FlSize; + return true; +} + /*}}}*/ +// CacheDB::GetCurStatCompatOldFormat /*{{{*/ +// --------------------------------------------------------------------- +/* Read the old (32bit FileSize) StateStore format from disk */ +bool CacheDB::GetCurStatCompatOldFormat() +{ + InitQueryStats(); + Data.data = &CurStatOldFormat; + Data.flags = DB_DBT_USERMEM; + Data.ulen = sizeof(CurStatOldFormat); + if (Get() == false) + { + CurStat.Flags = 0; + } else { + CurStat.Flags = CurStatOldFormat.Flags; + CurStat.mtime = CurStatOldFormat.mtime; + CurStat.FileSize = CurStatOldFormat.FileSize; + memcpy(CurStat.MD5, CurStatOldFormat.MD5, sizeof(CurStat.MD5)); + memcpy(CurStat.SHA1, CurStatOldFormat.SHA1, sizeof(CurStat.SHA1)); + memcpy(CurStat.SHA256, CurStatOldFormat.SHA256, sizeof(CurStat.SHA256)); + } + return true; +} + /*}}}*/ +// CacheDB::GetCurStatCompatOldFormat /*{{{*/ +// --------------------------------------------------------------------- +/* Read the new (64bit FileSize) StateStore format from disk */ +bool CacheDB::GetCurStatCompatNewFormat() +{ + InitQueryStats(); + Data.data = &CurStat; + Data.flags = DB_DBT_USERMEM; + Data.ulen = sizeof(CurStat); + if (Get() == false) + { + CurStat.Flags = 0; + } return true; } /*}}}*/ @@ -198,19 +237,29 @@ bool CacheDB::GetCurStat() if (DBLoaded) { - /* First see if there is anything about it - in the database */ - - /* Get the flags (and mtime) */ + // do a first query to just get the size of the data on disk InitQueryStats(); - // Ensure alignment of the returned structure Data.data = &CurStat; - Data.ulen = sizeof(CurStat); Data.flags = DB_DBT_USERMEM; - if (Get() == false) + Data.ulen = 0; + Get(); + + if (Data.size == 0) { - CurStat.Flags = 0; - } + // nothing needs to be done, we just have not data for this deb + } + // check if the record is written in the old format (32bit filesize) + else if(Data.size == sizeof(CurStatOldFormat)) + { + GetCurStatCompatOldFormat(); + } + else if(Data.size == sizeof(CurStat)) + { + GetCurStatCompatNewFormat(); + } else { + return _error->Error("Cache record size mismatch (%ul)", Data.size); + } + CurStat.Flags = ntohl(CurStat.Flags); CurStat.FileSize = ntohl(CurStat.FileSize); } diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index edb8594bf..29d710d2c 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -85,8 +85,12 @@ class CacheDB bool OpenDebFile(); void CloseDebFile(); - bool GetFileStat(bool const &doStat = false); + // GetCurStat needs some compat code, see lp #1274466) + bool GetCurStatCompatOldFormat(); + bool GetCurStatCompatNewFormat(); bool GetCurStat(); + + bool GetFileStat(bool const &doStat = false); bool LoadControl(); bool LoadContents(bool const &GenOnly); bool LoadSource(); @@ -101,6 +105,20 @@ class CacheDB FlSHA512=(1<<6), FlSource=(1<<7), }; + // the on-disk format changed (FileSize increased to 64bit) in + // commit 650faab0 which will lead to corruption with old caches + struct StatStoreOldFormat + { + uint32_t Flags; + uint32_t mtime; + uint32_t FileSize; + uint8_t MD5[16]; + uint8_t SHA1[20]; + uint8_t SHA256[32]; + } CurStatOldFormat; + + // WARNING: this struct is read/written to the DB so do not change the + // layout of the fields (see lp #1274466), only append to it struct StatStore { uint32_t Flags; -- cgit v1.2.3 From 308b6fc991d2b65a0bf410fb968f1c7b81b4a83b Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Fri, 18 Jul 2014 16:06:34 +0100 Subject: Make Packages & Sources generation optional, during Generate call refactor a bit, extract code out of Generate() into DoGenerate{PackagesAndSources,Contents}, add new APT::FTPArchive::ContentsOnly option to allow skipping the generation of Package/Source files (if they are generated e.g. by some db outside of apt-ftparchives control) --- ftparchive/apt-ftparchive.cc | 99 +++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 33 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index ba71ee225..ebf99a8f8 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -781,33 +781,14 @@ static bool SimpleGenRelease(CommandLine &CmdL) } /*}}}*/ -// Generate - Full generate, using a config file /*{{{*/ +// DoGeneratePackagesAndSources - Helper for Generate /*{{{*/ // --------------------------------------------------------------------- -/* */ -static bool Generate(CommandLine &CmdL) +static bool DoGeneratePackagesAndSources(Configuration &Setup, + vector &PkgList, + struct CacheDB::Stats &SrcStats, + struct CacheDB::Stats &Stats, + CommandLine &CmdL) { - struct CacheDB::Stats SrcStats; - if (CmdL.FileSize() < 2) - return ShowHelp(CmdL); - - struct timeval StartTime; - gettimeofday(&StartTime,0); - struct CacheDB::Stats Stats; - - // Read the configuration file. - Configuration Setup; - if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false) - return false; - - vector PkgList; - LoadTree(PkgList,Setup); - LoadBinDir(PkgList,Setup); - - // Sort by cache DB to improve IO locality. - stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); - stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare()); - - // Generate packages if (CmdL.FileSize() <= 2) { for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ++I) @@ -876,9 +857,16 @@ static bool Generate(CommandLine &CmdL) if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0) delete I->TransWriter; - if (_config->FindB("APT::FTPArchive::Contents",true) == false) - return true; - + return true; +} + + /*}}}*/ +// DoGenerateContents - Helper for Generate to generate the Contents /*{{{*/ +// --------------------------------------------------------------------- +static bool DoGenerateContents(Configuration &Setup, + vector &PkgList, + CommandLine &CmdL) +{ c1out << "Packages done, Starting contents." << endl; // Sort the contents file list by date @@ -935,17 +923,62 @@ static bool Generate(CommandLine &CmdL) break; } } + + return true; +} + + /*}}}*/ +// Generate - Full generate, using a config file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +static bool Generate(CommandLine &CmdL) +{ + struct CacheDB::Stats SrcStats; + if (CmdL.FileSize() < 2) + return ShowHelp(CmdL); + + struct timeval StartTime; + gettimeofday(&StartTime,0); + struct CacheDB::Stats Stats; + // Read the configuration file. + Configuration Setup; + if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false) + return false; + + vector PkgList; + LoadTree(PkgList,Setup); + LoadBinDir(PkgList,Setup); + + // Sort by cache DB to improve IO locality. + stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); + stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare()); + + // Generate packages + if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false) + { + if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false) + return false; + } else { + c1out << "Skipping Packages/Sources generation" << endl; + } + + // do Contents if needed + if (_config->FindB("APT::FTPArchive::Contents", true) == true) + if (DoGenerateContents(Setup, PkgList, CmdL) == false) + return false; + struct timeval NewTime; - gettimeofday(&NewTime,0); - double Delta = NewTime.tv_sec - StartTime.tv_sec + + gettimeofday(&NewTime,0); + double Delta = NewTime.tv_sec - StartTime.tv_sec + (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; - c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages + c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages << " archives. Took " << TimeToStr((long)Delta) << endl; - + return true; } - /*}}}*/ + + /*}}}*/ // Clean - Clean out the databases /*{{{*/ // --------------------------------------------------------------------- /* */ -- cgit v1.2.3 From 25613a61f6f3b9e54d5229af7e2278d0fa54bdd9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Sep 2014 22:16:26 +0200 Subject: fix: Member variable 'X' is not initialized in the constructor. Reported-By: cppcheck Git-Dch: Ignore --- ftparchive/sources.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ftparchive') diff --git a/ftparchive/sources.h b/ftparchive/sources.h index 91e0b1376..9ada15728 100644 --- a/ftparchive/sources.h +++ b/ftparchive/sources.h @@ -17,7 +17,7 @@ class DscExtract bool TakeDsc(const void *Data, unsigned long Size); bool Read(std::string FileName); - DscExtract() : Data(0), Length(0) { + DscExtract() : Data(0), Length(0), IsClearSigned(false) { Data = new char[maxSize]; }; ~DscExtract() { -- cgit v1.2.3 From bf3ad91fdf4967fe107e08a6887f9ed51f4f5eea Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Sep 2014 22:55:56 +0200 Subject: =?UTF-8?q?fix:=20warning:=20extra=20=E2=80=98;=E2=80=99=20[-Wpeda?= =?UTF-8?q?ntic]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported-By: gcc -Wpedantic Git-Dch: Ignore --- ftparchive/cachedb.cc | 6 +++--- ftparchive/cachedb.h | 4 ++-- ftparchive/contents.cc | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 7770a3244..da45eb8d2 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -34,18 +34,18 @@ #include /*}}}*/ -CacheDB::CacheDB(std::string const &DB) +CacheDB::CacheDB(std::string const &DB) : Dbp(0), Fd(NULL), DebFile(0) { TmpKey[0]='\0'; ReadyDB(DB); -}; +} CacheDB::~CacheDB() { ReadyDB(); delete DebFile; -}; +} // CacheDB::ReadyDB - Ready the DB2 /*{{{*/ // --------------------------------------------------------------------- diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index b40fd7df7..613963f6f 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -99,8 +99,8 @@ class CacheDB // Stat info stored in the DB, Fixed types since it is written to disk. enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2), - FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), - FlSHA512=(1<<6), FlSource=(1<<7), + FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), + FlSHA512=(1<<6), FlSource=(1<<7) }; // the on-disk format changed (FileSize increased to 64bit) in diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc index 91dd2b8bd..8c4181eda 100644 --- a/ftparchive/contents.cc +++ b/ftparchive/contents.cc @@ -302,17 +302,17 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf) DoPrint(Out,Top->BTreeRight,Buf); } /*}}}*/ -// ContentsExtract Constructor /*{{{*/ +// ContentsExtract Constructor /*{{{*/ ContentsExtract::ContentsExtract() - : Data(0), MaxSize(0), CurSize(0) + : Data(0), MaxSize(0), CurSize(0) { -}; +} /*}}}*/ -// ContentsExtract Destructor /*{{{*/ +// ContentsExtract Destructor /*{{{*/ ContentsExtract::~ContentsExtract() { free(Data); -}; +} /*}}}*/ // ContentsExtract::Read - Read the archive /*{{{*/ // --------------------------------------------------------------------- -- cgit v1.2.3