From ce928105d7279c5604f034740b04dc6a745fb859 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 14:30:17 +0200 Subject: Implement CacheDB for source packages in apt-ftparchive --- ftparchive/cachedb.h | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'ftparchive/cachedb.h') diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 49b9a0ef5..4e33c8635 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -22,9 +22,11 @@ #include #include "contents.h" +#include "sources.h" class FileFd; + class CacheDB { protected: @@ -65,10 +67,16 @@ class CacheDB return true; } bool OpenFile(); + void CloseFile(); + + bool OpenDebFile(); + void CloseDebFile(); + bool GetFileStat(bool const &doStat = false); bool GetCurStat(); bool LoadControl(); bool LoadContents(bool const &GenOnly); + bool LoadSource(); bool GetMD5(bool const &GenOnly); bool GetSHA1(bool const &GenOnly); bool GetSHA256(bool const &GenOnly); @@ -77,7 +85,8 @@ class CacheDB // Stat info stored in the DB, Fixed types since it is written to disk. enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2), FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), - FlSHA512=(1<<6)}; + FlSHA512=(1<<6), FlSource=(1<<7), + }; struct StatStore { @@ -101,6 +110,8 @@ class CacheDB // Data collection helpers debDebFile::MemControlExtract Control; ContentsExtract Contents; + DscExtract Dsc; + std::string MD5Res; std::string SHA1Res; std::string SHA256Res; @@ -139,8 +150,19 @@ class CacheDB inline unsigned long long GetFileSize(void) {return CurStat.FileSize;} bool SetFile(std::string const &FileName,struct stat St,FileFd *Fd); - bool GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents, bool const &GenContentsOnly, - bool const &DoMD5, bool const &DoSHA1, bool const &DoSHA256, bool const &DoSHA512, bool const &checkMtime = false); + + // terrible old overloaded interface + bool GetFileInfo(std::string const &FileName, + bool const &DoControl, + bool const &DoContents, + bool const &GenContentsOnly, + bool const &DoSource, + bool const &DoMD5, + bool const &DoSHA1, + bool const &DoSHA256, + bool const &DoSHA512, + bool const &checkMtime = false); + bool Finish(); bool Clean(); -- cgit v1.2.3 From 37497bd5fa0f070e12c1c28d849aef1af8f369b5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 17:21:40 +0200 Subject: refactor _InitQuery() --- ftparchive/cachedb.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'ftparchive/cachedb.h') diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 4e33c8635..54a274944 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -41,7 +41,7 @@ class CacheDB std::string DBFile; // Generate a key for the DB of a given type - inline void InitQuery(const char *Type) + void _InitQuery(const char *Type) { memset(&Key,0,sizeof(Key)); memset(&Data,0,sizeof(Data)); @@ -49,6 +49,19 @@ class CacheDB Key.size = snprintf(TmpKey,sizeof(TmpKey),"%s:%s",FileName.c_str(), Type); } + void InitQueryStats() { + _InitQuery("st"); + } + void InitQuerySource() { + _InitQuery("cs"); + } + void InitQueryControl() { + _InitQuery("cl"); + } + void InitQueryContent() { + _InitQuery("cn"); + } + inline bool Get() { return Dbp->get(Dbp,0,&Key,&Data,0) == 0; -- cgit v1.2.3 From a311fb96b84757ef8628e6a754232614a53b7891 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 27 Apr 2014 18:23:20 +0200 Subject: deal with hashes in ftparchive more dynamic as well Now that libapts acquire system happily passes around hashes and can be made to support new ones without an ABI break in the future, we can free ftparchive from all the deprecation warnings the last commit introduced for it. The goal here isn't to preserve ABI as we have none to keep here, but to help avoiding introduction problems of 'new' hashes later as bugs creep into the copy&paste parts, so short/less of them is good. --- ftparchive/cachedb.h | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'ftparchive/cachedb.h') diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 54a274944..14dba5578 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -12,6 +12,7 @@ #ifndef CACHEDB_H #define CACHEDB_H +#include #include #include @@ -90,11 +91,8 @@ class CacheDB bool LoadControl(); bool LoadContents(bool const &GenOnly); bool LoadSource(); - bool GetMD5(bool const &GenOnly); - bool GetSHA1(bool const &GenOnly); - bool GetSHA256(bool const &GenOnly); - bool GetSHA512(bool const &GenOnly); - + bool GetHashes(bool const GenOnly, unsigned int const DoHashes); + // Stat info stored in the DB, Fixed types since it is written to disk. enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2), FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), @@ -124,12 +122,8 @@ class CacheDB debDebFile::MemControlExtract Control; ContentsExtract Contents; DscExtract Dsc; + HashStringList HashesList; - std::string MD5Res; - std::string SHA1Res; - std::string SHA256Res; - std::string SHA512Res; - // Runtime statistics struct Stats { @@ -165,16 +159,13 @@ class CacheDB bool SetFile(std::string const &FileName,struct stat St,FileFd *Fd); // terrible old overloaded interface - bool GetFileInfo(std::string const &FileName, - bool const &DoControl, - bool const &DoContents, - bool const &GenContentsOnly, - bool const &DoSource, - bool const &DoMD5, - bool const &DoSHA1, - bool const &DoSHA256, - bool const &DoSHA512, - bool const &checkMtime = false); + bool GetFileInfo(std::string const &FileName, + bool const &DoControl, + bool const &DoContents, + bool const &GenContentsOnly, + bool const DoSource, + unsigned int const DoHashes, + bool const &checkMtime = false); bool Finish(); -- cgit v1.2.3 From 21ea1dbb50176a89e7f456f9b31220ff3097fdf2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 27 May 2014 16:25:43 +0200 Subject: use free() instead of delete() when realloc is used ContentsExtract::~ContentsExtract() needs to use free() because Data got allocated via realloc() Reported-By: clang -fsanitize=address -fno-omit-frame-pointer --- ftparchive/cachedb.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'ftparchive/cachedb.h') diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 54a274944..edb8594bf 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -156,7 +156,7 @@ class CacheDB SHA512Bytes(0),Packages(0), Misses(0), DeLinkBytes(0) {}; } Stats; - bool ReadyDB(std::string const &DB); + bool ReadyDB(std::string const &DB = ""); inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;}; inline bool Loaded() {return DBLoaded == true;}; @@ -180,8 +180,8 @@ class CacheDB bool Clean(); - CacheDB(std::string const &DB) : Dbp(0), Fd(NULL), DebFile(0) {TmpKey[0]='\0'; ReadyDB(DB);}; - ~CacheDB() {ReadyDB(std::string()); delete DebFile;}; + CacheDB(std::string const &DB); + ~CacheDB(); }; #endif -- cgit v1.2.3 From 243b2a381f4a12939d91084ecf100ee6d3dcb007 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 4 Jun 2014 12:39:36 +0200 Subject: Add compat mode for old (32bit FileSize) CacheDB (LP: #1274466) --- ftparchive/cachedb.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'ftparchive/cachedb.h') diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index edb8594bf..29d710d2c 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -85,8 +85,12 @@ class CacheDB bool OpenDebFile(); void CloseDebFile(); - bool GetFileStat(bool const &doStat = false); + // GetCurStat needs some compat code, see lp #1274466) + bool GetCurStatCompatOldFormat(); + bool GetCurStatCompatNewFormat(); bool GetCurStat(); + + bool GetFileStat(bool const &doStat = false); bool LoadControl(); bool LoadContents(bool const &GenOnly); bool LoadSource(); @@ -101,6 +105,20 @@ class CacheDB FlSHA512=(1<<6), FlSource=(1<<7), }; + // the on-disk format changed (FileSize increased to 64bit) in + // commit 650faab0 which will lead to corruption with old caches + struct StatStoreOldFormat + { + uint32_t Flags; + uint32_t mtime; + uint32_t FileSize; + uint8_t MD5[16]; + uint8_t SHA1[20]; + uint8_t SHA256[32]; + } CurStatOldFormat; + + // WARNING: this struct is read/written to the DB so do not change the + // layout of the fields (see lp #1274466), only append to it struct StatStore { uint32_t Flags; -- cgit v1.2.3 From bf3ad91fdf4967fe107e08a6887f9ed51f4f5eea Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Sep 2014 22:55:56 +0200 Subject: =?UTF-8?q?fix:=20warning:=20extra=20=E2=80=98;=E2=80=99=20[-Wpeda?= =?UTF-8?q?ntic]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported-By: gcc -Wpedantic Git-Dch: Ignore --- ftparchive/cachedb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ftparchive/cachedb.h') diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index b40fd7df7..613963f6f 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -99,8 +99,8 @@ class CacheDB // Stat info stored in the DB, Fixed types since it is written to disk. enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2), - FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), - FlSHA512=(1<<6), FlSource=(1<<7), + FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5), + FlSHA512=(1<<6), FlSource=(1<<7) }; // the on-disk format changed (FileSize increased to 64bit) in -- cgit v1.2.3