From ce928105d7279c5604f034740b04dc6a745fb859 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 14:30:17 +0200 Subject: Implement CacheDB for source packages in apt-ftparchive --- ftparchive/cachedb.cc | 146 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 101 insertions(+), 45 deletions(-) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 523c6b5fa..a63b5b9d9 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include // htonl, etc #include @@ -99,13 +100,32 @@ bool CacheDB::OpenFile() Fd = new FileFd(FileName,FileFd::ReadOnly); if (_error->PendingError() == true) { - delete Fd; - Fd = NULL; - return false; + CloseFile(); + return false; } return true; } /*}}}*/ +void CacheDB::CloseFile() +{ + delete Fd; + Fd = NULL; +} + +bool CacheDB::OpenDebFile() +{ + DebFile = new debDebFile(*Fd); + if (_error->PendingError() == true) + return false; + return true; +} + +void CacheDB::CloseDebFile() +{ + delete DebFile; + DebFile = NULL; +} + // CacheDB::GetFileStat - Get stats from the file /*{{{*/ // --------------------------------------------------------------------- /* This gets the size from the database if it's there. If we need @@ -168,56 +188,94 @@ bool CacheDB::GetCurStat() /*}}}*/ // CacheDB::GetFileInfo - Get all the info about the file /*{{{*/ // --------------------------------------------------------------------- -bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents, - bool const &GenContentsOnly, bool const &DoMD5, bool const &DoSHA1, - bool const &DoSHA256, bool const &DoSHA512, +bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, + bool const &DoContents, + bool const &GenContentsOnly, + bool const &DoSource, + bool const &DoMD5, bool const &DoSHA1, + bool const &DoSHA256, bool const &DoSHA512, bool const &checkMtime) { - this->FileName = FileName; + bool result = true; + this->FileName = FileName; - if (GetCurStat() == false) + if (GetCurStat() == false) { - return false; + return false; } OldStat = CurStat; - if (GetFileStat(checkMtime) == false) - { - delete Fd; - Fd = NULL; - return false; - } + if (GetFileStat(checkMtime) == false) + { + CloseFile(); + return false; + } /* if mtime changed, update CurStat from disk */ if (checkMtime == true && OldStat.mtime != CurStat.mtime) - CurStat.Flags = FlSize; - - Stats.Bytes += CurStat.FileSize; - Stats.Packages++; - - if ((DoControl && LoadControl() == false) - || (DoContents && LoadContents(GenContentsOnly) == false) - || (DoMD5 && GetMD5(false) == false) - || (DoSHA1 && GetSHA1(false) == false) - || (DoSHA256 && GetSHA256(false) == false) - || (DoSHA512 && GetSHA512(false) == false) - ) - { - delete Fd; - Fd = NULL; - delete DebFile; - DebFile = NULL; - return false; - } + CurStat.Flags = FlSize; - delete Fd; - Fd = NULL; - delete DebFile; - DebFile = NULL; + Stats.Bytes += CurStat.FileSize; + Stats.Packages++; - return true; + if ((DoControl && LoadControl() == false) + || (DoContents && LoadContents(GenContentsOnly) == false) + || (DoSource && LoadSource() == false) + || (DoMD5 && GetMD5(false) == false) + || (DoSHA1 && GetSHA1(false) == false) + || (DoSHA256 && GetSHA256(false) == false) + || (DoSHA512 && GetSHA512(false) == false) + ) + { + result = false; + } + + CloseFile(); + CloseDebFile(); + + return result; } /*}}}*/ + +bool CacheDB::LoadSource() +{ + // Try to read the control information out of the DB. + if ((CurStat.Flags & FlSource) == FlSource) + { + // Lookup the control information + InitQuery("cs"); + if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true) + return true; + CurStat.Flags &= ~FlSource; + } + + if (Fd == NULL && OpenFile() == false) + { + return false; + } + + // Read the .dsc file + if (Fd == NULL) + { + if(OpenFile() == false) + return false; + } + + Stats.Misses++; + if (Dsc.Read(FileName) == false) + return false; + + if (Dsc.Data == 0) + return _error->Error(_("Failed to read .dsc")); + + // Write back the control information + InitQuery("cs"); + if (Put(Dsc.Data, Dsc.Length) == true) + CurStat.Flags |= FlSource; + + return true; +} + // CacheDB::LoadControl - Load Control information /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -238,11 +296,10 @@ bool CacheDB::LoadControl() return false; } // Create a deb instance to read the archive - if (DebFile == 0) + if (DebFile == NULL) { - DebFile = new debDebFile(*Fd); - if (_error->PendingError() == true) - return false; + if(OpenDebFile() == false) + return false; } Stats.Misses++; @@ -288,8 +345,7 @@ bool CacheDB::LoadContents(bool const &GenOnly) // Create a deb instance to read the archive if (DebFile == 0) { - DebFile = new debDebFile(*Fd); - if (_error->PendingError() == true) + if(OpenDebFile() == false) return false; } -- cgit v1.2.3 From 0a3b93fc3da95c5cbeb18b2d92738cbd50e95d83 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 15:36:42 +0200 Subject: add test for binary cachedb and contents generation --- ftparchive/cachedb.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index a63b5b9d9..f6cbad668 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -349,6 +349,7 @@ bool CacheDB::LoadContents(bool const &GenOnly) return false; } + Stats.Misses++; if (Contents.Read(*DebFile) == false) return false; -- cgit v1.2.3 From 215b0faf7b00f836e54f9903c4fe7398c0927e0f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 16:15:26 +0200 Subject: refactor to make OpenFile/OpenDebFile more robust --- ftparchive/cachedb.cc | 230 ++++++++++++++++++++++++-------------------------- 1 file changed, 110 insertions(+), 120 deletions(-) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index f6cbad668..539ed671b 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -97,64 +97,92 @@ bool CacheDB::ReadyDB(std::string const &DB) /* */ bool CacheDB::OpenFile() { - Fd = new FileFd(FileName,FileFd::ReadOnly); - if (_error->PendingError() == true) - { - CloseFile(); - return false; - } - return true; + // its open already + if(Fd && Fd->Name() == this->FileName) + return true; + + // a different file is open, close it first + if(Fd && Fd->Name() != this->FileName) + CloseFile(); + + // open a new file + Fd = new FileFd(FileName,FileFd::ReadOnly); + if (_error->PendingError() == true) + { + CloseFile(); + return false; + } + return true; } /*}}}*/ +// CacheDB::CloseFile - Close the file /*{{{*/ void CacheDB::CloseFile() { - delete Fd; - Fd = NULL; + if(Fd != NULL) + { + delete Fd; + Fd = NULL; + } } - + /*}}}*/ +// CacheDB::OpenDebFile - Open a debfile /*{{{*/ bool CacheDB::OpenDebFile() { + // debfile is already open + if(DebFile && &DebFile->GetFile() == Fd) + return true; + + // a different debfile is open, close it first + if(DebFile && &DebFile->GetFile() != Fd) + CloseDebFile(); + + // first open the fd, then pass it to the debDebFile + if(OpenFile() == false) + return false; DebFile = new debDebFile(*Fd); if (_error->PendingError() == true) return false; return true; } - + /*}}}*/ +// CacheDB::CloseDebFile - Close a debfile again /*{{{*/ void CacheDB::CloseDebFile() { - delete DebFile; - DebFile = NULL; -} + CloseFile(); + if(DebFile != NULL) + { + delete DebFile; + DebFile = NULL; + } +} + /*}}}*/ // CacheDB::GetFileStat - Get stats from the file /*{{{*/ // --------------------------------------------------------------------- /* This gets the size from the database if it's there. If we need * to look at the file, also get the mtime from the file. */ bool CacheDB::GetFileStat(bool const &doStat) { - if ((CurStat.Flags & FlSize) == FlSize && doStat == false) - { - /* Already worked out the file size */ - } - else - { - /* Get it from the file. */ - if (Fd == NULL && OpenFile() == false) - { - return false; - } - // Stat the file - struct stat St; - if (fstat(Fd->Fd(),&St) != 0) - { - return _error->Errno("fstat", - _("Failed to stat %s"),FileName.c_str()); - } - CurStat.FileSize = St.st_size; - CurStat.mtime = htonl(St.st_mtime); - CurStat.Flags |= FlSize; - } - return true; + if ((CurStat.Flags & FlSize) == FlSize && doStat == false) + return true; + + /* Get it from the file. */ + if (OpenFile() == false) + return false; + + // Stat the file + struct stat St; + if (fstat(Fd->Fd(),&St) != 0) + { + CloseFile(); + return _error->Errno("fstat", + _("Failed to stat %s"),FileName.c_str()); + } + CurStat.FileSize = St.st_size; + CurStat.mtime = htonl(St.st_mtime); + CurStat.Flags |= FlSize; + + return true; } /*}}}*/ // CacheDB::GetCurStat - Set the CurStat variable. /*{{{*/ @@ -165,25 +193,25 @@ bool CacheDB::GetCurStat() { memset(&CurStat,0,sizeof(CurStat)); - if (DBLoaded) - { - /* First see if there is anything about it - in the database */ - - /* Get the flags (and mtime) */ - InitQuery("st"); - // Ensure alignment of the returned structure - Data.data = &CurStat; - Data.ulen = sizeof(CurStat); - Data.flags = DB_DBT_USERMEM; - if (Get() == false) + if (DBLoaded) + { + /* First see if there is anything about it + in the database */ + + /* Get the flags (and mtime) */ + InitQuery("st"); + // Ensure alignment of the returned structure + Data.data = &CurStat; + Data.ulen = sizeof(CurStat); + Data.flags = DB_DBT_USERMEM; + if (Get() == false) { CurStat.Flags = 0; } - CurStat.Flags = ntohl(CurStat.Flags); - CurStat.FileSize = ntohl(CurStat.FileSize); + CurStat.Flags = ntohl(CurStat.Flags); + CurStat.FileSize = ntohl(CurStat.FileSize); } - return true; + return true; } /*}}}*/ // CacheDB::GetFileInfo - Get all the info about the file /*{{{*/ @@ -200,40 +228,31 @@ bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, this->FileName = FileName; if (GetCurStat() == false) - { return false; - } OldStat = CurStat; if (GetFileStat(checkMtime) == false) - { - CloseFile(); return false; - } - /* if mtime changed, update CurStat from disk */ - if (checkMtime == true && OldStat.mtime != CurStat.mtime) - CurStat.Flags = FlSize; - - Stats.Bytes += CurStat.FileSize; - Stats.Packages++; - - if ((DoControl && LoadControl() == false) - || (DoContents && LoadContents(GenContentsOnly) == false) - || (DoSource && LoadSource() == false) - || (DoMD5 && GetMD5(false) == false) - || (DoSHA1 && GetSHA1(false) == false) - || (DoSHA256 && GetSHA256(false) == false) - || (DoSHA512 && GetSHA512(false) == false) - ) - { - result = false; - } - - CloseFile(); - CloseDebFile(); + /* if mtime changed, update CurStat from disk */ + if (checkMtime == true && OldStat.mtime != CurStat.mtime) + CurStat.Flags = FlSize; + + Stats.Bytes += CurStat.FileSize; + Stats.Packages++; + + if ((DoControl && LoadControl() == false) + || (DoContents && LoadContents(GenContentsOnly) == false) + || (DoSource && LoadSource() == false) + || (DoMD5 && GetMD5(false) == false) + || (DoSHA1 && GetSHA1(false) == false) + || (DoSHA256 && GetSHA256(false) == false) + || (DoSHA512 && GetSHA512(false) == false) ) + { + result = false; + } - return result; + return result; } /*}}}*/ @@ -249,18 +268,9 @@ bool CacheDB::LoadSource() CurStat.Flags &= ~FlSource; } - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } - // Read the .dsc file - if (Fd == NULL) - { - if(OpenFile() == false) - return false; - } - Stats.Misses++; if (Dsc.Read(FileName) == false) return false; @@ -291,16 +301,8 @@ bool CacheDB::LoadControl() CurStat.Flags &= ~FlControl; } - if (Fd == NULL && OpenFile() == false) - { + if(OpenDebFile() == false) return false; - } - // Create a deb instance to read the archive - if (DebFile == NULL) - { - if(OpenDebFile() == false) - return false; - } Stats.Misses++; if (Control.Read(*DebFile) == false) @@ -338,16 +340,8 @@ bool CacheDB::LoadContents(bool const &GenOnly) CurStat.Flags &= ~FlContents; } - if (Fd == NULL && OpenFile() == false) - { + if(OpenDebFile() == false) return false; - } - // Create a deb instance to read the archive - if (DebFile == 0) - { - if(OpenDebFile() == false) - return false; - } Stats.Misses++; if (Contents.Read(*DebFile) == false) @@ -404,14 +398,13 @@ bool CacheDB::GetMD5(bool const &GenOnly) MD5Res = bytes2hex(CurStat.MD5, sizeof(CurStat.MD5)); return true; - } + } Stats.MD5Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + MD5Summation MD5; if (Fd->Seek(0) == false || MD5.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -439,10 +432,9 @@ bool CacheDB::GetSHA1(bool const &GenOnly) Stats.SHA1Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + SHA1Summation SHA1; if (Fd->Seek(0) == false || SHA1.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -470,10 +462,9 @@ bool CacheDB::GetSHA256(bool const &GenOnly) Stats.SHA256Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + SHA256Summation SHA256; if (Fd->Seek(0) == false || SHA256.AddFD(*Fd, CurStat.FileSize) == false) return false; @@ -501,10 +492,9 @@ bool CacheDB::GetSHA512(bool const &GenOnly) Stats.SHA512Bytes += CurStat.FileSize; - if (Fd == NULL && OpenFile() == false) - { + if (OpenFile() == false) return false; - } + SHA512Summation SHA512; if (Fd->Seek(0) == false || SHA512.AddFD(*Fd, CurStat.FileSize) == false) return false; -- cgit v1.2.3 From 53ba4e2c2dd29758be0a911489ca5c23e5107513 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 17:09:43 +0200 Subject: ensure clean works --- ftparchive/cachedb.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 539ed671b..f63aa88ab 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -264,10 +264,11 @@ bool CacheDB::LoadSource() // Lookup the control information InitQuery("cs"); if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true) + { return true; + } CurStat.Flags &= ~FlSource; } - if (OpenFile() == false) return false; @@ -551,16 +552,24 @@ bool CacheDB::Clean() { if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 || stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 || + stringcmp(Colon + 1, (char *)Key.data+Key.size,"cs") == 0 || stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0) { - if (FileExists(std::string((const char *)Key.data,Colon)) == true) - continue; + std::string FileName = std::string((const char *)Key.data,Colon); + if (FileExists(FileName) == true) { + continue; + } } } - Cursor->c_del(Cursor,0); } - Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL); + int res = Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL); + if (res < 0) + _error->Warning("compact failed with result %i", res); + + if(_config->FindB("Debug::APT::FTPArchive::Clean", false) == true) + Dbp->stat_print(Dbp, 0); + return true; } -- cgit v1.2.3 From 37497bd5fa0f070e12c1c28d849aef1af8f369b5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 4 Apr 2014 17:21:40 +0200 Subject: refactor _InitQuery() --- ftparchive/cachedb.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index f63aa88ab..4feb7bbfb 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -199,7 +199,7 @@ bool CacheDB::GetCurStat() in the database */ /* Get the flags (and mtime) */ - InitQuery("st"); + InitQueryStats(); // Ensure alignment of the returned structure Data.data = &CurStat; Data.ulen = sizeof(CurStat); @@ -262,7 +262,7 @@ bool CacheDB::LoadSource() if ((CurStat.Flags & FlSource) == FlSource) { // Lookup the control information - InitQuery("cs"); + InitQuerySource(); if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true) { return true; @@ -280,7 +280,7 @@ bool CacheDB::LoadSource() return _error->Error(_("Failed to read .dsc")); // Write back the control information - InitQuery("cs"); + InitQuerySource(); if (Put(Dsc.Data, Dsc.Length) == true) CurStat.Flags |= FlSource; @@ -296,7 +296,7 @@ bool CacheDB::LoadControl() if ((CurStat.Flags & FlControl) == FlControl) { // Lookup the control information - InitQuery("cl"); + InitQueryControl(); if (Get() == true && Control.TakeControl(Data.data,Data.size) == true) return true; CurStat.Flags &= ~FlControl; @@ -313,7 +313,7 @@ bool CacheDB::LoadControl() return _error->Error(_("Archive has no control record")); // Write back the control information - InitQuery("cl"); + InitQueryControl(); if (Put(Control.Control,Control.Length) == true) CurStat.Flags |= FlControl; return true; @@ -331,7 +331,7 @@ bool CacheDB::LoadContents(bool const &GenOnly) return true; // Lookup the contents information - InitQuery("cn"); + InitQueryContent(); if (Get() == true) { if (Contents.TakeContents(Data.data,Data.size) == true) @@ -349,7 +349,7 @@ bool CacheDB::LoadContents(bool const &GenOnly) return false; // Write back the control information - InitQuery("cn"); + InitQueryContent(); if (Put(Contents.Data,Contents.CurSize) == true) CurStat.Flags |= FlContents; return true; @@ -519,7 +519,7 @@ bool CacheDB::Finish() // Write the stat information CurStat.Flags = htonl(CurStat.Flags); CurStat.FileSize = htonl(CurStat.FileSize); - InitQuery("st"); + InitQueryStats(); Put(&CurStat,sizeof(CurStat)); CurStat.Flags = ntohl(CurStat.Flags); CurStat.FileSize = ntohl(CurStat.FileSize); -- cgit v1.2.3 From cf6bbca0a93b21ab7d3378f26dd9b57951a1d987 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 7 Apr 2014 09:41:20 +0200 Subject: ensure "--db" also works with the new srcpkgdb --- ftparchive/cachedb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 4feb7bbfb..d589c4c5a 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -515,7 +515,7 @@ bool CacheDB::Finish() if (CurStat.Flags == OldStat.Flags && CurStat.mtime == OldStat.mtime) return true; - + // Write the stat information CurStat.Flags = htonl(CurStat.Flags); CurStat.FileSize = htonl(CurStat.FileSize); -- cgit v1.2.3 From acea28d0a3a55c4df1390c42288043002610fbc9 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 8 May 2014 11:46:29 +0200 Subject: fix regression from commit 215b0faf --- ftparchive/cachedb.cc | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) (limited to 'ftparchive/cachedb.cc') diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index d589c4c5a..e56deae1e 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -97,13 +97,8 @@ bool CacheDB::ReadyDB(std::string const &DB) /* */ bool CacheDB::OpenFile() { - // its open already - if(Fd && Fd->Name() == this->FileName) - return true; - - // a different file is open, close it first - if(Fd && Fd->Name() != this->FileName) - CloseFile(); + // always close existing file first + CloseFile(); // open a new file Fd = new FileFd(FileName,FileFd::ReadOnly); @@ -128,13 +123,8 @@ void CacheDB::CloseFile() // CacheDB::OpenDebFile - Open a debfile /*{{{*/ bool CacheDB::OpenDebFile() { - // debfile is already open - if(DebFile && &DebFile->GetFile() == Fd) - return true; - - // a different debfile is open, close it first - if(DebFile && &DebFile->GetFile() != Fd) - CloseDebFile(); + // always close existing file first + CloseDebFile(); // first open the fd, then pass it to the debDebFile if(OpenFile() == false) -- cgit v1.2.3