summaryrefslogtreecommitdiff
path: root/ftparchive
diff options
context:
space:
mode:
Diffstat (limited to 'ftparchive')
-rw-r--r--ftparchive/apt-ftparchive.cc99
-rw-r--r--ftparchive/cachedb.cc82
-rw-r--r--ftparchive/cachedb.h30
-rw-r--r--ftparchive/contents.cc13
-rw-r--r--ftparchive/contents.h4
-rw-r--r--ftparchive/sources.h2
-rw-r--r--ftparchive/writer.h6
7 files changed, 181 insertions, 55 deletions
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index ba71ee225..ebf99a8f8 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -781,33 +781,14 @@ static bool SimpleGenRelease(CommandLine &CmdL)
}
/*}}}*/
-// Generate - Full generate, using a config file /*{{{*/
+// DoGeneratePackagesAndSources - Helper for Generate /*{{{*/
// ---------------------------------------------------------------------
-/* */
-static bool Generate(CommandLine &CmdL)
+static bool DoGeneratePackagesAndSources(Configuration &Setup,
+ vector<PackageMap> &PkgList,
+ struct CacheDB::Stats &SrcStats,
+ struct CacheDB::Stats &Stats,
+ CommandLine &CmdL)
{
- struct CacheDB::Stats SrcStats;
- if (CmdL.FileSize() < 2)
- return ShowHelp(CmdL);
-
- struct timeval StartTime;
- gettimeofday(&StartTime,0);
- struct CacheDB::Stats Stats;
-
- // Read the configuration file.
- Configuration Setup;
- if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
- return false;
-
- vector<PackageMap> PkgList;
- LoadTree(PkgList,Setup);
- LoadBinDir(PkgList,Setup);
-
- // Sort by cache DB to improve IO locality.
- stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
- stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
-
- // Generate packages
if (CmdL.FileSize() <= 2)
{
for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
@@ -876,9 +857,16 @@ static bool Generate(CommandLine &CmdL)
if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
delete I->TransWriter;
- if (_config->FindB("APT::FTPArchive::Contents",true) == false)
- return true;
-
+ return true;
+}
+
+ /*}}}*/
+// DoGenerateContents - Helper for Generate to generate the Contents /*{{{*/
+// ---------------------------------------------------------------------
+static bool DoGenerateContents(Configuration &Setup,
+ vector<PackageMap> &PkgList,
+ CommandLine &CmdL)
+{
c1out << "Packages done, Starting contents." << endl;
// Sort the contents file list by date
@@ -935,17 +923,62 @@ static bool Generate(CommandLine &CmdL)
break;
}
}
+
+ return true;
+}
+
+ /*}}}*/
+// Generate - Full generate, using a config file /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static bool Generate(CommandLine &CmdL)
+{
+ struct CacheDB::Stats SrcStats;
+ if (CmdL.FileSize() < 2)
+ return ShowHelp(CmdL);
+
+ struct timeval StartTime;
+ gettimeofday(&StartTime,0);
+ struct CacheDB::Stats Stats;
+ // Read the configuration file.
+ Configuration Setup;
+ if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
+ return false;
+
+ vector<PackageMap> PkgList;
+ LoadTree(PkgList,Setup);
+ LoadBinDir(PkgList,Setup);
+
+ // Sort by cache DB to improve IO locality.
+ stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
+ stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
+
+ // Generate packages
+ if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
+ {
+ if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
+ return false;
+ } else {
+ c1out << "Skipping Packages/Sources generation" << endl;
+ }
+
+ // do Contents if needed
+ if (_config->FindB("APT::FTPArchive::Contents", true) == true)
+ if (DoGenerateContents(Setup, PkgList, CmdL) == false)
+ return false;
+
struct timeval NewTime;
- gettimeofday(&NewTime,0);
- double Delta = NewTime.tv_sec - StartTime.tv_sec +
+ gettimeofday(&NewTime,0);
+ double Delta = NewTime.tv_sec - StartTime.tv_sec +
(NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
- c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
+ c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
<< " archives. Took " << TimeToStr((long)Delta) << endl;
-
+
return true;
}
- /*}}}*/
+
+ /*}}}*/
// Clean - Clean out the databases /*{{{*/
// ---------------------------------------------------------------------
/* */
diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index be54ea62c..da45eb8d2 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -34,6 +34,19 @@
#include <apti18n.h>
/*}}}*/
+CacheDB::CacheDB(std::string const &DB)
+ : Dbp(0), Fd(NULL), DebFile(0)
+{
+ TmpKey[0]='\0';
+ ReadyDB(DB);
+}
+
+CacheDB::~CacheDB()
+{
+ ReadyDB();
+ delete DebFile;
+}
+
// CacheDB::ReadyDB - Ready the DB2 /*{{{*/
// ---------------------------------------------------------------------
/* This opens the DB2 file for caching package information */
@@ -88,7 +101,7 @@ bool CacheDB::ReadyDB(std::string const &DB)
return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
}
}
-
+
DBFile = DB;
DBLoaded = true;
return true;
@@ -177,6 +190,45 @@ bool CacheDB::GetFileStat(bool const &doStat)
return true;
}
/*}}}*/
+// CacheDB::GetCurStatCompatOldFormat /*{{{*/
+// ---------------------------------------------------------------------
+/* Read the old (32bit FileSize) StateStore format from disk */
+bool CacheDB::GetCurStatCompatOldFormat()
+{
+ InitQueryStats();
+ Data.data = &CurStatOldFormat;
+ Data.flags = DB_DBT_USERMEM;
+ Data.ulen = sizeof(CurStatOldFormat);
+ if (Get() == false)
+ {
+ CurStat.Flags = 0;
+ } else {
+ CurStat.Flags = CurStatOldFormat.Flags;
+ CurStat.mtime = CurStatOldFormat.mtime;
+ CurStat.FileSize = CurStatOldFormat.FileSize;
+ memcpy(CurStat.MD5, CurStatOldFormat.MD5, sizeof(CurStat.MD5));
+ memcpy(CurStat.SHA1, CurStatOldFormat.SHA1, sizeof(CurStat.SHA1));
+ memcpy(CurStat.SHA256, CurStatOldFormat.SHA256, sizeof(CurStat.SHA256));
+ }
+ return true;
+}
+ /*}}}*/
+// CacheDB::GetCurStatCompatOldFormat /*{{{*/
+// ---------------------------------------------------------------------
+/* Read the new (64bit FileSize) StateStore format from disk */
+bool CacheDB::GetCurStatCompatNewFormat()
+{
+ InitQueryStats();
+ Data.data = &CurStat;
+ Data.flags = DB_DBT_USERMEM;
+ Data.ulen = sizeof(CurStat);
+ if (Get() == false)
+ {
+ CurStat.Flags = 0;
+ }
+ return true;
+}
+ /*}}}*/
// CacheDB::GetCurStat - Set the CurStat variable. /*{{{*/
// ---------------------------------------------------------------------
/* Sets the CurStat variable. Either to 0 if no database is used
@@ -187,19 +239,29 @@ bool CacheDB::GetCurStat()
if (DBLoaded)
{
- /* First see if there is anything about it
- in the database */
-
- /* Get the flags (and mtime) */
+ // do a first query to just get the size of the data on disk
InitQueryStats();
- // Ensure alignment of the returned structure
Data.data = &CurStat;
- Data.ulen = sizeof(CurStat);
Data.flags = DB_DBT_USERMEM;
- if (Get() == false)
+ Data.ulen = 0;
+ Get();
+
+ if (Data.size == 0)
+ {
+ // nothing needs to be done, we just have not data for this deb
+ }
+ // check if the record is written in the old format (32bit filesize)
+ else if(Data.size == sizeof(CurStatOldFormat))
{
- CurStat.Flags = 0;
- }
+ GetCurStatCompatOldFormat();
+ }
+ else if(Data.size == sizeof(CurStat))
+ {
+ GetCurStatCompatNewFormat();
+ } else {
+ return _error->Error("Cache record size mismatch (%ul)", Data.size);
+ }
+
CurStat.Flags = ntohl(CurStat.Flags);
CurStat.FileSize = ntohl(CurStat.FileSize);
}
diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h
index 14dba5578..613963f6f 100644
--- a/ftparchive/cachedb.h
+++ b/ftparchive/cachedb.h
@@ -86,8 +86,12 @@ class CacheDB
bool OpenDebFile();
void CloseDebFile();
- bool GetFileStat(bool const &doStat = false);
+ // GetCurStat needs some compat code, see lp #1274466)
+ bool GetCurStatCompatOldFormat();
+ bool GetCurStatCompatNewFormat();
bool GetCurStat();
+
+ bool GetFileStat(bool const &doStat = false);
bool LoadControl();
bool LoadContents(bool const &GenOnly);
bool LoadSource();
@@ -95,10 +99,24 @@ class CacheDB
// Stat info stored in the DB, Fixed types since it is written to disk.
enum FlagList {FlControl = (1<<0),FlMD5=(1<<1),FlContents=(1<<2),
- FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5),
- FlSHA512=(1<<6), FlSource=(1<<7),
+ FlSize=(1<<3), FlSHA1=(1<<4), FlSHA256=(1<<5),
+ FlSHA512=(1<<6), FlSource=(1<<7)
};
+ // the on-disk format changed (FileSize increased to 64bit) in
+ // commit 650faab0 which will lead to corruption with old caches
+ struct StatStoreOldFormat
+ {
+ uint32_t Flags;
+ uint32_t mtime;
+ uint32_t FileSize;
+ uint8_t MD5[16];
+ uint8_t SHA1[20];
+ uint8_t SHA256[32];
+ } CurStatOldFormat;
+
+ // WARNING: this struct is read/written to the DB so do not change the
+ // layout of the fields (see lp #1274466), only append to it
struct StatStore
{
uint32_t Flags;
@@ -150,7 +168,7 @@ class CacheDB
SHA512Bytes(0),Packages(0), Misses(0), DeLinkBytes(0) {};
} Stats;
- bool ReadyDB(std::string const &DB);
+ bool ReadyDB(std::string const &DB = "");
inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;};
inline bool Loaded() {return DBLoaded == true;};
@@ -171,8 +189,8 @@ class CacheDB
bool Clean();
- CacheDB(std::string const &DB) : Dbp(0), Fd(NULL), DebFile(0) {TmpKey[0]='\0'; ReadyDB(DB);};
- ~CacheDB() {ReadyDB(std::string()); delete DebFile;};
+ CacheDB(std::string const &DB);
+ ~CacheDB();
};
#endif
diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc
index 7a1fb779e..8c4181eda 100644
--- a/ftparchive/contents.cc
+++ b/ftparchive/contents.cc
@@ -302,7 +302,18 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
DoPrint(Out,Top->BTreeRight,Buf);
}
/*}}}*/
-
+// ContentsExtract Constructor /*{{{*/
+ContentsExtract::ContentsExtract()
+ : Data(0), MaxSize(0), CurSize(0)
+{
+}
+ /*}}}*/
+// ContentsExtract Destructor /*{{{*/
+ContentsExtract::~ContentsExtract()
+{
+ free(Data);
+}
+ /*}}}*/
// ContentsExtract::Read - Read the archive /*{{{*/
// ---------------------------------------------------------------------
/* */
diff --git a/ftparchive/contents.h b/ftparchive/contents.h
index dbbb83350..f58e3278e 100644
--- a/ftparchive/contents.h
+++ b/ftparchive/contents.h
@@ -85,8 +85,8 @@ class ContentsExtract : public pkgDirStream
bool TakeContents(const void *Data,unsigned long long Length);
void Add(GenContents &Contents,std::string const &Package);
- ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {};
- virtual ~ContentsExtract() {delete [] Data;};
+ ContentsExtract();
+ virtual ~ContentsExtract();
};
#endif
diff --git a/ftparchive/sources.h b/ftparchive/sources.h
index 91e0b1376..9ada15728 100644
--- a/ftparchive/sources.h
+++ b/ftparchive/sources.h
@@ -17,7 +17,7 @@ class DscExtract
bool TakeDsc(const void *Data, unsigned long Size);
bool Read(std::string FileName);
- DscExtract() : Data(0), Length(0) {
+ DscExtract() : Data(0), Length(0), IsClearSigned(false) {
Data = new char[maxSize];
};
~DscExtract() {
diff --git a/ftparchive/writer.h b/ftparchive/writer.h
index c62d4addf..226996475 100644
--- a/ftparchive/writer.h
+++ b/ftparchive/writer.h
@@ -126,8 +126,10 @@ class PackagesWriter : public FTWScanner
{return Over.ReadExtraOverride(File);};
virtual bool DoPackage(string FileName);
- PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides=string(),
- string const &Arch=string());
+ PackagesWriter(string const &DB,
+ string const &Overrides,
+ string const &ExtOverrides = "",
+ string const &Arch = "");
virtual ~PackagesWriter() {};
};
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-06 04:40:30 +0000