From a93a32b5cda4f2a0572365a8b919329ea6af2c66 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 29 May 2014 10:22:09 +0200 Subject: add hashtable stats --- cmdline/apt-cache.cc | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 84b775390..a620740e1 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -373,7 +373,63 @@ static bool Stats(CommandLine &) Cache->Head().VerFileCount*Cache->Head().VerFileSz + Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; - + + // get the hash collisions average + long NumBuckets = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); + long UsedBuckets = 0; + long UnusedBuckets = 0; + long LongestBucket = 0; + long ShortestBucket = NumBuckets; + for (unsigned int i=0; i < NumBuckets; ++i) + { + pkgCache::Package *Pkg = Cache->PkgP + Cache->HeaderP->PkgHashTable[i]; + if(Pkg == 0 || Pkg == Cache->PkgP) + { + UnusedBuckets++; + continue; + } + long ThisBucketSize = 0; + for (; Pkg != Cache->PkgP; Pkg = Cache->PkgP + Pkg->NextPackage) + ThisBucketSize++; + LongestBucket = std::max(ThisBucketSize, LongestBucket); + ShortestBucket = std::min(ThisBucketSize, ShortestBucket); + UsedBuckets += ThisBucketSize; + } + cout << _("Pkg Hashtable stats:") << endl; + cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; + cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; + cout << _("Used buckets: ") << UsedBuckets << endl; + cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; + cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; + + // get the hash collisions average + NumBuckets = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); + UsedBuckets = 0; + UnusedBuckets = 0; + LongestBucket = 0; + ShortestBucket = NumBuckets; + for (unsigned int i=0; i < NumBuckets; ++i) + { + pkgCache::Group *Grp = Cache->GrpP + Cache->HeaderP->GrpHashTable[i]; + if(Grp == 0 || Grp == Cache->GrpP) + { + UnusedBuckets++; + continue; + } + long ThisBucketSize = 0; + for (; Grp != Cache->GrpP; Grp = Cache->GrpP + Grp->Next) + ThisBucketSize++; + LongestBucket = std::max(ThisBucketSize, LongestBucket); + ShortestBucket = std::min(ThisBucketSize, ShortestBucket); + UsedBuckets += ThisBucketSize; + } + cout << _("Grp Hashtable stats:") << endl; + cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; + cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; + cout << _("Used buckets: ") << UsedBuckets << endl; + cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; + cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; + return true; } /*}}}*/ -- cgit v1.2.3 From 4dde2b4285fc6288e44e915a0d7bc0faac114a2e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jun 2014 14:12:12 +0200 Subject: support Acquire::GzipIndexes in dumpavail Closes: 742835 --- cmdline/apt-cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 84b775390..1414617eb 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -507,7 +507,7 @@ static bool DumpAvail(CommandLine &) break; } - FileFd PkgF(File.FileName(),FileFd::ReadOnly); + FileFd PkgF(File.FileName(),FileFd::ReadOnly, FileFd::Extension); if (_error->PendingError() == true) break; -- cgit v1.2.3 From eceb418be84f6271c10e374281c1a8379647679b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 18 Jun 2014 08:51:59 +0200 Subject: improve formating of the hash stats --- cmdline/apt-cache.cc | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index a620740e1..072cf4ef7 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -374,7 +374,7 @@ static bool Stats(CommandLine &) Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; - // get the hash collisions average + // hashtable stats for the PkgHashTable long NumBuckets = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); long UsedBuckets = 0; long UnusedBuckets = 0; @@ -395,14 +395,14 @@ static bool Stats(CommandLine &) ShortestBucket = std::min(ThisBucketSize, ShortestBucket); UsedBuckets += ThisBucketSize; } - cout << _("Pkg Hashtable stats:") << endl; - cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; - cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; - cout << _("Used buckets: ") << UsedBuckets << endl; - cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; - cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; - - // get the hash collisions average + cout << "Total PkgHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; + cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << " Used: " << UsedBuckets << std::endl; + cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Longest: " << LongestBucket << std::endl; + cout << " Shortest: " << ShortestBucket << std::endl; + + // hashtable stats for the GrpHashTable NumBuckets = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); UsedBuckets = 0; UnusedBuckets = 0; @@ -423,12 +423,12 @@ static bool Stats(CommandLine &) ShortestBucket = std::min(ThisBucketSize, ShortestBucket); UsedBuckets += ThisBucketSize; } - cout << _("Grp Hashtable stats:") << endl; - cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; - cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; - cout << _("Used buckets: ") << UsedBuckets << endl; - cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; - cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; + cout << "Total GrpHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; + cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << " Used: " << UsedBuckets << std::endl; + cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Longest: " << LongestBucket << std::endl; + cout << " Shortest: " << ShortestBucket << std::endl; return true; } -- cgit v1.2.3 From 637c3b232223c17827a8842cb1c24655469329ba Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 18 Jun 2014 09:36:48 +0200 Subject: Provide ShowHashTableStats function --- cmdline/apt-cache.cc | 98 +++++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 55 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 072cf4ef7..35e9cc3a8 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -264,6 +264,44 @@ static bool DumpPackage(CommandLine &CmdL) return true; } /*}}}*/ +// ShowHashTableStats - Show stats about a hashtable /*{{{*/ +// --------------------------------------------------------------------- +/* */ +template +static void ShowHashTableStats(std::string Type, + T *StartP, + map_ptrloc *Hashtable, + unsigned long Size) +{ + // hashtable stats for the HashTable + long NumBuckets = Size; + long UsedBuckets = 0; + long UnusedBuckets = 0; + long LongestBucket = 0; + long ShortestBucket = NumBuckets; + for (unsigned int i=0; i < NumBuckets; ++i) + { + T *P = StartP + Hashtable[i]; + if(P == 0 || P == StartP) + { + UnusedBuckets++; + continue; + } + long ThisBucketSize = 0; + for (; P != StartP; P = StartP + P->Next) + ThisBucketSize++; + LongestBucket = std::max(ThisBucketSize, LongestBucket); + ShortestBucket = std::min(ThisBucketSize, ShortestBucket); + UsedBuckets += ThisBucketSize; + } + cout << "Total buckets " << Type << ": " << SizeToStr(NumBuckets) << std::endl; + cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << " Used: " << UsedBuckets << std::endl; + cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Longest: " << LongestBucket << std::endl; + cout << " Shortest: " << ShortestBucket << std::endl; +} + /*}}}*/ // Stats - Dump some nice statistics /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -374,61 +412,11 @@ static bool Stats(CommandLine &) Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; - // hashtable stats for the PkgHashTable - long NumBuckets = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); - long UsedBuckets = 0; - long UnusedBuckets = 0; - long LongestBucket = 0; - long ShortestBucket = NumBuckets; - for (unsigned int i=0; i < NumBuckets; ++i) - { - pkgCache::Package *Pkg = Cache->PkgP + Cache->HeaderP->PkgHashTable[i]; - if(Pkg == 0 || Pkg == Cache->PkgP) - { - UnusedBuckets++; - continue; - } - long ThisBucketSize = 0; - for (; Pkg != Cache->PkgP; Pkg = Cache->PkgP + Pkg->NextPackage) - ThisBucketSize++; - LongestBucket = std::max(ThisBucketSize, LongestBucket); - ShortestBucket = std::min(ThisBucketSize, ShortestBucket); - UsedBuckets += ThisBucketSize; - } - cout << "Total PkgHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; - cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; - cout << " Used: " << UsedBuckets << std::endl; - cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; - cout << " Longest: " << LongestBucket << std::endl; - cout << " Shortest: " << ShortestBucket << std::endl; - - // hashtable stats for the GrpHashTable - NumBuckets = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); - UsedBuckets = 0; - UnusedBuckets = 0; - LongestBucket = 0; - ShortestBucket = NumBuckets; - for (unsigned int i=0; i < NumBuckets; ++i) - { - pkgCache::Group *Grp = Cache->GrpP + Cache->HeaderP->GrpHashTable[i]; - if(Grp == 0 || Grp == Cache->GrpP) - { - UnusedBuckets++; - continue; - } - long ThisBucketSize = 0; - for (; Grp != Cache->GrpP; Grp = Cache->GrpP + Grp->Next) - ThisBucketSize++; - LongestBucket = std::max(ThisBucketSize, LongestBucket); - ShortestBucket = std::min(ThisBucketSize, ShortestBucket); - UsedBuckets += ThisBucketSize; - } - cout << "Total GrpHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; - cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; - cout << " Used: " << UsedBuckets << std::endl; - cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; - cout << " Longest: " << LongestBucket << std::endl; - cout << " Shortest: " << ShortestBucket << std::endl; + // hashtable stats + int HashTableSize = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->HeaderP->PkgHashTable, HashTableSize); + HashTableSize = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->HeaderP->GrpHashTable, HashTableSize); return true; } -- cgit v1.2.3 From 4ad8619bb1f0bf777d17c568bb7a6cf7f30aac34 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 12 Jun 2014 12:22:45 +0200 Subject: cleanup datatypes mix used in binary cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We had a wild mixture of (unsigned) int, long and long long here without much sense, so this commit adds a few typedefs to get some sense in the typesystem and ensures that a ID isn't sometimes computed as int, stored as long and compared with a long long… as this could potentially bite us later on as the size of the archive only increases over time. --- cmdline/apt-cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 2ed1bf5d4..620cb6c01 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1275,7 +1275,7 @@ static bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V) struct ExDescFile { pkgCache::DescFile *Df; - map_ptrloc ID; + map_id_t ID; }; // Search - Perform a search /*{{{*/ -- cgit v1.2.3 From ac2c559b33db7a2dbec3a95760ee2497baf7017a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 13 Jun 2014 09:10:11 +0200 Subject: correct 'apt-cache stats' to include more It still doesn't reflect the size the cache has on the disk compared to what is given as total size (90 vs 103 MB), but by counting all structs in we are at least a bit closer to the reality. Git-Dch: ignore --- cmdline/apt-cache.cc | 56 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 24 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 620cb6c01..88a323280 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -270,34 +270,36 @@ static bool DumpPackage(CommandLine &CmdL) template static void ShowHashTableStats(std::string Type, T *StartP, - map_ptrloc *Hashtable, + map_pointer_t *Hashtable, unsigned long Size) { // hashtable stats for the HashTable - long NumBuckets = Size; - long UsedBuckets = 0; - long UnusedBuckets = 0; - long LongestBucket = 0; - long ShortestBucket = NumBuckets; + unsigned long NumBuckets = Size; + unsigned long UsedBuckets = 0; + unsigned long UnusedBuckets = 0; + unsigned long LongestBucket = 0; + unsigned long ShortestBucket = NumBuckets; + unsigned long Entries = 0; for (unsigned int i=0; i < NumBuckets; ++i) { T *P = StartP + Hashtable[i]; if(P == 0 || P == StartP) { - UnusedBuckets++; + ++UnusedBuckets; continue; } - long ThisBucketSize = 0; + ++UsedBuckets; + unsigned long ThisBucketSize = 0; for (; P != StartP; P = StartP + P->Next) - ThisBucketSize++; + ++ThisBucketSize; + Entries += ThisBucketSize; LongestBucket = std::max(ThisBucketSize, LongestBucket); ShortestBucket = std::min(ThisBucketSize, ShortestBucket); - UsedBuckets += ThisBucketSize; } - cout << "Total buckets " << Type << ": " << SizeToStr(NumBuckets) << std::endl; - cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << "Total buckets in " << Type << ": " << NumBuckets << std::endl; + cout << " Unused: " << UnusedBuckets << std::endl; cout << " Used: " << UsedBuckets << std::endl; - cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Average entries: " << Entries/(double)NumBuckets << std::endl; cout << " Longest: " << LongestBucket << std::endl; cout << " Shortest: " << ShortestBucket << std::endl; } @@ -398,25 +400,31 @@ static bool Stats(CommandLine &) } } cout << _("Total dependency version space: ") << SizeToStr(DepVerSize) << endl; - + unsigned long Slack = 0; for (int I = 0; I != 7; I++) Slack += Cache->Head().Pools[I].ItemSize*Cache->Head().Pools[I].Count; cout << _("Total slack space: ") << SizeToStr(Slack) << endl; - + unsigned long Total = 0; - Total = Slack + Size + Cache->Head().DependsCount*Cache->Head().DependencySz + - Cache->Head().VersionCount*Cache->Head().VersionSz + - Cache->Head().PackageCount*Cache->Head().PackageSz + - Cache->Head().VerFileCount*Cache->Head().VerFileSz + - Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; +#define APT_CACHESIZE(X,Y) (Cache->Head().X * Cache->Head().Y) + Total = Slack + Size + + APT_CACHESIZE(GroupCount, GroupSz) + + APT_CACHESIZE(PackageCount, PackageSz) + + APT_CACHESIZE(VersionCount, VersionSz) + + APT_CACHESIZE(DescriptionCount, DescriptionSz) + + APT_CACHESIZE(DependsCount, DependencySz) + + APT_CACHESIZE(PackageFileCount, PackageFileSz) + + APT_CACHESIZE(VerFileCount, VerFileSz) + + APT_CACHESIZE(DescFileCount, DescFileSz) + + APT_CACHESIZE(ProvidesCount, ProvidesSz) + + (2 * Cache->Head().HashTableSize * sizeof(map_id_t)); cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; +#undef APT_CACHESIZE // hashtable stats - int HashTableSize = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); - ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->HeaderP->PkgHashTable, HashTableSize); - HashTableSize = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); - ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->HeaderP->GrpHashTable, HashTableSize); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTable(), Cache->Head().HashTableSize); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTable(), Cache->Head().HashTableSize); return true; } -- cgit v1.2.3 From 097248092ea4ff4543dfb17deb4e0d31dd697c71 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 2 Sep 2014 17:06:52 +0200 Subject: Use heap to allocate PatternMatch to avoid potential stack overflow When apt-cache search with many args (> 130) is given the allocation of PatternMatch on the stack may fail resulting in a segmentation fault. By using the heap the max size is much bigger and we also get a bad_alloc expection instead of a segfault (which we can catch *if* this ever becomes a pratical problem). No test for the crash as its not reproducable with the MALLOC_ settings in framework. Closes: 759612 --- cmdline/apt-cache.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1414617eb..ac0d48a36 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1278,8 +1278,8 @@ static bool Search(CommandLine &CmdL) ExDescFile *DFList = new ExDescFile[descCount]; memset(DFList,0,sizeof(*DFList) * descCount); - bool PatternMatch[descCount * NumPatterns]; - memset(PatternMatch,false,sizeof(PatternMatch)); + bool *PatternMatch = new bool[descCount * NumPatterns]; + memset(PatternMatch,false,sizeof(*PatternMatch) * descCount * NumPatterns); // Map versions that we want to write out onto the VerList array. for (pkgCache::GrpIterator G = Cache->GrpBegin(); G.end() == false; ++G) @@ -1389,6 +1389,7 @@ static bool Search(CommandLine &CmdL) } delete [] DFList; + delete [] PatternMatch; for (unsigned I = 0; I != NumPatterns; I++) regfree(&Patterns[I]); if (ferror(stdout)) -- cgit v1.2.3 From a3a91fd7bab34983a008854728baf26034d4033e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 20:28:54 +0200 Subject: count strings more accurately for stats So far, only the few strings stored in stringitems were counted, but many more strings are directly inserted into the cache. We account for this now by identifying all these different strings and measure their length. We are still not at the correct size of the cache in 'stats' this way, but we are now again a bit closer. Git-Dch: Ignore --- cmdline/apt-cache.cc | 53 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 14 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1447feb81..4030ce4e6 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -375,31 +375,56 @@ static bool Stats(CommandLine &) SizeToStr(Cache->Head().DescFileCount*Cache->Head().DescFileSz) << ')' << endl; cout << _("Total Provides mappings: ") << Cache->Head().ProvidesCount << " (" << SizeToStr(Cache->Head().ProvidesCount*Cache->Head().ProvidesSz) << ')' << endl; - - // String list stats - unsigned long Size = 0; - unsigned long Count = 0; - for (pkgCache::StringItem *I = Cache->StringItemP + Cache->Head().StringList; - I!= Cache->StringItemP; I = Cache->StringItemP + I->NextItem) - { - Count++; - Size += strlen(Cache->StrP + I->String) + 1; - } - cout << _("Total globbed strings: ") << Count << " (" << SizeToStr(Size) << ')' << endl; - unsigned long DepVerSize = 0; + // String list stats + std::set stritems; + for (pkgCache::GrpIterator G = Cache->GrpBegin(); G.end() == false; ++G) + stritems.insert(G->Name); for (pkgCache::PkgIterator P = Cache->PkgBegin(); P.end() == false; ++P) { + stritems.insert(P->Arch); for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V) { + if (V->VerStr != 0) + stritems.insert(V->VerStr); + if (V->Section != 0) + stritems.insert(V->Section); for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) - DepVerSize += strlen(D.TargetVer()) + 1; + stritems.insert(D->Version); + } + for (pkgCache::DescIterator D = V.DescriptionList(); D.end() == false; ++D) + { + stritems.insert(D->md5sum); + stritems.insert(D->language_code); } } + for (pkgCache::PrvIterator Prv = P.ProvidesList(); Prv.end() == false; ++Prv) + { + if (Prv->ProvideVersion != 0) + stritems.insert(Prv->ProvideVersion); + } } - cout << _("Total dependency version space: ") << SizeToStr(DepVerSize) << endl; + for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + { + stritems.insert(F->FileName); + stritems.insert(F->Archive); + stritems.insert(F->Codename); + stritems.insert(F->Component); + stritems.insert(F->Version); + stritems.insert(F->Origin); + stritems.insert(F->Label); + stritems.insert(F->Architecture); + stritems.insert(F->Site); + stritems.insert(F->IndexType); + } + unsigned long Size = 0; + for (std::set::const_iterator i = stritems.begin(); i != stritems.end(); ++i) + Size += strlen(Cache->StrP + *i) + 1; + + cout << _("Total globbed strings: ") << stritems.size() << " (" << SizeToStr(Size) << ')' << endl; + stritems.clear(); unsigned long Slack = 0; for (int I = 0; I != 7; I++) -- cgit v1.2.3 From fdba4d53d6b9b594531c34792798f4044a25157e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 3 Sep 2014 18:16:16 +0200 Subject: rework cachesets API to allow future extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The introduction of Fnmatch showed that each new selector would require multiple new virtual methods in the CacheSetHelper to work correctly, which isn't that great. We now flip to a single virtual method which handles all cases separated by an enum – as new enum values can be added without an ABI break. Great care was taken to make old code work with the new way of organisation, which means in return that you might be bombarded with deprecation warnings now if you don't adapt, but code should still compile and work as before as can be seen in apt itself with this commit. Git-Dch: Ignore --- cmdline/apt-cache.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 4030ce4e6..bd10a41ef 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -838,9 +838,9 @@ static bool XVcg(CommandLine &CmdL) // Load the list of packages from the command line into the show list APT::CacheSetHelper helper(true, GlobalError::NOTICE); - std::list mods; - mods.push_back(APT::PackageSet::Modifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); - mods.push_back(APT::PackageSet::Modifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); + std::list mods; + mods.push_back(APT::CacheSetHelper::PkgModifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); + mods.push_back(APT::CacheSetHelper::PkgModifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); std::map pkgsets = APT::PackageSet::GroupedFromCommandLine(CacheFile, CmdL.FileList + 1, mods, 0, helper); @@ -1050,9 +1050,9 @@ static bool Dotty(CommandLine &CmdL) // Load the list of packages from the command line into the show list APT::CacheSetHelper helper(true, GlobalError::NOTICE); - std::list mods; - mods.push_back(APT::PackageSet::Modifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); - mods.push_back(APT::PackageSet::Modifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); + std::list mods; + mods.push_back(APT::CacheSetHelper::PkgModifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); + mods.push_back(APT::CacheSetHelper::PkgModifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); std::map pkgsets = APT::PackageSet::GroupedFromCommandLine(CacheFile, CmdL.FileList + 1, mods, 0, helper); -- cgit v1.2.3 From e6f0c9bca4b052d20a2e48ce9715b89e187b671a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 3 Sep 2014 19:02:05 +0200 Subject: adapt to the new CacheSetHelper API Git-Dch: Ignore --- cmdline/apt-cache.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index bd10a41ef..5a5dde088 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -191,7 +191,7 @@ static bool UnMet(CommandLine &CmdL) { CacheSetHelperVirtuals helper(true, GlobalError::NOTICE); APT::VersionList verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, - APT::VersionList::CANDIDATE, helper); + APT::CacheSetHelper::CANDIDATE, helper); for (APT::VersionList::iterator V = verset.begin(); V != verset.end(); ++V) if (ShowUnMet(V, Important) == false) return false; @@ -656,7 +656,7 @@ static bool ShowDepends(CommandLine &CmdL, bool const RevDepends) return false; CacheSetHelperVirtuals helper(false); - APT::VersionList verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::VersionList::CANDIDATE, helper); + APT::VersionList verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true && helper.virtualPkgs.empty() == true) return _error->Error(_("No packages found")); std::vector Shown(Cache->Head().PackageCount); @@ -724,7 +724,7 @@ static bool ShowDepends(CommandLine &CmdL, bool const RevDepends) if (Recurse == true && Shown[Trg->ID] == false) { Shown[Trg->ID] = true; - verset.insert(APT::VersionSet::FromPackage(CacheFile, Trg, APT::VersionSet::CANDIDATE, helper)); + verset.insert(APT::VersionSet::FromPackage(CacheFile, Trg, APT::CacheSetHelper::CANDIDATE, helper)); } } @@ -743,7 +743,7 @@ static bool ShowDepends(CommandLine &CmdL, bool const RevDepends) if (Recurse == true && Shown[V.ParentPkg()->ID] == false) { Shown[V.ParentPkg()->ID] = true; - verset.insert(APT::VersionSet::FromPackage(CacheFile, V.ParentPkg(), APT::VersionSet::CANDIDATE, helper)); + verset.insert(APT::VersionSet::FromPackage(CacheFile, V.ParentPkg(), APT::CacheSetHelper::CANDIDATE, helper)); } } @@ -1506,8 +1506,8 @@ static bool ShowPackage(CommandLine &CmdL) { pkgCacheFile CacheFile; CacheSetHelperVirtuals helper(true, GlobalError::NOTICE); - APT::VersionList::Version const select = _config->FindB("APT::Cache::AllVersions", true) ? - APT::VersionList::ALL : APT::VersionList::CANDIDATE; + APT::CacheSetHelper::VerSelector const select = _config->FindB("APT::Cache::AllVersions", true) ? + APT::CacheSetHelper::ALL : APT::CacheSetHelper::CANDIDATE; APT::VersionList const verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, select, helper); for (APT::VersionList::const_iterator Ver = verset.begin(); Ver != verset.end(); ++Ver) if (DisplayRecord(CacheFile, Ver) == false) -- cgit v1.2.3 From a221efc331693f8905da870141756c892911c433 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 19:34:40 +0200 Subject: store source name and version in binary cache Accessing the package records to acquire this information is pretty costly, so that information wasn't used so far in many places. The most noticeable user by far is EDSP at the moment, but there are ideas to change that which this commit tries to enable. --- cmdline/apt-cache.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 5a5dde088..0f4f7e1ce 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -389,6 +389,8 @@ static bool Stats(CommandLine &) stritems.insert(V->VerStr); if (V->Section != 0) stritems.insert(V->Section); + stritems.insert(V->SourcePkgName); + stritems.insert(V->SourceVerStr); for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) -- cgit v1.2.3 From 765190e493645e13b5651625d87fd9c8db910a85 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 16:45:18 +0100 Subject: guard ABI changes for SourcePkg/Ver in pkgCache Git-Dch: Ignore --- cmdline/apt-cache.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 0f4f7e1ce..1bd75dfba 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -389,8 +389,10 @@ static bool Stats(CommandLine &) stritems.insert(V->VerStr); if (V->Section != 0) stritems.insert(V->Section); +#if APT_PKG_ABI >= 413 stritems.insert(V->SourcePkgName); stritems.insert(V->SourceVerStr); +#endif for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) -- cgit v1.2.3 From 32ab4bd05cb298f6bf1f9574f5b20570beaae429 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 19:18:21 +0100 Subject: guard pkg/grp hashtable creation changes The change itself is no problem ABI wise, but the remove of the old undynamic hashtables is, so we bring it back for older abis and happily use the now available free space to backport more recent additions like the dynamic hashtable itself. Git-Dch: Ignore --- cmdline/apt-cache.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1bd75dfba..9bac45029 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -267,11 +267,14 @@ static bool DumpPackage(CommandLine &CmdL) // ShowHashTableStats - Show stats about a hashtable /*{{{*/ // --------------------------------------------------------------------- /* */ +static map_pointer_t PackageNext(pkgCache::Package const * const P) { return P->NextPackage; } +static map_pointer_t GroupNext(pkgCache::Group const * const G) { return G->Next; } template static void ShowHashTableStats(std::string Type, T *StartP, map_pointer_t *Hashtable, - unsigned long Size) + unsigned long Size, + map_pointer_t(*Next)(T const * const)) { // hashtable stats for the HashTable unsigned long NumBuckets = Size; @@ -290,7 +293,7 @@ static void ShowHashTableStats(std::string Type, } ++UsedBuckets; unsigned long ThisBucketSize = 0; - for (; P != StartP; P = StartP + P->Next) + for (; P != StartP; P = StartP + Next(P)) ++ThisBucketSize; Entries += ThisBucketSize; LongestBucket = std::max(ThisBucketSize, LongestBucket); @@ -447,13 +450,13 @@ static bool Stats(CommandLine &) APT_CACHESIZE(VerFileCount, VerFileSz) + APT_CACHESIZE(DescFileCount, DescFileSz) + APT_CACHESIZE(ProvidesCount, ProvidesSz) + - (2 * Cache->Head().HashTableSize * sizeof(map_id_t)); + (2 * Cache->Head().GetHashTableSize() * sizeof(map_id_t)); cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; #undef APT_CACHESIZE // hashtable stats - ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTable(), Cache->Head().HashTableSize); - ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTable(), Cache->Head().HashTableSize); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTableP(), Cache->Head().GetHashTableSize(), PackageNext); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTableP(), Cache->Head().GetHashTableSize(), GroupNext); return true; } -- cgit v1.2.3 From ad7e0941b376d792911f240377094a2e78ca8756 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 18:14:46 +0100 Subject: streamline display of --help in all tools By convention, if I run a tool with --help or --version I expect it to exit successfully with the usage, while if I do call it wrong (like without any parameters) I expect the usage message shown with a non-zero exit. --- cmdline/apt-cache.cc | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 9bac45029..12ed4f719 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1896,23 +1896,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); -- cgit v1.2.3 From d9e518c6f7dc0ad464495b586d1b8e115d54d41a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 20:44:44 +0100 Subject: use the same code to detect quiet setting in all tools Git-Dch: Ignore --- cmdline/apt-cache.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 12ed4f719..342ad1858 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1899,9 +1899,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); + InitOutput(); if (_config->Exists("APT::Cache::Generate") == true) _config->Set("pkgCacheFile::Generate", _config->FindB("APT::Cache::Generate", true)); -- cgit v1.2.3 From 081c9d442a6d39fb9bc419fe3ce697cc791cb844 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 21:38:53 +0100 Subject: various small additional tests and testcases Usually they don't provide a lot in terms of what they test, but they help in covering many lines from strictly anecdotal commands (stats, moo) and error messages, so that stuff which really needs to be tested, but isn't is better visible in coverage reports. Git-Dch: Ignore --- cmdline/apt-cache.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 342ad1858..a5024c581 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -116,7 +116,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) continue; // Skip conflicts and replaces - if (End.IsNegative() == true) + if (End.IsNegative() == true || End->Type == pkgCache::Dep::Replaces) continue; // Verify the or group @@ -133,7 +133,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) break; } delete [] VList; - + if (Start == End) break; ++Start; -- cgit v1.2.3 From 249aec3b7397662a678ea0014f94392085477b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= Date: Tue, 10 Mar 2015 10:09:44 +0100 Subject: stop displaying time of build in online help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As part of the “reproducible builds” effort [1], we have noticed that apt could not be built reproducibly. One issue is that it uses the __DATE__ and __TIME__ macros of the C preprocessor to display the time of build in the online help. We believe this information not to be really useful to users as they can always look at the package data and metadata to figure it out. The attached patch simply removes this information. All non-documentation packages can then be built reproducibly with our current experimental framework. [David: changed the string slightly to be untranslateable as well] Closes: 774342 --- cmdline/apt-cache.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index a5024c581..e2cf7e8b7 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1824,9 +1824,8 @@ static bool GenCaches(CommandLine &) /* */ static bool ShowHelp(CommandLine &) { - ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, - COMMON_ARCH,__DATE__,__TIME__); - + ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH); + if (_config->FindB("version") == true) return true; -- cgit v1.2.3 From 88593886a42025d51d76051da5929b044e42efee Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 11 May 2015 15:08:08 +0200 Subject: rewrite all TFRewrite instances to use the new pkgTagSection::Write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While it is mostly busywork to rewrite all instances it actually fixes bugs as the data storage used by the new method is std::string rather than a char*, the later mostly created by c_str() from a std::string which the caller has to ensure keeps in scope – something apt-ftparchive actually didn't ensure and relied on copy-on-write behavior instead which c++11 forbids and hence the new default gcc abi doesn't use it. --- cmdline/apt-cache.cc | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'cmdline/apt-cache.cc') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index e2cf7e8b7..690b03bcc 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -580,6 +580,12 @@ static bool DumpAvail(CommandLine &) LocalitySort(VFList,Count,sizeof(*VFList)); + std::vector RW; + RW.push_back(pkgTagSection::Tag::Remove("Status")); + RW.push_back(pkgTagSection::Tag::Remove("Config-Version")); + FileFd stdoutfd; + stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false); + // Iterate over all the package files and write them out. char *Buffer = new char[Cache->HeaderP->MaxVerFileSize+10]; for (pkgCache::VerFile **J = VFList; *J != 0;) @@ -620,35 +626,32 @@ static bool DumpAvail(CommandLine &) if (PkgF.Read(Buffer,VF.Size + Jitter) == false) break; Buffer[VF.Size + Jitter] = '\n'; - + // See above.. if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource) { pkgTagSection Tags; - TFRewriteData RW[] = {{"Status", NULL, NULL},{"Config-Version", NULL, NULL},{NULL, NULL, NULL}}; - const char *Zero = 0; if (Tags.Scan(Buffer+Jitter,VF.Size+1) == false || - TFRewrite(stdout,Tags,&Zero,RW) == false) + Tags.Write(stdoutfd, NULL, RW) == false || + stdoutfd.Write("\n", 1) == false) { _error->Error("Internal Error, Unable to parse a package record"); break; } - fputc('\n',stdout); } else { - if (fwrite(Buffer+Jitter,VF.Size+1,1,stdout) != 1) + if (stdoutfd.Write(Buffer + Jitter, VF.Size + 1) == false) break; } - + Pos = VF.Offset + VF.Size; } - fflush(stdout); if (_error->PendingError() == true) break; } - + delete [] Buffer; delete [] VFList; return !_error->PendingError(); -- cgit v1.2.3