From 9005f08e61abe99d2fa229a32fc3148eba29bf5f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 8 Jan 2013 16:44:38 +0100 Subject: * [ABI BREAK] apt-pkg/pkgcache.h: - adjust pkgCache::State::VerPriority enum, to match reality --- apt-pkg/pkgcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 1a7013551..565ee657c 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -136,7 +136,7 @@ class pkgCache /*{{{*/ /** \brief priority of a package version Zero is used for unparsable or absent Priority fields. */ - enum VerPriority {Important=1,Required=2,Standard=3,Optional=4,Extra=5}; + enum VerPriority {Required=1,Important=2,Standard=3,Optional=4,Extra=5}; enum PkgSelectedState {Unknown=0,Install=1,Hold=2,DeInstall=3,Purge=4}; enum PkgInstState {Ok=0,ReInstReq=1,HoldInst=2,HoldReInstReq=3}; enum PkgCurrentState {NotInstalled=0,UnPacked=1,HalfConfigured=2, -- cgit v1.2.3 From 7a223b933cab0447438ca2e964576da39078eaf4 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 May 2014 12:50:00 +0200 Subject: invalid cache if architecture set doesn't match The cache heavily depends on the architecture(s) it is build for, especially if you move from single- to multiarch. Adding a new architecture to dpkg therefore has to be detected and must invalidate the cache so that we don't operate on incorrect data. The incorrect data will prevent us from doing otherwise sensible actions (it doesn't allow bad things to happen) and the recovery is simple and automatic in most cases, so this hides pretty well and is also not as serious as it might sound at first. Closes: 745036 --- apt-pkg/pkgcache.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 151de7d25..22dc6218c 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -286,8 +286,10 @@ struct pkgCache::Header map_ptrloc StringList; /** \brief String representing the version system used */ map_ptrloc VerSysName; - /** \brief Architecture(s) the cache was built against */ + /** \brief native architecture the cache was built against */ map_ptrloc Architecture; + /** \brief all architectures the cache was built against */ + map_ptrloc Architectures; /** \brief The maximum size of a raw entry from the original Package file */ unsigned long MaxVerFileSize; /** \brief The maximum size of a raw entry from the original Translation file */ -- cgit v1.2.3 From 43f8819b7fbfd24c5013bbe3183cd85e10e77af3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 29 May 2014 10:29:21 +0200 Subject: increase Pkg/Grp hash table size from 2k to 64k --- apt-pkg/pkgcache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 5e8a9630a..4dd1b33d4 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -314,8 +314,8 @@ struct pkgCache::Header these packages are stored as a sequence in the list. Beware: The Hashmethod assumes that the hash table sizes are equal */ - map_ptrloc PkgHashTable[2*1048]; - map_ptrloc GrpHashTable[2*1048]; + map_ptrloc PkgHashTable[64*1048]; + map_ptrloc GrpHashTable[64*1048]; /** \brief Size of the complete cache file */ unsigned long CacheFileSize; -- cgit v1.2.3 From b686f453b14e11a69ecbd368509f7eaf1596c6e0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 18 Jun 2014 09:35:53 +0200 Subject: [API-Break] rename pkgCache::Package::NextPackage to pkgCache::Package::Next This is a internal struct not a external interface so the actual breakage should be small. --- apt-pkg/pkgcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 4dd1b33d4..43d379ddf 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -388,7 +388,7 @@ struct pkgCache::Package // Linked list /** \brief Link to the next package in the same bucket */ - map_ptrloc NextPackage; // Package + map_ptrloc Next; // Package /** \brief List of all dependencies on this package */ map_ptrloc RevDepends; // Dependency /** \brief List of all "packages" this package provide */ -- cgit v1.2.3 From e8a7b0b28ca01cd8c2c1bee0d83e5997b40de689 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jun 2014 20:42:16 +0200 Subject: increase hashtable size for packages/groups by factor 5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It also makes the size configureable, so it can be adapted in the future without the need for an abi break - and even by users… The increase was long overdue as it gives a >10% decrease in runtime of e.g. 'apt-get check -s'. Some (useless) benchmark with 69933 groups and 187796 packages without a pre-built cache: time apt-get check -so APT::Cache-HashTableSize=1 → 20m time apt-get check -so APT::Cache-HashTableSize=1000 → 6,41s time apt-get check -so APT::Cache-HashTableSize=2000 → 5,64s (old) time apt-get check -so APT::Cache-HashTableSize=3000 → 5,30s time apt-get check -so APT::Cache-HashTableSize=5000 → 5,08s time apt-get check -so APT::Cache-HashTableSize=6000 → 5,05s time apt-get check -so APT::Cache-HashTableSize=7000 → 5,02s time apt-get check -so APT::Cache-HashTableSize=8000 → 5,00s time apt-get check -so APT::Cache-HashTableSize=9000 → 4,98s time apt-get check -so APT::Cache-HashTableSize=10000 → 4,96s (new) time apt-get check -so APT::Cache-HashTableSize=15000 → 4,90s time apt-get check -so APT::Cache-HashTableSize=20000 → 4,86s time apt-get check -so APT::Cache-HashTableSize=30000 → 4,77s time apt-get check -so APT::Cache-HashTableSize=40000 → 4,74s time apt-get check -so APT::Cache-HashTableSize=50000 → 4,73s time apt-get check -so APT::Cache-HashTableSize=60000 → 4,71s The gap increases further for operations which have more package lookups. Factor 5 was chosen as higher values do not provide any really significant timing advantage anymore compared to the memory increase in my testing and there is always the possibility to increase it now if that changes. (also most users will not have 3 releases and 4 architectures in the cache, so theirs will be much smaller and faster). --- apt-pkg/pkgcache.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 55f0187f9..b6b2894cf 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -304,20 +304,20 @@ struct pkgCache::Header stores this information so future additions can make use of any unused pool blocks. */ DynamicMMap::Pool Pools[9]; - + /** \brief hash tables providing rapid group/package name lookup - Each group/package name is inserted into the hash table using pkgCache::Hash(const &string) + Each group/package name is inserted into a hash table using pkgCache::Hash(const &string) By iterating over each entry in the hash table it is possible to iterate over the entire list of packages. Hash Collisions are handled with a singly linked list of packages based at the hash item. The linked list contains only packages that match the hashing function. In the PkgHashTable is it possible that multiple packages have the same name - these packages are stored as a sequence in the list. - - Beware: The Hashmethod assumes that the hash table sizes are equal */ - map_ptrloc PkgHashTable[64*1048]; - map_ptrloc GrpHashTable[64*1048]; + The size of both tables is the same. */ + unsigned int HashTableSize; + map_ptrloc * PkgHashTable() const { return (map_ptrloc*) (this + 1); } + map_ptrloc * GrpHashTable() const { return PkgHashTable() + HashTableSize; } /** \brief Size of the complete cache file */ unsigned long CacheFileSize; -- cgit v1.2.3 From 4ad8619bb1f0bf777d17c568bb7a6cf7f30aac34 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 12 Jun 2014 12:22:45 +0200 Subject: cleanup datatypes mix used in binary cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We had a wild mixture of (unsigned) int, long and long long here without much sense, so this commit adds a few typedefs to get some sense in the typesystem and ensures that a ID isn't sometimes computed as int, stored as long and compared with a long long… as this could potentially bite us later on as the size of the archive only increases over time. --- apt-pkg/pkgcache.h | 190 ++++++++++++++++++++++++++++------------------------- 1 file changed, 101 insertions(+), 89 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index b6b2894cf..f57c31b98 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -79,11 +79,23 @@ #include #include +#include #ifndef APT_8_CLEANER_HEADERS using std::string; #endif +// storing file sizes of indexes, which are way below 4 GB for now +typedef uint32_t map_filesize_t; +// each package/group/dependency gets an id +typedef uint32_t map_id_t; +// some files get an id, too, but in far less absolute numbers +typedef uint16_t map_fileid_t; +// relative pointer from cache start +typedef uint32_t map_pointer_t; +// same as the previous, but documented to be to a string item +typedef map_pointer_t map_stringitem_t; + class pkgVersioningSystem; class pkgCache /*{{{*/ { @@ -158,8 +170,8 @@ class pkgCache /*{{{*/ std::string CacheFile; MMap ⤅ - unsigned long sHash(const std::string &S) const APT_PURE; - unsigned long sHash(const char *S) const APT_PURE; + map_id_t sHash(const std::string &S) const APT_PURE; + map_id_t sHash(const char *S) const APT_PURE; public: @@ -183,8 +195,8 @@ class pkgCache /*{{{*/ inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();} // String hashing function (512 range) - inline unsigned long Hash(const std::string &S) const {return sHash(S);} - inline unsigned long Hash(const char *S) const {return sHash(S);} + inline map_id_t Hash(const std::string &S) const {return sHash(S);} + inline map_id_t Hash(const char *S) const {return sHash(S);} // Useful transformation things const char *Priority(unsigned char Priority); @@ -263,37 +275,37 @@ struct pkgCache::Header These indicate the number of each structure contained in the cache. PackageCount is especially useful for generating user state structures. See Package::Id for more info. */ - unsigned long GroupCount; - unsigned long PackageCount; - unsigned long VersionCount; - unsigned long DescriptionCount; - unsigned long DependsCount; - unsigned long PackageFileCount; - unsigned long VerFileCount; - unsigned long DescFileCount; - unsigned long ProvidesCount; + map_id_t GroupCount; + map_id_t PackageCount; + map_id_t VersionCount; + map_id_t DescriptionCount; + map_id_t DependsCount; + map_fileid_t PackageFileCount; + map_fileid_t VerFileCount; + map_fileid_t DescFileCount; + map_id_t ProvidesCount; /** \brief index of the first PackageFile structure The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ - map_ptrloc FileList; + map_pointer_t FileList; /** \brief index of the first StringItem structure The cache contains a list of all the unique strings (StringItems). The parser reads this list into memory so it can match strings against it.*/ - map_ptrloc StringList; + map_pointer_t StringList; /** \brief String representing the version system used */ - map_ptrloc VerSysName; + map_pointer_t VerSysName; /** \brief native architecture the cache was built against */ - map_ptrloc Architecture; + map_pointer_t Architecture; /** \brief all architectures the cache was built against */ - map_ptrloc Architectures; + map_pointer_t Architectures; /** \brief The maximum size of a raw entry from the original Package file */ - unsigned long MaxVerFileSize; + map_filesize_t MaxVerFileSize; /** \brief The maximum size of a raw entry from the original Translation file */ - unsigned long MaxDescFileSize; + map_filesize_t MaxDescFileSize; /** \brief The Pool structures manage the allocation pools that the generator uses @@ -316,11 +328,11 @@ struct pkgCache::Header these packages are stored as a sequence in the list. The size of both tables is the same. */ unsigned int HashTableSize; - map_ptrloc * PkgHashTable() const { return (map_ptrloc*) (this + 1); } - map_ptrloc * GrpHashTable() const { return PkgHashTable() + HashTableSize; } + map_pointer_t * PkgHashTable() const { return (map_pointer_t*) (this + 1); } + map_pointer_t * GrpHashTable() const { return PkgHashTable() + HashTableSize; } /** \brief Size of the complete cache file */ - unsigned long CacheFileSize; + unsigned long long CacheFileSize; bool CheckSizes(Header &Against) const APT_PURE; Header(); @@ -336,17 +348,17 @@ struct pkgCache::Header struct pkgCache::Group { /** \brief Name of the group */ - map_ptrloc Name; // StringItem + map_stringitem_t Name; // Linked List /** \brief Link to the first package which belongs to the group */ - map_ptrloc FirstPackage; // Package + map_pointer_t FirstPackage; // Package /** \brief Link to the last package which belongs to the group */ - map_ptrloc LastPackage; // Package + map_pointer_t LastPackage; // Package /** \brief Link to the next Group */ - map_ptrloc Next; // Group + map_pointer_t Next; // Group /** \brief unique sequel ID */ - unsigned int ID; + map_id_t ID; }; /*}}}*/ @@ -365,9 +377,9 @@ struct pkgCache::Group struct pkgCache::Package { /** \brief Name of the package */ - map_ptrloc Name; // StringItem + map_stringitem_t Name; /** \brief Architecture of the package */ - map_ptrloc Arch; // StringItem + map_stringitem_t Arch; /** \brief Base of a singly linked list of versions Each structure represents a unique version of the package. @@ -377,24 +389,24 @@ struct pkgCache::Package versions of a package can be cleanly handled by the system. Furthermore, this linked list is guaranteed to be sorted from Highest version to lowest version with no duplicate entries. */ - map_ptrloc VersionList; // Version + map_pointer_t VersionList; // Version /** \brief index to the installed version */ - map_ptrloc CurrentVer; // Version + map_pointer_t CurrentVer; // Version /** \brief indicates the deduced section Should be the index to the string "Unknown" or to the section of the last parsed item. */ - map_ptrloc Section; // StringItem + map_stringitem_t Section; /** \brief index of the group this package belongs to */ - map_ptrloc Group; // Group the Package belongs to + map_pointer_t Group; // Group the Package belongs to // Linked list /** \brief Link to the next package in the same bucket */ - map_ptrloc Next; // Package + map_pointer_t Next; // Package /** \brief List of all dependencies on this package */ - map_ptrloc RevDepends; // Dependency + map_pointer_t RevDepends; // Dependency /** \brief List of all "packages" this package provide */ - map_ptrloc ProvidesList; // Provides + map_pointer_t ProvidesList; // Provides // Install/Remove/Purge etc /** \brief state that the user wishes the package to be in */ @@ -414,7 +426,7 @@ struct pkgCache::Package This allows clients to create an array of size PackageCount and use it to store state information for the package map. For instance the status file emitter uses this to track which packages have been emitted already. */ - unsigned int ID; + map_id_t ID; /** \brief some useful indicators of the package's state */ unsigned long Flags; }; @@ -428,30 +440,30 @@ struct pkgCache::Package struct pkgCache::PackageFile { /** \brief physical disk file that this PackageFile represents */ - map_ptrloc FileName; // StringItem + map_pointer_t FileName; // StringItem /** \brief the release information Please see the files document for a description of what the release information means. */ - map_ptrloc Archive; // StringItem - map_ptrloc Codename; // StringItem - map_ptrloc Component; // StringItem - map_ptrloc Version; // StringItem - map_ptrloc Origin; // StringItem - map_ptrloc Label; // StringItem - map_ptrloc Architecture; // StringItem + map_stringitem_t Archive; + map_stringitem_t Codename; + map_stringitem_t Component; + map_stringitem_t Version; + map_stringitem_t Origin; + map_stringitem_t Label; + map_stringitem_t Architecture; /** \brief The site the index file was fetched from */ - map_ptrloc Site; // StringItem + map_stringitem_t Site; /** \brief indicates what sort of index file this is @TODO enumerate at least the possible indexes */ - map_ptrloc IndexType; // StringItem + map_stringitem_t IndexType; /** \brief Size of the file Used together with the modification time as a simple check to ensure that the Packages file has not been altered since Cache generation. */ - unsigned long Size; + map_filesize_t Size; /** \brief Modification time for the file */ time_t mtime; @@ -460,9 +472,9 @@ struct pkgCache::PackageFile // Linked list /** \brief Link to the next PackageFile in the Cache */ - map_ptrloc NextFile; // PackageFile + map_pointer_t NextFile; // PackageFile /** \brief unique sequel ID */ - unsigned int ID; + map_fileid_t ID; }; /*}}}*/ // VerFile structure /*{{{*/ @@ -473,13 +485,13 @@ struct pkgCache::PackageFile struct pkgCache::VerFile { /** \brief index of the package file that this version was found in */ - map_ptrloc File; // PackageFile + map_pointer_t File; // PackageFile /** \brief next step in the linked list */ - map_ptrloc NextFile; // PkgVerFile + map_pointer_t NextFile; // PkgVerFile /** \brief position in the package file */ - map_ptrloc Offset; // File offset + map_filesize_t Offset; // File offset /** @TODO document pkgCache::VerFile::Size */ - unsigned long Size; + map_filesize_t Size; }; /*}}}*/ // DescFile structure /*{{{*/ @@ -487,13 +499,13 @@ struct pkgCache::VerFile struct pkgCache::DescFile { /** \brief index of the file that this description was found in */ - map_ptrloc File; // PackageFile + map_pointer_t File; // PackageFile /** \brief next step in the linked list */ - map_ptrloc NextFile; // PkgVerFile + map_pointer_t NextFile; // PkgVerFile /** \brief position in the file */ - map_ptrloc Offset; // File offset + map_filesize_t Offset; // File offset /** @TODO document pkgCache::DescFile::Size */ - unsigned long Size; + map_filesize_t Size; }; /*}}}*/ // Version structure /*{{{*/ @@ -505,9 +517,9 @@ struct pkgCache::DescFile struct pkgCache::Version { /** \brief complete version string */ - map_ptrloc VerStr; // StringItem + map_stringitem_t VerStr; /** \brief section this version is filled in */ - map_ptrloc Section; // StringItem + map_stringitem_t Section; /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ @@ -529,33 +541,33 @@ struct pkgCache::Version applies to. If FileList is 0 then this is a blank version. The structure should also have a 0 in all other fields excluding pkgCache::Version::VerStr and Possibly pkgCache::Version::NextVer. */ - map_ptrloc FileList; // VerFile + map_pointer_t FileList; // VerFile /** \brief next (lower or equal) version in the linked list */ - map_ptrloc NextVer; // Version + map_pointer_t NextVer; // Version /** \brief next description in the linked list */ - map_ptrloc DescriptionList; // Description + map_pointer_t DescriptionList; // Description /** \brief base of the dependency list */ - map_ptrloc DependsList; // Dependency + map_pointer_t DependsList; // Dependency /** \brief links to the owning package This allows reverse dependencies to determine the package */ - map_ptrloc ParentPkg; // Package + map_pointer_t ParentPkg; // Package /** \brief list of pkgCache::Provides */ - map_ptrloc ProvidesList; // Provides + map_pointer_t ProvidesList; // Provides /** \brief archive size for this version For Debian this is the size of the .deb file. */ - unsigned long long Size; // These are the .deb size + uint64_t Size; // These are the .deb size /** \brief uncompressed size for this version */ - unsigned long long InstalledSize; + uint64_t InstalledSize; /** \brief characteristic value representing this version No two packages in existence should have the same VerStr and Hash with different contents. */ unsigned short Hash; /** \brief unique sequel ID */ - unsigned int ID; + map_id_t ID; /** \brief parsed priority value */ unsigned char Priority; }; @@ -568,22 +580,22 @@ struct pkgCache::Description If the value has a 0 length then this is read using the Package file else the Translation-CODE file is used. */ - map_ptrloc language_code; // StringItem + map_stringitem_t language_code; /** \brief MD5sum of the original description Used to map Translations of a description to a version and to check that the Translation is up-to-date. */ - map_ptrloc md5sum; // StringItem + map_stringitem_t md5sum; /** @TODO document pkgCache::Description::FileList */ - map_ptrloc FileList; // DescFile + map_pointer_t FileList; // DescFile /** \brief next translation for this description */ - map_ptrloc NextDesc; // Description + map_pointer_t NextDesc; // Description /** \brief the text is a description of this package */ - map_ptrloc ParentPkg; // Package + map_pointer_t ParentPkg; // Package /** \brief unique sequel ID */ - unsigned int ID; + map_id_t ID; }; /*}}}*/ // Dependency structure /*{{{*/ @@ -596,21 +608,21 @@ struct pkgCache::Description struct pkgCache::Dependency { /** \brief string of the version the dependency is applied against */ - map_ptrloc Version; // StringItem + map_stringitem_t Version; // StringItem /** \brief index of the package this depends applies to The generator will - if the package does not already exist - create a blank (no version records) package. */ - map_ptrloc Package; // Package + map_pointer_t Package; // Package /** \brief next dependency of this version */ - map_ptrloc NextDepends; // Dependency + map_pointer_t NextDepends; // Dependency /** \brief next reverse dependency of this package */ - map_ptrloc NextRevDepends; // Dependency + map_pointer_t NextRevDepends; // Dependency /** \brief version of the package which has the reverse depends */ - map_ptrloc ParentVer; // Version + map_pointer_t ParentVer; // Version /** \brief unique sequel ID */ - map_ptrloc ID; + map_id_t ID; /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ unsigned char Type; /** \brief comparison operator specified on the depends line @@ -631,19 +643,19 @@ struct pkgCache::Dependency struct pkgCache::Provides { /** \brief index of the package providing this */ - map_ptrloc ParentPkg; // Package + map_pointer_t ParentPkg; // Package /** \brief index of the version this provide line applies to */ - map_ptrloc Version; // Version + map_pointer_t Version; // Version /** \brief version in the provides line (if any) This version allows dependencies to depend on specific versions of a Provides, as well as allowing Provides to override existing packages. This is experimental. Note that Debian doesn't allow versioned provides */ - map_ptrloc ProvideVersion; // StringItem + map_stringitem_t ProvideVersion; /** \brief next provides (based of package) */ - map_ptrloc NextProvides; // Provides + map_pointer_t NextProvides; // Provides /** \brief next provides (based of version) */ - map_ptrloc NextPkgProv; // Provides + map_pointer_t NextPkgProv; // Provides }; /*}}}*/ // StringItem structure /*{{{*/ @@ -658,9 +670,9 @@ struct pkgCache::Provides struct pkgCache::StringItem { /** \brief string this refers to */ - map_ptrloc String; // StringItem + map_stringitem_t String; /** \brief Next link in the chain */ - map_ptrloc NextItem; // StringItem + map_stringitem_t NextItem; }; /*}}}*/ -- cgit v1.2.3 From 7a66977486804d46d5860f568cbd80f54f0c42d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 13 Jun 2014 08:35:32 +0200 Subject: remove the Section member from package struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A version belongs to a section and has hence a section member of its own. A package on the other hand can have multiple versions from different sections. This was "solved" by using the section which was parsed first as order of sources.list defines, but that is obviously a horribly unpredictable thing. We therefore directly remove this struct member to free some space and mark the access method as deprecated, which is told to return the section of the 'newest' known version, which is at least predictable, but possible not what it returned before – but nobody knows. Users are way better of with the Section() as returned by the version they are dealing with. It is likely the same for all versions of a package, but in the few cases it isn't, it is important (like packages moving from main/* to contrib/* or into oldlibs …). --- apt-pkg/pkgcache.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index f57c31b98..0ce2a2878 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -392,11 +392,6 @@ struct pkgCache::Package map_pointer_t VersionList; // Version /** \brief index to the installed version */ map_pointer_t CurrentVer; // Version - /** \brief indicates the deduced section - - Should be the index to the string "Unknown" or to the section - of the last parsed item. */ - map_stringitem_t Section; /** \brief index of the group this package belongs to */ map_pointer_t Group; // Group the Package belongs to -- cgit v1.2.3 From fe86debbae914b51d2799e7b24abfd1f944f8caf Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 19 Jun 2014 11:00:02 +0200 Subject: deprecate Pkg->Name in favor of Grp->Name They both store the same information, so this field just takes up space in the Package struct for no good reason. We mark it "just" as deprecated instead of instantly removing it though as it isn't misleading like Section was and is potentially used in the wild more often. --- apt-pkg/pkgcache.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 0ce2a2878..8179e05aa 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -376,8 +376,11 @@ struct pkgCache::Group */ struct pkgCache::Package { - /** \brief Name of the package */ - map_stringitem_t Name; + /** \brief Name of the package + * Note that the access method Name() will remain. It is just this data member + * deprecated as this information is already stored and available via the + * associated Group – so it is wasting precious binary cache space */ + APT_DEPRECATED map_stringitem_t Name; /** \brief Architecture of the package */ map_stringitem_t Arch; /** \brief Base of a singly linked list of versions -- cgit v1.2.3 From 78a5476f3177a2a74ae51a1878c26ca322a25003 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 21:33:56 +0200 Subject: drop stored StringItems in favor of in-memory mappings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strings like Section names or architectures are needed vary often. Instead of writing them each time we need them, we deploy sharing for these special strings. Until now, this was done with a linked list of strings in which we would search, which was stored in the cache. It turns out we can do this just as well in memory as well with a bunch of std::map's. In memory means here that it isn't available anymore if we have a partly invalid cache, but that isn't much of a problem in practice as the status file is compared to the other files we parse very small and includes mostly duplicates, so the space we would gain by storing is more or less equal to the size of the stored linked list… --- apt-pkg/pkgcache.h | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 8179e05aa..4284a318c 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -109,7 +109,6 @@ class pkgCache /*{{{*/ struct Description; struct Provides; struct Dependency; - struct StringItem; struct VerFile; struct DescFile; @@ -186,7 +185,6 @@ class pkgCache /*{{{*/ Description *DescP; Provides *ProvideP; Dependency *DepP; - StringItem *StringItemP; char *StrP; virtual bool ReMap(bool const &Errorchecks = true); @@ -290,12 +288,6 @@ struct pkgCache::Header The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ map_pointer_t FileList; - /** \brief index of the first StringItem structure - - The cache contains a list of all the unique strings (StringItems). - The parser reads this list into memory so it can match strings - against it.*/ - map_pointer_t StringList; /** \brief String representing the version system used */ map_pointer_t VerSysName; /** \brief native architecture the cache was built against */ @@ -438,7 +430,7 @@ struct pkgCache::Package struct pkgCache::PackageFile { /** \brief physical disk file that this PackageFile represents */ - map_pointer_t FileName; // StringItem + map_stringitem_t FileName; /** \brief the release information Please see the files document for a description of what the @@ -606,7 +598,7 @@ struct pkgCache::Description struct pkgCache::Dependency { /** \brief string of the version the dependency is applied against */ - map_stringitem_t Version; // StringItem + map_stringitem_t Version; /** \brief index of the package this depends applies to The generator will - if the package does not already exist - @@ -656,24 +648,6 @@ struct pkgCache::Provides map_pointer_t NextPkgProv; // Provides }; /*}}}*/ -// StringItem structure /*{{{*/ -/** \brief used for generating single instances of strings - - Some things like Section Name are are useful to have as unique tags. - It is part of a linked list based at pkgCache::Header::StringList - - All strings are simply inlined any place in the file that is natural - for the writer. The client should make no assumptions about the positioning - of strings. All StringItems should be null-terminated. */ -struct pkgCache::StringItem -{ - /** \brief string this refers to */ - map_stringitem_t String; - /** \brief Next link in the chain */ - map_stringitem_t NextItem; -}; - /*}}}*/ - inline char const * pkgCache::NativeArch() { return StrP + HeaderP->Architecture; } -- cgit v1.2.3 From a221efc331693f8905da870141756c892911c433 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 19:34:40 +0200 Subject: store source name and version in binary cache Accessing the package records to acquire this information is pretty costly, so that information wasn't used so far in many places. The most noticeable user by far is EDSP at the moment, but there are ideas to change that which this commit tries to enable. --- apt-pkg/pkgcache.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 4284a318c..6a89eabd7 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -510,6 +510,12 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; + /** \brief source package name this version comes from + Always contains the name, even if it is the same as the binary name */ + map_stringitem_t SourcePkgName; + /** \brief source version this version comes from + Always contains the version string, even if it is the same as the binary version */ + map_stringitem_t SourceVerStr; /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ -- cgit v1.2.3 From 3809194b662f48733916e6248cd0c141f281313d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 29 Sep 2014 15:41:12 +0200 Subject: mark private methods as hidden We are the only possible users of private methods, so we are also the only users who can potentially export them via using them in inline methods. The point is: We don't need these symbols exported if we don't do this, so marking them as hidden removes some methods from the API without breaking anything as nobody could have used them. Git-Dch: Ignore --- apt-pkg/pkgcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 6a89eabd7..4f8568205 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -228,7 +228,7 @@ class pkgCache /*{{{*/ private: bool MultiArchEnabled; - PkgIterator SingleArchFindPkg(const std::string &Name); + APT_HIDDEN PkgIterator SingleArchFindPkg(const std::string &Name); }; /*}}}*/ // Header structure /*{{{*/ -- cgit v1.2.3 From 765190e493645e13b5651625d87fd9c8db910a85 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 16:45:18 +0100 Subject: guard ABI changes for SourcePkg/Ver in pkgCache Git-Dch: Ignore --- apt-pkg/pkgcache.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 4f8568205..a7b7fa539 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -510,12 +510,14 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; +#if APT_PKG_ABI >= 413 /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ map_stringitem_t SourcePkgName; /** \brief source version this version comes from Always contains the version string, even if it is the same as the binary version */ map_stringitem_t SourceVerStr; +#endif /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ -- cgit v1.2.3 From 32ab4bd05cb298f6bf1f9574f5b20570beaae429 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 19:18:21 +0100 Subject: guard pkg/grp hashtable creation changes The change itself is no problem ABI wise, but the remove of the old undynamic hashtables is, so we bring it back for older abis and happily use the now available free space to backport more recent additions like the dynamic hashtable itself. Git-Dch: Ignore --- apt-pkg/pkgcache.h | 93 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 14 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index a7b7fa539..2ba23c5c0 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -85,16 +85,45 @@ using std::string; #endif +#if APT_PKG_ABI >= 413 // storing file sizes of indexes, which are way below 4 GB for now typedef uint32_t map_filesize_t; +typedef map_filesize_t should_be_map_filesize_t; +#else +typedef unsigned long map_filesize_t; +typedef unsigned int should_be_map_filesize_t; +#endif +#if APT_PKG_ABI >= 413 // each package/group/dependency gets an id typedef uint32_t map_id_t; +typedef map_id_t should_be_map_id_t; +#else +typedef unsigned long map_id_t; +typedef unsigned int should_be_map_id_t; +#endif +#if APT_PKG_ABI >= 413 // some files get an id, too, but in far less absolute numbers typedef uint16_t map_fileid_t; +typedef map_fileid_t should_be_map_fileid_t; +#else +typedef unsigned long map_fileid_t; +typedef unsigned int should_be_map_fileid_t; +#endif +#if APT_PKG_ABI >= 413 // relative pointer from cache start typedef uint32_t map_pointer_t; +#else +typedef unsigned int map_pointer_t; +#endif // same as the previous, but documented to be to a string item typedef map_pointer_t map_stringitem_t; +#if APT_PKG_ABI >= 413 +typedef uint64_t should_be_uint64_t; +typedef uint64_t should_be_uint64_small_t; +#else +typedef unsigned long long should_be_uint64_t; +typedef unsigned long should_be_uint64_small_t; +#endif class pkgVersioningSystem; class pkgCache /*{{{*/ @@ -109,6 +138,7 @@ class pkgCache /*{{{*/ struct Description; struct Provides; struct Dependency; + struct StringItem; struct VerFile; struct DescFile; @@ -185,6 +215,7 @@ class pkgCache /*{{{*/ Description *DescP; Provides *ProvideP; Dependency *DepP; + APT_DEPRECATED StringItem *StringItemP; char *StrP; virtual bool ReMap(bool const &Errorchecks = true); @@ -288,12 +319,17 @@ struct pkgCache::Header The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ map_pointer_t FileList; +#if APT_PKG_ABI < 413 + APT_DEPRECATED map_pointer_t StringList; +#endif /** \brief String representing the version system used */ map_pointer_t VerSysName; /** \brief native architecture the cache was built against */ map_pointer_t Architecture; +#if APT_PKG_ABI >= 413 /** \brief all architectures the cache was built against */ map_pointer_t Architectures; +#endif /** \brief The maximum size of a raw entry from the original Package file */ map_filesize_t MaxVerFileSize; /** \brief The maximum size of a raw entry from the original Translation file */ @@ -319,12 +355,26 @@ struct pkgCache::Header In the PkgHashTable is it possible that multiple packages have the same name - these packages are stored as a sequence in the list. The size of both tables is the same. */ +#if APT_PKG_ABI >= 413 unsigned int HashTableSize; - map_pointer_t * PkgHashTable() const { return (map_pointer_t*) (this + 1); } - map_pointer_t * GrpHashTable() const { return PkgHashTable() + HashTableSize; } + unsigned int GetHashTableSize() const { return HashTableSize; } + void SetHashTableSize(unsigned int const sz) { HashTableSize = sz; } + map_pointer_t GetArchitectures() const { return Architectures; } + void SetArchitectures(map_pointer_t const idx) { Architectures = idx; } +#else + // BEWARE: these tables are pretty much empty and just here for abi compat + map_ptrloc PkgHashTable[2*1048]; + map_ptrloc GrpHashTable[2*1048]; + unsigned int GetHashTableSize() const { return PkgHashTable[0]; } + void SetHashTableSize(unsigned int const sz) { PkgHashTable[0] = sz; } + map_pointer_t GetArchitectures() const { return PkgHashTable[1]; } + void SetArchitectures(map_pointer_t const idx) { PkgHashTable[1] = idx; } +#endif + map_pointer_t * PkgHashTableP() const { return (map_pointer_t*) (this + 1); } + map_pointer_t * GrpHashTableP() const { return PkgHashTableP() + GetHashTableSize(); } /** \brief Size of the complete cache file */ - unsigned long long CacheFileSize; + should_be_uint64_small_t CacheFileSize; bool CheckSizes(Header &Against) const APT_PURE; Header(); @@ -350,7 +400,7 @@ struct pkgCache::Group /** \brief Link to the next Group */ map_pointer_t Next; // Group /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; }; /*}}}*/ @@ -387,12 +437,18 @@ struct pkgCache::Package map_pointer_t VersionList; // Version /** \brief index to the installed version */ map_pointer_t CurrentVer; // Version + /** \brief indicates nothing (consistently) + This field used to contain ONE section the package belongs to, + if those differs between versions it is a RANDOM one. + The Section() method tries to reproduce it, but the only sane + thing to do is use the Section field from the version! */ + APT_DEPRECATED map_ptrloc Section; // StringItem /** \brief index of the group this package belongs to */ map_pointer_t Group; // Group the Package belongs to // Linked list /** \brief Link to the next package in the same bucket */ - map_pointer_t Next; // Package + map_pointer_t NextPackage; // Package /** \brief List of all dependencies on this package */ map_pointer_t RevDepends; // Dependency /** \brief List of all "packages" this package provide */ @@ -416,7 +472,7 @@ struct pkgCache::Package This allows clients to create an array of size PackageCount and use it to store state information for the package map. For instance the status file emitter uses this to track which packages have been emitted already. */ - map_id_t ID; + should_be_map_id_t ID; /** \brief some useful indicators of the package's state */ unsigned long Flags; }; @@ -464,7 +520,7 @@ struct pkgCache::PackageFile /** \brief Link to the next PackageFile in the Cache */ map_pointer_t NextFile; // PackageFile /** \brief unique sequel ID */ - map_fileid_t ID; + should_be_map_fileid_t ID; }; /*}}}*/ // VerFile structure /*{{{*/ @@ -479,7 +535,7 @@ struct pkgCache::VerFile /** \brief next step in the linked list */ map_pointer_t NextFile; // PkgVerFile /** \brief position in the package file */ - map_filesize_t Offset; // File offset + should_be_map_filesize_t Offset; // File offset /** @TODO document pkgCache::VerFile::Size */ map_filesize_t Size; }; @@ -493,7 +549,7 @@ struct pkgCache::DescFile /** \brief next step in the linked list */ map_pointer_t NextFile; // PkgVerFile /** \brief position in the file */ - map_filesize_t Offset; // File offset + should_be_map_filesize_t Offset; // File offset /** @TODO document pkgCache::DescFile::Size */ map_filesize_t Size; }; @@ -556,16 +612,16 @@ struct pkgCache::Version /** \brief archive size for this version For Debian this is the size of the .deb file. */ - uint64_t Size; // These are the .deb size + should_be_uint64_t Size; // These are the .deb size /** \brief uncompressed size for this version */ - uint64_t InstalledSize; + should_be_uint64_t InstalledSize; /** \brief characteristic value representing this version No two packages in existence should have the same VerStr and Hash with different contents. */ unsigned short Hash; /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; /** \brief parsed priority value */ unsigned char Priority; }; @@ -593,7 +649,7 @@ struct pkgCache::Description map_pointer_t ParentPkg; // Package /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; }; /*}}}*/ // Dependency structure /*{{{*/ @@ -620,7 +676,7 @@ struct pkgCache::Dependency map_pointer_t ParentVer; // Version /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ unsigned char Type; /** \brief comparison operator specified on the depends line @@ -656,6 +712,15 @@ struct pkgCache::Provides map_pointer_t NextPkgProv; // Provides }; /*}}}*/ +// UNUSED StringItem structure /*{{{*/ +struct APT_DEPRECATED pkgCache::StringItem +{ + /** \brief string this refers to */ + map_ptrloc String; // StringItem + /** \brief Next link in the chain */ + map_ptrloc NextItem; // StringItem +}; + /*}}}*/ inline char const * pkgCache::NativeArch() { return StrP + HeaderP->Architecture; } -- cgit v1.2.3