From 1d29e0e4c3072a18a0748ec89af1c1b8074a35a3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 14 Oct 2011 14:51:56 +0200 Subject: cherry pick -r 2184.1.15..2184.1.18 from debian-experimental2 --- apt-pkg/contrib/hashsum_template.h | 6 +- apt-pkg/deb/deblistparser.cc | 5 +- apt-pkg/pkgcachegen.cc | 340 +++++++++++++-------- apt-pkg/pkgcachegen.h | 6 + debian/changelog | 10 + .../test-bug-601016-description-translation | 34 ++- 6 files changed, 269 insertions(+), 132 deletions(-) diff --git a/apt-pkg/contrib/hashsum_template.h b/apt-pkg/contrib/hashsum_template.h index 85d94c2af..fe0984f4e 100644 --- a/apt-pkg/contrib/hashsum_template.h +++ b/apt-pkg/contrib/hashsum_template.h @@ -29,7 +29,11 @@ class HashSumValue bool operator ==(const HashSumValue &rhs) const { return memcmp(Sum,rhs.Sum,sizeof(Sum)) == 0; - }; + }; + bool operator !=(const HashSumValue &rhs) const + { + return memcmp(Sum,rhs.Sum,sizeof(Sum)) != 0; + }; string Value() const { diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 08c24204b..89999fb44 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -69,10 +69,7 @@ string debListParser::Package() { // --------------------------------------------------------------------- /* This will return the Architecture of the package this section describes */ string debListParser::Architecture() { - std::string const Arch = Section.FindS("Architecture"); - if (Arch.empty() == true) - return _config->Find("APT::Architecture"); - return Arch; + return Section.FindS("Architecture"); } /*}}}*/ // ListParser::ArchitectureAll /*{{{*/ diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index f41bb5a0a..52586c41b 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -38,6 +38,9 @@ typedef vector::iterator FileIterator; template std::vector pkgCacheGenerator::Dynamic::toReMap; +bool IsDuplicateDescription(pkgCache::DescIterator Desc, + MD5SumValue const &CurMd5, std::string const &CurLang); + // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- /* We set the dirty flag and make sure that is written to the disk */ @@ -182,82 +185,150 @@ bool pkgCacheGenerator::MergeList(ListParser &List, if (PackageName.empty() == true) return false; - string const Arch = List.Architecture(); - + Counter++; + if (Counter % 100 == 0 && Progress != 0) + Progress->Progress(List.Offset()); + + string Arch = List.Architecture(); + string const Version = List.Version(); + if (Version.empty() == true && Arch.empty() == true) + { + if (MergeListGroup(List, PackageName) == false) + return false; + } + + if (Arch.empty() == true) + Arch = _config->Find("APT::Architecture"); + // Get a pointer to the package structure pkgCache::PkgIterator Pkg; Dynamic DynPkg(Pkg); if (NewPackage(Pkg, PackageName, Arch) == false) return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str()); - Counter++; - if (Counter % 100 == 0 && Progress != 0) - Progress->Progress(List.Offset()); - /* Get a pointer to the version structure. We know the list is sorted - so we use that fact in the search. Insertion of new versions is - done with correct sorting */ - string Version = List.Version(); + if (Version.empty() == true) { - // we first process the package, then the descriptions - // (this has the bonus that we get MMap error when we run out - // of MMap space) - pkgCache::VerIterator Ver(Cache); - Dynamic DynVer(Ver); - if (List.UsePackage(Pkg, Ver) == false) - return _error->Error(_("Error occurred while processing %s (UsePackage1)"), - PackageName.c_str()); - - // Find the right version to write the description - MD5SumValue CurMd5 = List.Description_md5(); - Ver = Pkg.VersionList(); - - for (; Ver.end() == false; ++Ver) - { - pkgCache::DescIterator Desc = Ver.DescriptionList(); - Dynamic DynDesc(Desc); - map_ptrloc *LastDesc = &Ver->DescriptionList; - bool duplicate=false; - - // don't add a new description if we have one for the given - // md5 && language - for ( ; Desc.end() == false; ++Desc) - if (MD5SumValue(Desc.md5()) == CurMd5 && - Desc.LanguageCode() == List.DescriptionLanguage()) - duplicate=true; - if(duplicate) - continue; - - for (Desc = Ver.DescriptionList(); - Desc.end() == false; - LastDesc = &Desc->NextDesc, ++Desc) - { - if (MD5SumValue(Desc.md5()) == CurMd5) - { - // Add new description - void const * const oldMap = Map.Data(); - map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); - if (oldMap != Map.Data()) - LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; - *LastDesc = descindex; - Desc->ParentPkg = Pkg.Index(); - - if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) - return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str()); - break; - } - } - } + if (MergeListPackage(List, Pkg) == false) + return false; + } + else + { + if (MergeListVersion(List, Pkg, Version, OutVer) == false) + return false; + } - continue; + if (OutVer != 0) + { + FoundFileDeps |= List.HasFileDeps(); + return true; } + } - pkgCache::VerIterator Ver = Pkg.VersionList(); - Dynamic DynVer(Ver); - map_ptrloc *LastVer = &Pkg->VersionList; - void const * oldMap = Map.Data(); + if (Cache.HeaderP->PackageCount >= (1ULL<ID)*8)-1) + return _error->Error(_("Wow, you exceeded the number of package " + "names this APT is capable of.")); + if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1) + return _error->Error(_("Wow, you exceeded the number of versions " + "this APT is capable of.")); + if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1) + return _error->Error(_("Wow, you exceeded the number of descriptions " + "this APT is capable of.")); + if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL) + return _error->Error(_("Wow, you exceeded the number of dependencies " + "this APT is capable of.")); + + FoundFileDeps |= List.HasFileDeps(); + return true; +} +// CacheGenerator::MergeListGroup /*{{{*/ +bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName) +{ + pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName); + // a group has no data on it's own, only packages have it but these + // stanzas like this come from Translation- files to add descriptions, + // but without a version we don't need a description for it… + if (Grp.end() == true) + return true; + Dynamic DynGrp(Grp); + + pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); + for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) + if (MergeListPackage(List, Pkg) == false) + return false; + + return true; +} + /*}}}*/ +// CacheGenerator::MergeListPackage /*{{{*/ +bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg) +{ + // we first process the package, then the descriptions + // (for deb this package processing is in fact a no-op) + pkgCache::VerIterator Ver(Cache); + Dynamic DynVer(Ver); + if (List.UsePackage(Pkg, Ver) == false) + return _error->Error(_("Error occurred while processing %s (UsePackage1)"), + Pkg.Name()); + + // Find the right version to write the description + MD5SumValue CurMd5 = List.Description_md5(); + std::string CurLang = List.DescriptionLanguage(); + + for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver) + { + pkgCache::DescIterator Desc = Ver.DescriptionList(); + + // a version can only have one md5 describing it + if (MD5SumValue(Desc.md5()) != CurMd5) + continue; + + // don't add a new description if we have one for the given + // md5 && language + if (IsDuplicateDescription(Desc, CurMd5, CurLang) == true) + continue; + + Dynamic DynDesc(Desc); + // we add at the end, so that the start is constant as we need + // that to be able to efficiently share these lists + map_ptrloc *LastDesc = &Ver->DescriptionList; + for (;Desc.end() == false && Desc->NextDesc != 0; ++Desc); + if (Desc.end() == false) + LastDesc = &Desc->NextDesc; + + void const * const oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; + Desc->ParentPkg = Pkg.Index(); + + if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) + return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"), Pkg.Name()); + + // we can stop here as all "same" versions will share the description + break; + } + + return true; +} + /*}}}*/ +// CacheGenerator::MergeListVersion /*{{{*/ +bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg, + std::string const &Version, pkgCache::VerIterator* &OutVer) +{ + pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); + map_ptrloc *LastVer = &Pkg->VersionList; + void const * oldMap = Map.Data(); + + unsigned long const Hash = List.VersionHash(); + if (Ver.end() == false) + { + /* We know the list is sorted so we use that fact in the search. + Insertion of new versions is done with correct sorting */ int Res = 1; - unsigned long const Hash = List.VersionHash(); for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) { Res = Cache.VS->CmpVersion(Version,Ver.VerStr()); @@ -276,93 +347,96 @@ bool pkgCacheGenerator::MergeList(ListParser &List, { if (List.UsePackage(Pkg,Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage2)"), - PackageName.c_str()); + Pkg.Name()); if (NewFileVer(Ver,List) == false) return _error->Error(_("Error occurred while processing %s (NewFileVer1)"), - PackageName.c_str()); - + Pkg.Name()); + // Read only a single record and return if (OutVer != 0) { *OutVer = Ver; - FoundFileDeps |= List.HasFileDeps(); return true; } - - continue; + + return true; } + } - // Add a new version - map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer); - if (verindex == 0 && _error->PendingError()) - return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), - PackageName.c_str(), 1); + // Add a new version + map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer); + if (verindex == 0 && _error->PendingError()) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + Pkg.Name(), 1); - if (oldMap != Map.Data()) + if (oldMap != Map.Data()) LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; - *LastVer = verindex; - Ver->ParentPkg = Pkg.Index(); - Ver->Hash = Hash; + *LastVer = verindex; + Ver->ParentPkg = Pkg.Index(); + Ver->Hash = Hash; - if (List.NewVersion(Ver) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), - PackageName.c_str(), 2); + if (List.NewVersion(Ver) == false) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + Pkg.Name(), 2); - if (List.UsePackage(Pkg,Ver) == false) - return _error->Error(_("Error occurred while processing %s (UsePackage3)"), - PackageName.c_str()); - - if (NewFileVer(Ver,List) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), - PackageName.c_str(), 3); + if (List.UsePackage(Pkg,Ver) == false) + return _error->Error(_("Error occurred while processing %s (UsePackage3)"), + Pkg.Name()); - // Read only a single record and return - if (OutVer != 0) - { - *OutVer = Ver; - FoundFileDeps |= List.HasFileDeps(); - return true; - } + if (NewFileVer(Ver,List) == false) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + Pkg.Name(), 3); - /* Record the Description data. Description data always exist in - Packages and Translation-* files. */ - pkgCache::DescIterator Desc = Ver.DescriptionList(); - Dynamic DynDesc(Desc); - map_ptrloc *LastDesc = &Ver->DescriptionList; + // Read only a single record and return + if (OutVer != 0) + { + *OutVer = Ver; + return true; + } - // Skip to the end of description set - for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++); + /* Record the Description (it is not translated) */ + MD5SumValue CurMd5 = List.Description_md5(); + if (CurMd5.Value().empty() == true) + return true; + std::string CurLang = List.DescriptionLanguage(); + + /* Before we add a new description we first search in the group for + a version with a description of the same MD5 - if so we reuse this + description group instead of creating our own for this version */ + pkgCache::GrpIterator Grp = Pkg.Group(); + for (pkgCache::PkgIterator P = Grp.PackageList(); + P.end() == false; P = Grp.NextPkg(P)) + { + for (pkgCache::VerIterator V = P.VersionList(); + V.end() == false; ++V) + { + if (IsDuplicateDescription(V.DescriptionList(), CurMd5, "") == false) + continue; + Ver->DescriptionList = V->DescriptionList; + return true; + } + } - // Add new description - oldMap = Map.Data(); - map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); - if (oldMap != Map.Data()) - LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; - *LastDesc = descindex; - Desc->ParentPkg = Pkg.Index(); + // We haven't found reusable descriptions, so add the first description + pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); + map_ptrloc *LastDesc = &Ver->DescriptionList; - if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) - return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str()); - } + oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; + Desc->ParentPkg = Pkg.Index(); - FoundFileDeps |= List.HasFileDeps(); + if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) + return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),Pkg.Name()); - if (Cache.HeaderP->PackageCount >= (1ULL<ID)*8)-1) - return _error->Error(_("Wow, you exceeded the number of package " - "names this APT is capable of.")); - if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1) - return _error->Error(_("Wow, you exceeded the number of versions " - "this APT is capable of.")); - if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1) - return _error->Error(_("Wow, you exceeded the number of descriptions " - "this APT is capable of.")); - if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL) - return _error->Error(_("Wow, you exceeded the number of dependencies " - "this APT is capable of.")); return true; } /*}}}*/ + /*}}}*/ // CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/ // --------------------------------------------------------------------- /* If we found any file depends while parsing the main list we need to @@ -1351,3 +1425,17 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O return true; } /*}}}*/ +// IsDuplicateDescription /*{{{*/ +bool IsDuplicateDescription(pkgCache::DescIterator Desc, + MD5SumValue const &CurMd5, std::string const &CurLang) +{ + // Descriptions in the same link-list have all the same md5 + if (MD5SumValue(Desc.md5()) != CurMd5) + return false; + for (; Desc.end() == false; ++Desc) + if (Desc.LanguageCode() == CurLang) + return true; + return false; +} + /*}}}*/ + diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index c26051182..688eb0f11 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -105,6 +105,12 @@ class pkgCacheGenerator /*{{{*/ pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); + + private: + bool MergeListGroup(ListParser &List, std::string const &GrpName); + bool MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg); + bool MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg, + std::string const &Version, pkgCache::VerIterator* &OutVer); }; /*}}}*/ // This is the abstract package list parser class. /*{{{*/ diff --git a/debian/changelog b/debian/changelog index f4f139280..5a76bf00f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +apt (868977) UNRELEASED; urgency=low + + [ David Kalnischkies ] + * apt-pkg/pkgcachegen.cc: + - refactor MergeList by creating -Group, -Package and -Version specialist + - share description list between "same" versions (LP: #868977) + This also means that descriptions are shared across archives now. + + -- Michael Vogt Fri, 14 Oct 2011 14:51:06 +0200 + apt (0.8.16~exp5ubuntu13) oneiric; urgency=low [ Adam Conrad ] diff --git a/test/integration/test-bug-601016-description-translation b/test/integration/test-bug-601016-description-translation index 2a323a201..3a2114914 100755 --- a/test/integration/test-bug-601016-description-translation +++ b/test/integration/test-bug-601016-description-translation @@ -4,7 +4,7 @@ set -e TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'i386' 'amd64' # we need a valid locale here, otherwise the language configuration # will be overridden by LC_ALL=C @@ -22,8 +22,22 @@ Size: 2140230 MD5sum: 74769bfbcef9ebc4fa74f7a5271b9c08 Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c" +PACKAGESTANZA2='Package: apt +Priority: important +Section: admin +Installed-Size: 5984 +Maintainer: APT Development Team +Architecture: amd64 +Version: 0.8.7 +Filename: pool/main/a/apt/apt_0.8.7_amd64.deb +Size: 2210342 +MD5sum: 4a869bfbdef9ebc9fa74f7a5271e8d1a +Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c' echo "$PACKAGESTANZA +Description: Advanced front-end for dpkg + +$PACKAGESTANZA2 Description: Advanced front-end for dpkg" > aptarchive/Packages echo "Package: apt @@ -53,6 +67,13 @@ Description-${LOCALE}: Mächtige Oberfläche für dpkg APT-Dselect-Methode. Beides sind einfache und sicherere Wege, um Pakete zu installieren und Upgrades durchzuführen. " +LOCALESTANZA2="$PACKAGESTANZA2 +Description-${LOCALE}: Mächtige Oberfläche für dpkg + Das Paket bietet dem Nutzer technisch führende Methoden für den Zugriff + auf den dpkg-Paketmanager. Es beinhaltet das apt-get-Werkzeug und die + APT-Dselect-Methode. Beides sind einfache und sicherere Wege, + um Pakete zu installieren und Upgrades durchzuführen. +" testrun() { echo "Acquire::Languages { \"${LOCALE}\"; \"en\"; };" > rootdir/etc/apt/apt.conf.d/00languages @@ -60,6 +81,8 @@ testrun() { rm -rf rootdir/var/lib/apt/lists rootdir/var/cache/apt/ setupaptarchive testequal "$LOCALESTANZA" aptcache show apt -o Test=File-${LOCALE} + testequal "$LOCALESTANZA" aptcache show apt:i386 -o Test=File-${LOCALE} + testequal "$LOCALESTANZA2" aptcache show apt:amd64 -o Test=File-${LOCALE} testequal "$NOLONGSTANZA" aptcache show apt -o Acquire::Languages="ww" -o Test=File-${LOCALE} testequal "$LOCALESTANZA" aptcache show apt -o Acquire::Languages::="ww" -o Test=File-${LOCALE} LC_ALL=C testequal "$ENGLISHSTANZA" aptcache show apt -o Test=File-${LOCALE} @@ -73,6 +96,9 @@ testrun() { testrun echo "$PACKAGESTANZA +Description: Advanced front-end for dpkg + +$PACKAGESTANZA2 Description: Advanced front-end for dpkg" > aptarchive/Packages echo "Package: apt @@ -88,5 +114,11 @@ Description-en: Advanced front-end for dpkg It provides the apt-get utility and APT dselect method that provides a simpler, safer way to install and upgrade packages. " +ENGLISHSTANZA2="$PACKAGESTANZA2 +Description-en: Advanced front-end for dpkg + This is Debian's next generation front-end for the dpkg package manager. + It provides the apt-get utility and APT dselect method that provides a + simpler, safer way to install and upgrade packages. +" testrun -- cgit v1.2.3