From b0f4b486e6850c5f98520ccf19da71d0ed748ae4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 21 Sep 2014 10:18:03 +0200 Subject: generalize Acquire::GzipIndex --- apt-pkg/deb/debindexfile.cc | 60 ++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 22 deletions(-) (limited to 'apt-pkg/deb') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index a0dd15cd8..5b4289e92 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -80,14 +80,18 @@ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const { string SourcesURI = _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI("Sources")); - string SourcesURIgzip = SourcesURI + ".gz"; - if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip)) - return NULL; - else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip)) - SourcesURI = SourcesURIgzip; - - return new debSrcRecordParser(SourcesURI,this); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p; + p = SourcesURI + '.' + *t; + if (FileExists(p)) + return new debSrcRecordParser(p, this); + } + if (FileExists(SourcesURI)) + return new debSrcRecordParser(SourcesURI, this); + return NULL; } /*}}}*/ // SourcesIndex::Describe - Give a descriptive path to the index /*{{{*/ @@ -129,11 +133,15 @@ string debSourcesIndex::Info(const char *Type) const inline string debSourcesIndex::IndexFile(const char *Type) const { string s = URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debSourcesIndex::IndexURI(const char *Type) const @@ -259,11 +267,15 @@ string debPackagesIndex::Info(const char *Type) const inline string debPackagesIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debPackagesIndex::IndexURI(const char *Type) const { @@ -411,11 +423,15 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section inline string debTranslationsIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debTranslationsIndex::IndexURI(const char *Type) const { -- cgit v1.2.3 From 78a5476f3177a2a74ae51a1878c26ca322a25003 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 21:33:56 +0200 Subject: drop stored StringItems in favor of in-memory mappings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strings like Section names or architectures are needed vary often. Instead of writing them each time we need them, we deploy sharing for these special strings. Until now, this was done with a linked list of strings in which we would search, which was stored in the cache. It turns out we can do this just as well in memory as well with a bunch of std::map's. In memory means here that it isn't available anymore if we have a partly invalid cache, but that isn't much of a problem in practice as the status file is compared to the other files we parse very small and includes mostly duplicates, so the space we would gain by storing is more or less equal to the size of the stored linked list… --- apt-pkg/deb/debindexfile.cc | 2 +- apt-pkg/deb/deblistparser.cc | 45 +++++++++++++++++++------------------------- apt-pkg/deb/deblistparser.h | 1 - 3 files changed, 20 insertions(+), 28 deletions(-) (limited to 'apt-pkg/deb') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index feda8d0d8..c35a89ade 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -619,7 +619,7 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); - map_stringitem_t const storage = Gen.WriteUniqString("now"); + map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now"); CFile->Archive = storage; if (Gen.MergeList(Parser) == false) diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 3e0d3a791..9919d2036 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -58,18 +58,6 @@ debListParser::debListParser(FileFd *File, string const &Arch) : Tags(File), MultiArchEnabled = Architectures.size() > 1; } /*}}}*/ -// ListParser::UniqFindTagWrite - Find the tag and write a unq string /*{{{*/ -// --------------------------------------------------------------------- -/* */ -map_stringitem_t debListParser::UniqFindTagWrite(const char *Tag) -{ - const char *Start; - const char *Stop; - if (Section.Find(Tag,Start,Stop) == false) - return 0; - return WriteUniqString(Start,Stop - Start); -} - /*}}}*/ // ListParser::Package - Return the package name /*{{{*/ // --------------------------------------------------------------------- /* This is to return the name of the package this section describes */ @@ -144,9 +132,16 @@ unsigned char debListParser::ParseMultiArch(bool const showErrors) /*{{{*/ /* */ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) { + const char *Start; + const char *Stop; + // Parse the section - unsigned long const idxSection = UniqFindTagWrite("Section"); - Ver->Section = idxSection; + if (Section.Find("Section",Start,Stop) == true) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::SECTION, Start, Stop - Start); + Ver->Section = idx; + } + Ver->MultiArch = ParseMultiArch(true); // Archive Size Ver->Size = Section.FindULL("Size"); @@ -155,10 +150,8 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) Ver->InstalledSize *= 1024; // Priority - const char *Start; - const char *Stop; if (Section.Find("Priority",Start,Stop) == true) - { + { if (GrabWord(string(Start,Stop-Start),PrioList,Ver->Priority) == false) Ver->Priority = pkgCache::State::Extra; } @@ -891,7 +884,7 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, { // apt-secure does no longer download individual (per-section) Release // file. to provide Component pinning we use the section name now - map_stringitem_t const storage = WriteUniqString(component); + map_stringitem_t const storage = StoreString(pkgCacheGenerator::MIXED, component); FileI->Component = storage; pkgTagFile TagFile(&File, File.Size()); @@ -900,19 +893,19 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, return false; std::string data; - #define APT_INRELEASE(TAG, STORE) \ + #define APT_INRELEASE(TYPE, TAG, STORE) \ data = Section.FindS(TAG); \ if (data.empty() == false) \ { \ - map_stringitem_t const storage = WriteUniqString(data); \ + map_stringitem_t const storage = StoreString(pkgCacheGenerator::TYPE, data); \ STORE = storage; \ } - APT_INRELEASE("Suite", FileI->Archive) - APT_INRELEASE("Component", FileI->Component) - APT_INRELEASE("Version", FileI->Version) - APT_INRELEASE("Origin", FileI->Origin) - APT_INRELEASE("Codename", FileI->Codename) - APT_INRELEASE("Label", FileI->Label) + APT_INRELEASE(MIXED, "Suite", FileI->Archive) + APT_INRELEASE(MIXED, "Component", FileI->Component) + APT_INRELEASE(VERSION, "Version", FileI->Version) + APT_INRELEASE(MIXED, "Origin", FileI->Origin) + APT_INRELEASE(MIXED, "Codename", FileI->Codename) + APT_INRELEASE(MIXED, "Label", FileI->Label) #undef APT_INRELEASE Section.FindFlag("NotAutomatic", FileI->Flags, pkgCache::Flag::NotAutomatic); Section.FindFlag("ButAutomaticUpgrades", FileI->Flags, pkgCache::Flag::ButAutomaticUpgrades); diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index f5ac47e1e..b55e57d41 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -49,7 +49,6 @@ class debListParser : public pkgCacheGenerator::ListParser std::vector Architectures; bool MultiArchEnabled; - map_stringitem_t UniqFindTagWrite(const char *Tag); virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver); bool ParseDepends(pkgCache::VerIterator &Ver,const char *Tag, unsigned int Type); -- cgit v1.2.3 From a221efc331693f8905da870141756c892911c433 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 19:34:40 +0200 Subject: store source name and version in binary cache Accessing the package records to acquire this information is pretty costly, so that information wasn't used so far in many places. The most noticeable user by far is EDSP at the moment, but there are ideas to change that which this commit tries to enable. --- apt-pkg/deb/deblistparser.cc | 51 ++++++++++++++++++++++++++++++++++++++++++++ apt-pkg/deb/dpkgpm.cc | 7 +----- 2 files changed, 52 insertions(+), 6 deletions(-) (limited to 'apt-pkg/deb') diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 9919d2036..103b126de 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -141,6 +141,57 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) map_stringitem_t const idx = StoreString(pkgCacheGenerator::SECTION, Start, Stop - Start); Ver->Section = idx; } + // Parse the source package name + pkgCache::GrpIterator const G = Ver.ParentPkg().Group(); + Ver->SourcePkgName = G->Name; + Ver->SourceVerStr = Ver->VerStr; + if (Section.Find("Source",Start,Stop) == true) + { + const char * const Space = (const char * const) memchr(Start, ' ', Stop - Start); + pkgCache::VerIterator V; + + if (Space != NULL) + { + Stop = Space; + const char * const Open = (const char * const) memchr(Space, '(', Stop - Space); + if (likely(Open != NULL)) + { + const char * const Close = (const char * const) memchr(Open, ')', Stop - Open); + if (likely(Close != NULL)) + { + std::string const version(Open + 1, (Close - Open) - 1); + if (version != Ver.VerStr()) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::VERSION, version); + Ver->SourceVerStr = idx; + } + } + } + } + + std::string const pkgname(Start, Stop - Start); + if (pkgname != G.Name()) + { + for (pkgCache::PkgIterator P = G.PackageList(); P.end() == false; P = G.NextPkg(P)) + { + for (V = P.VersionList(); V.end() == false; ++V) + { + if (pkgname == V.SourcePkgName()) + { + Ver->SourcePkgName = V->SourcePkgName; + break; + } + } + if (V.end() == false) + break; + } + if (V.end() == true) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::PKGNAME, pkgname); + Ver->SourcePkgName = idx; + } + } + } Ver->MultiArch = ParseMultiArch(true); // Archive Size diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index a5c05d5ea..95fae9a28 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1754,11 +1754,6 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) if (Ver.end() == true) return; pkgver = Ver.VerStr() == NULL ? "unknown" : Ver.VerStr(); - pkgRecords Recs(Cache); - pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); - srcpkgname = Parse.SourcePkg(); - if(srcpkgname.empty()) - srcpkgname = pkgname; // if the file exists already, we check: // - if it was reported already (touched by apport). @@ -1809,7 +1804,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) time_t now = time(NULL); fprintf(report, "Date: %s" , ctime(&now)); fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str()); - fprintf(report, "SourcePackage: %s\n", srcpkgname.c_str()); + fprintf(report, "SourcePackage: %s\n", Ver.SourcePkgName()); fprintf(report, "ErrorMessage:\n %s\n", errormsg); // ensure that the log is flushed -- cgit v1.2.3