From 448c38bdcd72b52f11ec5f326f822cf57653f81c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 6 Jun 2015 12:28:00 +0200 Subject: rework hashsum verification in the acquire system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having every item having its own code to verify the file(s) it handles is an errorprune process and easy to break, especially if items move through various stages (download, uncompress, patching, …). With a giant rework we centralize (most of) the verification to have a better enforcement rate and (hopefully) less chance for bugs, but it breaks the ABI bigtime in exchange – and as we break it anyway, it is broken even harder. It shouldn't effect most frontends as they don't deal with the acquire system at all or implement their own items, but some do and will need to be patched (might be an opportunity to use apt on-board material). The theory is simple: Items implement methods to decide if hashes need to be checked (in this stage) and to return the expected hashes for this item (in this stage). The verification itself is done in worker message passing which has the benefit that a hashsum error is now a proper error for the acquire system rather than a Done() which is later revised to a Failed(). --- apt-pkg/deb/debmetaindex.cc | 78 ++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 47 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index aa2db8149..eb5e78e3b 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -192,11 +192,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const { vector const SectionEntries = src->second; for (vector::const_iterator I = SectionEntries.begin(); I != SectionEntries.end(); ++I) { - IndexTarget * Target = new IndexTarget(); - Target->ShortDesc = "Sources"; - Target->MetaKey = SourceIndexURISuffix(Target->ShortDesc.c_str(), (*I)->Section); - Target->URI = SourceIndexURI(Target->ShortDesc.c_str(), (*I)->Section); - Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section); + char const * const ShortDesc = "Sources"; + IndexTarget * const Target = new IndexTarget( + SourceIndexURISuffix(ShortDesc, (*I)->Section), + ShortDesc, + Info(ShortDesc, (*I)->Section), + SourceIndexURI(ShortDesc, (*I)->Section) + ); IndexTargets->push_back (Target); } } @@ -212,11 +214,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const { continue; for (vector ::const_iterator I = a->second.begin(); I != a->second.end(); ++I) { - IndexTarget * Target = new IndexTarget(); - Target->ShortDesc = "Packages"; - Target->MetaKey = IndexURISuffix(Target->ShortDesc.c_str(), (*I)->Section, a->first); - Target->URI = IndexURI(Target->ShortDesc.c_str(), (*I)->Section, a->first); - Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section, a->first); + char const * const ShortDesc = "Packages"; + IndexTarget * const Target = new IndexTarget( + IndexURISuffix(ShortDesc, (*I)->Section, a->first), + ShortDesc, + Info (ShortDesc, (*I)->Section, a->first), + IndexURI(ShortDesc, (*I)->Section, a->first) + ); IndexTargets->push_back (Target); sections.insert((*I)->Section); } @@ -235,11 +239,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const { s != sections.end(); ++s) { for (std::vector::const_iterator l = lang.begin(); l != lang.end(); ++l) { - IndexTarget * Target = new OptionalIndexTarget(); - Target->ShortDesc = "Translation-" + *l; - Target->MetaKey = TranslationIndexURISuffix(l->c_str(), *s); - Target->URI = TranslationIndexURI(l->c_str(), *s); - Target->Description = Info (Target->ShortDesc.c_str(), *s); + std::string const ShortDesc = "Translation-" + *l; + IndexTarget * const Target = new OptionalIndexTarget( + TranslationIndexURISuffix(l->c_str(), *s), + ShortDesc, + Info (ShortDesc.c_str(), *s), + TranslationIndexURI(l->c_str(), *s) + ); IndexTargets->push_back(Target); } } @@ -249,8 +255,6 @@ vector * debReleaseIndex::ComputeIndexTargets() const { /*}}}*/ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const { - bool const tryInRelease = _config->FindB("Acquire::TryInRelease", true); - indexRecords * const iR = new indexRecords(Dist); if (Trusted == ALWAYS_TRUSTED) iR->SetTrusted(true); @@ -258,37 +262,17 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const iR->SetTrusted(false); // special case for --print-uris - if (GetAll) { - vector *targets = ComputeIndexTargets(); - for (vector ::const_iterator Target = targets->begin(); Target != targets->end(); ++Target) { - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, HashStringList()); - } - delete targets; - - // this is normally created in pkgAcqMetaSig, but if we run - // in --print-uris mode, we add it here - if (tryInRelease == false) - new pkgAcqMetaIndex(Owner, NULL, - MetaIndexURI("Release"), - MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", - ComputeIndexTargets(), - iR); + vector const * const targets = ComputeIndexTargets(); +#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X)) + pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner, + APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), + targets, iR); +#undef APT_TARGET + if (GetAll) + { + for (vector ::const_iterator Target = targets->begin(); Target != targets->end(); ++Target) + new pkgAcqIndex(Owner, TransactionManager, *Target); } - if (tryInRelease == true) - new pkgAcqMetaClearSig(Owner, - MetaIndexURI("InRelease"), MetaIndexInfo("InRelease"), "InRelease", - MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", - ComputeIndexTargets(), - iR); - else - new pkgAcqMetaIndex(Owner, NULL, - MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", - ComputeIndexTargets(), - iR); return true; } -- cgit v1.2.3 From 1e0f0f28e1025f42a8172eb72f3e87984eb2b939 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 9 Jun 2015 11:59:22 +0200 Subject: configureable acquire targets to download additional files First pass at making the acquire system capable of downloading files based on configuration rather than hardcoded entries. It is now possible to instruct 'deb' and 'deb-src' sources.list lines to download more than just Packages/Translation-* and Sources files. Details on how to do that can be found in the included documentation file. --- apt-pkg/deb/debmetaindex.cc | 317 +++++++++++++++++++++----------------------- 1 file changed, 150 insertions(+), 167 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index eb5e78e3b..8fef05ab0 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -23,25 +23,6 @@ using namespace std; -string debReleaseIndex::Info(const char *Type, string const &Section, string const &Arch) const -{ - string Info = ::URI::SiteOnly(URI) + ' '; - if (Dist[Dist.size() - 1] == '/') - { - if (Dist != "/") - Info += Dist; - } - else - { - Info += Dist + '/' + Section; - if (Arch.empty() != true) - Info += " " + Arch; - } - Info += " "; - Info += Type; - return Info; -} - string debReleaseIndex::MetaIndexInfo(const char *Type) const { string Info = ::URI::SiteOnly(URI) + ' '; @@ -92,81 +73,6 @@ std::string debReleaseIndex::LocalFileName() const return ""; } -string debReleaseIndex::IndexURISuffix(const char *Type, string const &Section, string const &Arch) const -{ - string Res =""; - if (Dist[Dist.size() - 1] != '/') - { - if (Arch == "native") - Res += Section + "/binary-" + _config->Find("APT::Architecture") + '/'; - else - Res += Section + "/binary-" + Arch + '/'; - } - return Res + Type; -} - - -string debReleaseIndex::IndexURI(const char *Type, string const &Section, string const &Arch) const -{ - if (Dist[Dist.size() - 1] == '/') - { - string Res; - if (Dist != "/") - Res = URI + Dist; - else - Res = URI; - return Res + Type; - } - else - return URI + "dists/" + Dist + '/' + IndexURISuffix(Type, Section, Arch); - } - -string debReleaseIndex::SourceIndexURISuffix(const char *Type, const string &Section) const -{ - string Res =""; - if (Dist[Dist.size() - 1] != '/') - Res += Section + "/source/"; - return Res + Type; -} - -string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section) const -{ - string Res; - if (Dist[Dist.size() - 1] == '/') - { - if (Dist != "/") - Res = URI + Dist; - else - Res = URI; - return Res + Type; - } - else - return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section); -} - -string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const -{ - string Res =""; - if (Dist[Dist.size() - 1] != '/') - Res += Section + "/i18n/Translation-"; - return Res + Type; -} - -string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const -{ - string Res; - if (Dist[Dist.size() - 1] == '/') - { - if (Dist != "/") - Res = URI + Dist; - else - Res = URI; - return Res + Type; - } - else - return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section); -} - debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) : metaIndex(URI, Dist, "deb"), Trusted(CHECK_TRUST) {} @@ -184,73 +90,161 @@ debReleaseIndex::~debReleaseIndex() { delete *S; } -vector * debReleaseIndex::ComputeIndexTargets() const { - vector * IndexTargets = new vector ; +vector * debReleaseIndex::ComputeIndexTargets() const +{ + vector * IndexTargets = new vector ; - map >::const_iterator const src = ArchEntries.find("source"); - if (src != ArchEntries.end()) { - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); - I != SectionEntries.end(); ++I) { - char const * const ShortDesc = "Sources"; - IndexTarget * const Target = new IndexTarget( - SourceIndexURISuffix(ShortDesc, (*I)->Section), - ShortDesc, - Info(ShortDesc, (*I)->Section), - SourceIndexURI(ShortDesc, (*I)->Section) - ); - IndexTargets->push_back (Target); - } - } + bool const flatArchive = (Dist[Dist.length() - 1] == '/'); + std::string baseURI = URI; + if (flatArchive) + { + if (Dist != "/") + baseURI += Dist; + } + else + baseURI += "dists/" + Dist + "/"; + std::string const Release = (Dist == "/") ? "" : Dist; + std::string const Site = ::URI::SiteOnly(URI); + std::vector lang = APT::Configuration::getLanguages(true); + if (lang.empty()) + lang.push_back("none"); + map >::const_iterator const src = ArchEntries.find("source"); + if (src != ArchEntries.end()) + { + std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true); + for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) + { +#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X)) + std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI"); + std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); + std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); + bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true); +#undef APT_T_CONFIG + if (URI.empty()) + continue; - // Only source release - if (IndexTargets->empty() == false && ArchEntries.size() == 1) - return IndexTargets; + vector const SectionEntries = src->second; + for (vector::const_iterator I = SectionEntries.begin(); + I != SectionEntries.end(); ++I) + { + for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) + { + if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos) + continue; + + struct SubstVar subst[] = { + { "$(SITE)", &Site }, + { "$(RELEASE)", &Release }, + { "$(COMPONENT)", &((*I)->Section) }, + { "$(LANGUAGE)", &(*l) }, + { NULL, NULL } + }; + std::string const name = SubstVar(URI, subst); + IndexTarget * Target; + if (IsOptional == true) + { + Target = new OptionalIndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + else + { + Target = new IndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + IndexTargets->push_back(Target); + + if (URI.find("$(LANGUAGE)") == std::string::npos) + break; + } - std::set sections; - for (map >::const_iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) { - if (a->first == "source") - continue; - for (vector ::const_iterator I = a->second.begin(); - I != a->second.end(); ++I) { - char const * const ShortDesc = "Packages"; - IndexTarget * const Target = new IndexTarget( - IndexURISuffix(ShortDesc, (*I)->Section, a->first), - ShortDesc, - Info (ShortDesc, (*I)->Section, a->first), - IndexURI(ShortDesc, (*I)->Section, a->first) - ); - IndexTargets->push_back (Target); - sections.insert((*I)->Section); - } - } + if (URI.find("$(COMPONENT)") == std::string::npos) + break; + } + } + } - std::vector lang = APT::Configuration::getLanguages(true); - std::vector::iterator lend = std::remove(lang.begin(), lang.end(), "none"); - if (lend != lang.end()) - lang.erase(lend); - - if (lang.empty() == true) - return IndexTargets; - - // get the Translation-* files, later we will skip download of non-existent if we have an index - for (std::set::const_iterator s = sections.begin(); - s != sections.end(); ++s) { - for (std::vector::const_iterator l = lang.begin(); - l != lang.end(); ++l) { - std::string const ShortDesc = "Translation-" + *l; - IndexTarget * const Target = new OptionalIndexTarget( - TranslationIndexURISuffix(l->c_str(), *s), - ShortDesc, - Info (ShortDesc.c_str(), *s), - TranslationIndexURI(l->c_str(), *s) - ); - IndexTargets->push_back(Target); - } - } + // Only source release + if (IndexTargets->empty() == false && ArchEntries.size() == 1) + return IndexTargets; - return IndexTargets; + std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true); + for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) + { +#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X)) + std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI"); + std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); + std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); + bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true); +#undef APT_T_CONFIG + if (URI.empty()) + continue; + + for (map >::const_iterator a = ArchEntries.begin(); + a != ArchEntries.end(); ++a) + { + if (a->first == "source") + continue; + + for (vector ::const_iterator I = a->second.begin(); + I != a->second.end(); ++I) { + + for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) + { + if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos) + continue; + + struct SubstVar subst[] = { + { "$(SITE)", &Site }, + { "$(RELEASE)", &Release }, + { "$(COMPONENT)", &((*I)->Section) }, + { "$(LANGUAGE)", &(*l) }, + { "$(ARCHITECTURE)", &(a->first) }, + { NULL, NULL } + }; + std::string const name = SubstVar(URI, subst); + IndexTarget * Target; + if (IsOptional == true) + { + Target = new OptionalIndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + else + { + Target = new IndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + IndexTargets->push_back(Target); + + if (URI.find("$(LANGUAGE)") == std::string::npos) + break; + } + + if (URI.find("$(COMPONENT)") == std::string::npos) + break; + } + + if (URI.find("$(ARCHITECTURE)") == std::string::npos) + break; + } + } + + return IndexTargets; } /*}}}*/ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const @@ -361,17 +355,6 @@ void debReleaseIndex::PushSectionEntry(string const &Arch, const debSectionEntry ArchEntries[Arch].push_back(Entry); } -void debReleaseIndex::PushSectionEntry(const debSectionEntry *Entry) { - if (Entry->IsSrc == true) - PushSectionEntry("source", Entry); - else { - for (map >::iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) { - a->second.push_back(Entry); - } - } -} - debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section, bool const &IsSrc): Section(Section), IsSrc(IsSrc) {} -- cgit v1.2.3 From 59148d9630bbbd53c6aa10da0fcdbd579797502a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Jun 2015 19:22:41 +0200 Subject: abstract the code to iterate over all targets a bit We have two places in the code which need to iterate over targets and do certain things with it. The first one is actually creating these targets for download and the second instance pepares certain targets for reading. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 197 +++++++++++++++++++++++--------------------- 1 file changed, 103 insertions(+), 94 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 8fef05ab0..c35f3b0a4 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -90,10 +90,11 @@ debReleaseIndex::~debReleaseIndex() { delete *S; } -vector * debReleaseIndex::ComputeIndexTargets() const +template +void foreachTarget(std::string const URI, std::string const Dist, + std::map > const &ArchEntries, + CallC Call) { - vector * IndexTargets = new vector ; - bool const flatArchive = (Dist[Dist.length() - 1] == '/'); std::string baseURI = URI; if (flatArchive) @@ -108,7 +109,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const std::vector lang = APT::Configuration::getLanguages(true); if (lang.empty()) lang.push_back("none"); - map >::const_iterator const src = ArchEntries.find("source"); + map >::const_iterator const src = ArchEntries.find("source"); if (src != ArchEntries.end()) { std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true); @@ -123,8 +124,8 @@ vector * debReleaseIndex::ComputeIndexTargets() const if (URI.empty()) continue; - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); + vector const SectionEntries = src->second; + for (vector::const_iterator I = SectionEntries.begin(); I != SectionEntries.end(); ++I) { for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) @@ -139,27 +140,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const { "$(LANGUAGE)", &(*l) }, { NULL, NULL } }; - std::string const name = SubstVar(URI, subst); - IndexTarget * Target; - if (IsOptional == true) - { - Target = new OptionalIndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - else - { - Target = new IndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - IndexTargets->push_back(Target); + Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst); if (URI.find("$(LANGUAGE)") == std::string::npos) break; @@ -171,10 +152,6 @@ vector * debReleaseIndex::ComputeIndexTargets() const } } - // Only source release - if (IndexTargets->empty() == false && ArchEntries.size() == 1) - return IndexTargets; - std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true); for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) { @@ -187,13 +164,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const if (URI.empty()) continue; - for (map >::const_iterator a = ArchEntries.begin(); + for (map >::const_iterator a = ArchEntries.begin(); a != ArchEntries.end(); ++a) { if (a->first == "source") continue; - for (vector ::const_iterator I = a->second.begin(); + for (vector ::const_iterator I = a->second.begin(); I != a->second.end(); ++I) { for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) @@ -209,27 +186,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const { "$(ARCHITECTURE)", &(a->first) }, { NULL, NULL } }; - std::string const name = SubstVar(URI, subst); - IndexTarget * Target; - if (IsOptional == true) - { - Target = new OptionalIndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - else - { - Target = new IndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - IndexTargets->push_back(Target); + Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst); if (URI.find("$(LANGUAGE)") == std::string::npos) break; @@ -243,9 +200,51 @@ vector * debReleaseIndex::ComputeIndexTargets() const break; } } +} + + +struct ComputeIndexTargetsClass +{ + vector * const IndexTargets; + + void operator()(std::string const &baseURI, std::string const &/*TargetName*/, + std::string const &URI, std::string const &ShortDesc, std::string const &LongDesc, + bool const IsOptional, struct SubstVar const * const subst) + { + std::string const name = SubstVar(URI, subst); + IndexTarget * Target; + if (IsOptional == true) + { + Target = new OptionalIndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + else + { + Target = new IndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + IndexTargets->push_back(Target); + } - return IndexTargets; + ComputeIndexTargetsClass() : IndexTargets(new vector ) {} +}; + +vector * debReleaseIndex::ComputeIndexTargets() const +{ + ComputeIndexTargetsClass comp; + foreachTarget(URI, Dist, ArchEntries, comp); + return comp.IndexTargets; } + + /*}}}*/ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const { @@ -303,45 +302,55 @@ bool debReleaseIndex::IsTrusted() const return FileExists(VerifiedSigFile); } -vector *debReleaseIndex::GetIndexFiles() { - if (Indexes != NULL) - return Indexes; - - Indexes = new vector ; - map >::const_iterator const src = ArchEntries.find("source"); - if (src != ArchEntries.end()) { - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); - I != SectionEntries.end(); ++I) - Indexes->push_back(new debSourcesIndex (URI, Dist, (*I)->Section, IsTrusted())); - } - - // Only source release - if (Indexes->empty() == false && ArchEntries.size() == 1) - return Indexes; - - std::vector const lang = APT::Configuration::getLanguages(true); - map > sections; - for (map >::const_iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) { - if (a->first == "source") - continue; - for (vector::const_iterator I = a->second.begin(); - I != a->second.end(); ++I) { - Indexes->push_back(new debPackagesIndex (URI, Dist, (*I)->Section, IsTrusted(), a->first)); - sections[(*I)->Section].insert(lang.begin(), lang.end()); - } - } - - for (map >::const_iterator s = sections.begin(); - s != sections.end(); ++s) - for (set::const_iterator l = s->second.begin(); - l != s->second.end(); ++l) { - if (*l == "none") continue; - Indexes->push_back(new debTranslationsIndex(URI,Dist,s->first,(*l).c_str())); - } - - return Indexes; +struct GetIndexFilesClass +{ + vector * const Indexes; + std::string const URI; + std::string const Release; + bool const IsTrusted; + + void operator()(std::string const &/*baseURI*/, std::string const &TargetName, + std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/, + bool const /*IsOptional*/, struct SubstVar const * const subst) + { + if (TargetName == "Packages") + { + Indexes->push_back(new debPackagesIndex( + URI, + Release, + SubstVar("$(COMPONENT)", subst), + IsTrusted, + SubstVar("$(ARCHITECTURE)", subst) + )); + } + else if (TargetName == "Sources") + Indexes->push_back(new debSourcesIndex( + URI, + Release, + SubstVar("$(COMPONENT)", subst), + IsTrusted + )); + else if (TargetName == "Translations") + Indexes->push_back(new debTranslationsIndex( + URI, + Release, + SubstVar("$(COMPONENT)", subst), + SubstVar("$(LANGUAGE)", subst) + )); + } + + GetIndexFilesClass(std::string const &URI, std::string const &Release, bool const IsTrusted) : + Indexes(new vector ), URI(URI), Release(Release), IsTrusted(IsTrusted) {} +}; + +std::vector *debReleaseIndex::GetIndexFiles() +{ + if (Indexes != NULL) + return Indexes; + + GetIndexFilesClass comp(URI, Dist, IsTrusted()); + foreachTarget(URI, Dist, ArchEntries, comp); + return Indexes = comp.Indexes; } void debReleaseIndex::PushSectionEntry(vector const &Archs, const debSectionEntry *Entry) { -- cgit v1.2.3 From d3a869e35503638e3483228fbfc95b7143568ad0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Jun 2015 21:24:47 +0200 Subject: store all targets data in IndexTarget struct We still need an API for the targets, so slowly prepare the IndexTargets to let them take this job. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 114 +++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 60 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index c35f3b0a4..f038f12f5 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -116,12 +116,12 @@ void foreachTarget(std::string const URI, std::string const Dist, for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) { #define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X)) - std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI"); + std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true); #undef APT_T_CONFIG - if (URI.empty()) + if (MetaKey.empty()) continue; vector const SectionEntries = src->second; @@ -130,23 +130,24 @@ void foreachTarget(std::string const URI, std::string const Dist, { for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) { - if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos) + if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos) continue; - struct SubstVar subst[] = { - { "$(SITE)", &Site }, - { "$(RELEASE)", &Release }, - { "$(COMPONENT)", &((*I)->Section) }, - { "$(LANGUAGE)", &(*l) }, - { NULL, NULL } - }; - Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst); - - if (URI.find("$(LANGUAGE)") == std::string::npos) + std::map Options; + Options.insert(std::make_pair("SITE", Site)); + Options.insert(std::make_pair("RELEASE", Release)); + Options.insert(std::make_pair("COMPONENT", (*I)->Section)); + Options.insert(std::make_pair("LANGUAGE", *l)); + Options.insert(std::make_pair("ARCHITECTURE", "source")); + Options.insert(std::make_pair("BASE_URI", baseURI)); + Options.insert(std::make_pair("CREATED_BY", *T)); + Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); + + if (MetaKey.find("$(LANGUAGE)") == std::string::npos) break; } - if (URI.find("$(COMPONENT)") == std::string::npos) + if (MetaKey.find("$(COMPONENT)") == std::string::npos) break; } } @@ -156,12 +157,12 @@ void foreachTarget(std::string const URI, std::string const Dist, for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) { #define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X)) - std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI"); + std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true); #undef APT_T_CONFIG - if (URI.empty()) + if (MetaKey.empty()) continue; for (map >::const_iterator a = ArchEntries.begin(); @@ -175,28 +176,28 @@ void foreachTarget(std::string const URI, std::string const Dist, for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) { - if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos) + if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos) continue; - struct SubstVar subst[] = { - { "$(SITE)", &Site }, - { "$(RELEASE)", &Release }, - { "$(COMPONENT)", &((*I)->Section) }, - { "$(LANGUAGE)", &(*l) }, - { "$(ARCHITECTURE)", &(a->first) }, - { NULL, NULL } - }; - Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst); - - if (URI.find("$(LANGUAGE)") == std::string::npos) + std::map Options; + Options.insert(std::make_pair("SITE", Site)); + Options.insert(std::make_pair("RELEASE", Release)); + Options.insert(std::make_pair("COMPONENT", (*I)->Section)); + Options.insert(std::make_pair("LANGUAGE", *l)); + Options.insert(std::make_pair("ARCHITECTURE", a->first)); + Options.insert(std::make_pair("BASE_URI", baseURI)); + Options.insert(std::make_pair("CREATED_BY", *T)); + Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); + + if (MetaKey.find("$(LANGUAGE)") == std::string::npos) break; } - if (URI.find("$(COMPONENT)") == std::string::npos) + if (MetaKey.find("$(COMPONENT)") == std::string::npos) break; } - if (URI.find("$(ARCHITECTURE)") == std::string::npos) + if (MetaKey.find("$(ARCHITECTURE)") == std::string::npos) break; } } @@ -207,30 +208,23 @@ struct ComputeIndexTargetsClass { vector * const IndexTargets; - void operator()(std::string const &baseURI, std::string const &/*TargetName*/, - std::string const &URI, std::string const &ShortDesc, std::string const &LongDesc, - bool const IsOptional, struct SubstVar const * const subst) + void operator()(std::string MetaKey, std::string ShortDesc, std::string LongDesc, + bool const IsOptional, std::map Options) { - std::string const name = SubstVar(URI, subst); - IndexTarget * Target; - if (IsOptional == true) - { - Target = new OptionalIndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - else + for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O) { - Target = new IndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); + MetaKey = SubstVar(MetaKey, std::string("$(") + O->first + ")", O->second); + ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second); + LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second); } + IndexTarget * Target = new IndexTarget( + MetaKey, + ShortDesc, + LongDesc, + Options.find("BASE_URI")->second + MetaKey, + IsOptional, + Options + ); IndexTargets->push_back(Target); } @@ -256,7 +250,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const // special case for --print-uris vector const * const targets = ComputeIndexTargets(); -#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X)) +#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), targets, iR); @@ -309,33 +303,33 @@ struct GetIndexFilesClass std::string const Release; bool const IsTrusted; - void operator()(std::string const &/*baseURI*/, std::string const &TargetName, - std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/, - bool const /*IsOptional*/, struct SubstVar const * const subst) + void operator()(std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/, + bool const /*IsOptional*/, std::map Options) { + std::string const TargetName = Options.find("CREATED_BY")->second; if (TargetName == "Packages") { Indexes->push_back(new debPackagesIndex( URI, Release, - SubstVar("$(COMPONENT)", subst), + Options.find("COMPONENT")->second, IsTrusted, - SubstVar("$(ARCHITECTURE)", subst) + Options.find("ARCHITECTURE")->second )); } else if (TargetName == "Sources") Indexes->push_back(new debSourcesIndex( URI, Release, - SubstVar("$(COMPONENT)", subst), + Options.find("COMPONENT")->second, IsTrusted )); else if (TargetName == "Translations") Indexes->push_back(new debTranslationsIndex( URI, Release, - SubstVar("$(COMPONENT)", subst), - SubstVar("$(LANGUAGE)", subst) + Options.find("COMPONENT")->second, + Options.find("LANGUAGE")->second )); } -- cgit v1.2.3 From dcbbb14df8c9a8a146697720874e9425c4b33792 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Jun 2015 22:10:48 +0200 Subject: stop using IndexTarget pointers which are never freed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creating and passing around a bunch of pointers of IndexTargets (and of a vector of pointers of IndexTargets) is probably done to avoid the 'costly' copy of container, but we are really not in a timecritical operation here and move semantics will help us even further in the future. On the other hand we never do a proper cleanup of these pointers, which is very dirty, even if structures aren't that big… The changes will effecting many items only effect our own hidden class, so we can do that without fearing breaking interfaces or anything. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index f038f12f5..44d66a725 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -93,7 +93,7 @@ debReleaseIndex::~debReleaseIndex() { template void foreachTarget(std::string const URI, std::string const Dist, std::map > const &ArchEntries, - CallC Call) + CallC &Call) { bool const flatArchive = (Dist[Dist.length() - 1] == '/'); std::string baseURI = URI; @@ -206,7 +206,7 @@ void foreachTarget(std::string const URI, std::string const Dist, struct ComputeIndexTargetsClass { - vector * const IndexTargets; + vector IndexTargets; void operator()(std::string MetaKey, std::string ShortDesc, std::string LongDesc, bool const IsOptional, std::map Options) @@ -217,7 +217,7 @@ struct ComputeIndexTargetsClass ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second); LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second); } - IndexTarget * Target = new IndexTarget( + IndexTarget Target( MetaKey, ShortDesc, LongDesc, @@ -225,13 +225,11 @@ struct ComputeIndexTargetsClass IsOptional, Options ); - IndexTargets->push_back(Target); + IndexTargets.push_back(Target); } - - ComputeIndexTargetsClass() : IndexTargets(new vector ) {} }; -vector * debReleaseIndex::ComputeIndexTargets() const +std::vector debReleaseIndex::ComputeIndexTargets() const { ComputeIndexTargetsClass comp; foreachTarget(URI, Dist, ArchEntries, comp); @@ -249,7 +247,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const iR->SetTrusted(false); // special case for --print-uris - vector const * const targets = ComputeIndexTargets(); + std::vector const targets = ComputeIndexTargets(); #define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), @@ -257,7 +255,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const #undef APT_TARGET if (GetAll) { - for (vector ::const_iterator Target = targets->begin(); Target != targets->end(); ++Target) + for (std::vector::const_iterator Target = targets.begin(); Target != targets.end(); ++Target) new pkgAcqIndex(Owner, TransactionManager, *Target); } -- cgit v1.2.3 From 261727f05ca7ff7d9d6d9b8927cb5c6ad293ebce Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Jun 2015 23:53:36 +0200 Subject: rename Calculate- to GetIndexTargets and use it as official API We need a general way to get from a sources.list entry to IndexTargets and with this change we can move from pkgSourceList over the list of metaIndexes it includes to the IndexTargets each metaIndex can have. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 44d66a725..e1f86d20f 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -229,7 +229,7 @@ struct ComputeIndexTargetsClass } }; -std::vector debReleaseIndex::ComputeIndexTargets() const +std::vector debReleaseIndex::GetIndexTargets() const { ComputeIndexTargetsClass comp; foreachTarget(URI, Dist, ArchEntries, comp); @@ -247,7 +247,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const iR->SetTrusted(false); // special case for --print-uris - std::vector const targets = ComputeIndexTargets(); + std::vector const targets = GetIndexTargets(); #define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), @@ -396,6 +396,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type map::const_iterator const trusted = Options.find("trusted"); + debReleaseIndex *Deb = NULL; for (vector::const_iterator I = List.begin(); I != List.end(); ++I) { @@ -403,34 +404,23 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type if (strcmp((*I)->GetType(), "deb") != 0) continue; - debReleaseIndex *Deb = (debReleaseIndex *) (*I); - if (trusted != Options.end()) - Deb->SetTrusted(StringToBool(trusted->second, false)); - /* This check insures that there will be only one Release file queued for all the Packages files and Sources files it corresponds to. */ - if (Deb->GetURI() == URI && Deb->GetDist() == Dist) + if ((*I)->GetURI() == URI && (*I)->GetDist() == Dist) { - if (IsSrc == true) - Deb->PushSectionEntry("source", new debReleaseIndex::debSectionEntry(Section, IsSrc)); - else - { - if (Dist[Dist.size() - 1] == '/') - Deb->PushSectionEntry("any", new debReleaseIndex::debSectionEntry(Section, IsSrc)); - else - Deb->PushSectionEntry(Archs, new debReleaseIndex::debSectionEntry(Section, IsSrc)); - } - return true; + Deb = dynamic_cast(*I); + if (Deb != NULL) + break; } } // No currently created Release file indexes this entry, so we create a new one. - debReleaseIndex *Deb; - if (trusted != Options.end()) - Deb = new debReleaseIndex(URI, Dist, StringToBool(trusted->second, false)); - else + if (Deb == NULL) + { Deb = new debReleaseIndex(URI, Dist); + List.push_back(Deb); + } if (IsSrc == true) Deb->PushSectionEntry ("source", new debReleaseIndex::debSectionEntry(Section, IsSrc)); @@ -441,7 +431,10 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type else Deb->PushSectionEntry (Archs, new debReleaseIndex::debSectionEntry(Section, IsSrc)); } - List.push_back(Deb); + + if (trusted != Options.end()) + Deb->SetTrusted(StringToBool(trusted->second, false)); + return true; } }; -- cgit v1.2.3 From 1da3b7b8e15b642135b54684e70a0c271471f07a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 11 Jun 2015 10:56:31 +0200 Subject: show URI.Path in all acquire item descriptions It is a rather strange sight that index items use SiteOnly which strips the Path, while e.g. deb files are downloaded with NoUserPassword which does not. Important to note here is that for the file transport Path is pretty important as there is no Host which would be displayed by Site, which always resulted in "interesting" unspecific errors for "file:". Adding a 'middle' ground between the two which does show the Path but potentially modifies it (it strips a pending / at the end if existing) solves this "file:" issue, syncs the output and in the end helps to identify which file is meant exactly in progress output and co as a single site can have multiple repositories in different paths. --- apt-pkg/deb/debmetaindex.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index e1f86d20f..d4e340be0 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -25,7 +25,7 @@ using namespace std; string debReleaseIndex::MetaIndexInfo(const char *Type) const { - string Info = ::URI::SiteOnly(URI) + ' '; + string Info = ::URI::ArchiveOnly(URI) + ' '; if (Dist[Dist.size() - 1] == '/') { if (Dist != "/") @@ -105,7 +105,7 @@ void foreachTarget(std::string const URI, std::string const Dist, else baseURI += "dists/" + Dist + "/"; std::string const Release = (Dist == "/") ? "" : Dist; - std::string const Site = ::URI::SiteOnly(URI); + std::string const Site = ::URI::ArchiveOnly(URI); std::vector lang = APT::Configuration::getLanguages(true); if (lang.empty()) lang.push_back("none"); -- cgit v1.2.3 From e3c1cfc767f17f5e9b2cd99f2658db3d6ac8edd9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 11 Jun 2015 11:38:04 +0200 Subject: use IndexTarget to get to IndexFile Removes a bunch of duplicated code in the deb-specific parts. Especially the Description part is now handled centrally by IndexTarget instead of being duplicated to the derivations of IndexFile. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 56 ++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 42 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index d4e340be0..9108a5097 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -140,6 +140,7 @@ void foreachTarget(std::string const URI, std::string const Dist, Options.insert(std::make_pair("LANGUAGE", *l)); Options.insert(std::make_pair("ARCHITECTURE", "source")); Options.insert(std::make_pair("BASE_URI", baseURI)); + Options.insert(std::make_pair("REPO_URI", URI)); Options.insert(std::make_pair("CREATED_BY", *T)); Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); @@ -186,6 +187,7 @@ void foreachTarget(std::string const URI, std::string const Dist, Options.insert(std::make_pair("LANGUAGE", *l)); Options.insert(std::make_pair("ARCHITECTURE", a->first)); Options.insert(std::make_pair("BASE_URI", baseURI)); + Options.insert(std::make_pair("REPO_URI", URI)); Options.insert(std::make_pair("CREATED_BY", *T)); Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); @@ -294,55 +296,25 @@ bool debReleaseIndex::IsTrusted() const return FileExists(VerifiedSigFile); } -struct GetIndexFilesClass +std::vector *debReleaseIndex::GetIndexFiles() { - vector * const Indexes; - std::string const URI; - std::string const Release; - bool const IsTrusted; + if (Indexes != NULL) + return Indexes; - void operator()(std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/, - bool const /*IsOptional*/, std::map Options) + Indexes = new std::vector(); + std::vector const Targets = GetIndexTargets(); + bool const istrusted = IsTrusted(); + for (std::vector::const_iterator T = Targets.begin(); T != Targets.end(); ++T) { - std::string const TargetName = Options.find("CREATED_BY")->second; + std::string const TargetName = T->Options.find("CREATED_BY")->second; if (TargetName == "Packages") - { - Indexes->push_back(new debPackagesIndex( - URI, - Release, - Options.find("COMPONENT")->second, - IsTrusted, - Options.find("ARCHITECTURE")->second - )); - } + Indexes->push_back(new debPackagesIndex(*T, istrusted)); else if (TargetName == "Sources") - Indexes->push_back(new debSourcesIndex( - URI, - Release, - Options.find("COMPONENT")->second, - IsTrusted - )); + Indexes->push_back(new debSourcesIndex(*T, istrusted)); else if (TargetName == "Translations") - Indexes->push_back(new debTranslationsIndex( - URI, - Release, - Options.find("COMPONENT")->second, - Options.find("LANGUAGE")->second - )); + Indexes->push_back(new debTranslationsIndex(*T)); } - - GetIndexFilesClass(std::string const &URI, std::string const &Release, bool const IsTrusted) : - Indexes(new vector ), URI(URI), Release(Release), IsTrusted(IsTrusted) {} -}; - -std::vector *debReleaseIndex::GetIndexFiles() -{ - if (Indexes != NULL) - return Indexes; - - GetIndexFilesClass comp(URI, Dist, IsTrusted()); - foreachTarget(URI, Dist, ArchEntries, comp); - return Indexes = comp.Indexes; + return Indexes; } void debReleaseIndex::PushSectionEntry(vector const &Archs, const debSectionEntry *Entry) { -- cgit v1.2.3 From 001c76fe204e17916a6c8b351ff30b67d32cb779 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 11 Jun 2015 11:59:16 +0200 Subject: use an enum instead of strings as IndexTarget::Option interface Strings are easy to typo and we can keep the extensibility we require here with a simple enum we can append to without endangering ABI. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 9108a5097..c05e7cdae 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -306,7 +306,7 @@ std::vector *debReleaseIndex::GetIndexFiles() bool const istrusted = IsTrusted(); for (std::vector::const_iterator T = Targets.begin(); T != Targets.end(); ++T) { - std::string const TargetName = T->Options.find("CREATED_BY")->second; + std::string const TargetName = T->Option(IndexTarget::CREATED_BY); if (TargetName == "Packages") Indexes->push_back(new debPackagesIndex(*T, istrusted)); else if (TargetName == "Sources") -- cgit v1.2.3 From 8881b11eacd735148d087c8c0f53827cb537b582 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 11 Jun 2015 16:40:45 +0200 Subject: implement 'apt-get files' to access index targets Downloading additional files is only half the job. We still need a way to allow external tools to know where the files are they requested for download given that we don't want them to choose their own location. 'apt-get files' is our answer to this showing by default in a deb822 format information about each IndexTarget with the potential to filter the records based on lines and an option to change the output format. The command serves also as an example on how to get to this information via libapt. --- apt-pkg/deb/debmetaindex.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index c05e7cdae..8e4c2be2d 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -136,11 +136,14 @@ void foreachTarget(std::string const URI, std::string const Dist, std::map Options; Options.insert(std::make_pair("SITE", Site)); Options.insert(std::make_pair("RELEASE", Release)); - Options.insert(std::make_pair("COMPONENT", (*I)->Section)); - Options.insert(std::make_pair("LANGUAGE", *l)); + if (MetaKey.find("$(COMPONENT)") != std::string::npos) + Options.insert(std::make_pair("COMPONENT", (*I)->Section)); + if (MetaKey.find("$(LANGUAGE)") != std::string::npos) + Options.insert(std::make_pair("LANGUAGE", *l)); Options.insert(std::make_pair("ARCHITECTURE", "source")); Options.insert(std::make_pair("BASE_URI", baseURI)); Options.insert(std::make_pair("REPO_URI", URI)); + Options.insert(std::make_pair("TARGET_OF", "deb-src")); Options.insert(std::make_pair("CREATED_BY", *T)); Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); @@ -183,11 +186,15 @@ void foreachTarget(std::string const URI, std::string const Dist, std::map Options; Options.insert(std::make_pair("SITE", Site)); Options.insert(std::make_pair("RELEASE", Release)); - Options.insert(std::make_pair("COMPONENT", (*I)->Section)); - Options.insert(std::make_pair("LANGUAGE", *l)); - Options.insert(std::make_pair("ARCHITECTURE", a->first)); + if (MetaKey.find("$(COMPONENT)") != std::string::npos) + Options.insert(std::make_pair("COMPONENT", (*I)->Section)); + if (MetaKey.find("$(LANGUAGE)") != std::string::npos) + Options.insert(std::make_pair("LANGUAGE", *l)); + if (MetaKey.find("$(ARCHITECTURE)") != std::string::npos) + Options.insert(std::make_pair("ARCHITECTURE", a->first)); Options.insert(std::make_pair("BASE_URI", baseURI)); Options.insert(std::make_pair("REPO_URI", URI)); + Options.insert(std::make_pair("TARGET_OF", "deb")); Options.insert(std::make_pair("CREATED_BY", *T)); Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); -- cgit v1.2.3 From b07aeb1a6e24825e534167a737043441e871de9f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 12 Jun 2015 02:08:53 +0200 Subject: store Release files data in the Cache We used to read the Release file for each Packages file and store the data in the PackageFile struct even through potentially many Packages (and Translation-*) files could use the same data. The point of the exercise isn't the duplicated data through. Having the Release files as first-class citizens in the Cache allows us to properly track their state as well as allows us to use the information also for files which aren't in the cache, but where we know to which Release file they belong (Sources are an example for this). This modifies the pkgCache structs, especially the PackagesFile struct which depending on how libapt users access the data in these structs can mean huge breakage or no visible change. As a single data point: aptitude seems to be fine with this. Even if there is breakage it is trivial to fix in a backportable way while avoiding breakage for everyone would be a huge pain for us. Note that not all PackageFile structs have a corresponding ReleaseFile. In particular the dpkg/status file as well as *.deb files have not. As these have only a Archive property need, the Component property takes over this duty and the ReleaseFile remains zero. This is also the reason why it isn't needed nor particularily recommended to change from PackagesFile to ReleaseFile blindly. Sticking with the earlier is usually the better option. --- apt-pkg/deb/debmetaindex.cc | 124 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 2 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 8e4c2be2d..994b95849 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -10,10 +11,12 @@ #include #include #include -#include #include +#include +#include +#include +#include -#include #include #include #include @@ -21,6 +24,11 @@ #include #include +#include +#include +#include +#include + using namespace std; string debReleaseIndex::MetaIndexInfo(const char *Type) const @@ -37,6 +45,10 @@ string debReleaseIndex::MetaIndexInfo(const char *Type) const Info += Type; return Info; } +std::string debReleaseIndex::Describe() const +{ + return MetaIndexInfo("Release"); +} string debReleaseIndex::MetaIndexFile(const char *Type) const { @@ -339,6 +351,114 @@ debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section, bool const &IsSrc): Section(Section), IsSrc(IsSrc) {} +static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile) +{ + ReleaseFile = That->MetaIndexFile("InRelease"); + bool releaseExists = false; + if (FileExists(ReleaseFile) == true) + releaseExists = true; + else + { + ReleaseFile = That->MetaIndexFile("Release"); + if (FileExists(ReleaseFile)) + releaseExists = true; + } + return releaseExists; +} + +bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/ +{ + std::string ReleaseFile; + bool const releaseExists = ReleaseFileName(this, ReleaseFile); + + ::URI Tmp(URI); + if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false) + return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str()); + + if (releaseExists == false) + return true; + + FileFd Rel; + // Beware: The 'Release' file might be clearsigned in case the + // signature for an 'InRelease' file couldn't be checked + if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false) + return false; + if (_error->PendingError() == true) + return false; + + // Store the IMS information + pkgCache::RlsFileIterator File = Gen.GetCurRlsFile(); + pkgCacheGenerator::Dynamic DynFile(File); + // Rel can't be used as this is potentially a temporary file + struct stat Buf; + if (stat(ReleaseFile.c_str(), &Buf) != 0) + return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str()); + File->Size = Buf.st_size; + File->mtime = Buf.st_mtime; + + pkgTagFile TagFile(&Rel, Rel.Size()); + pkgTagSection Section; + if (_error->PendingError() == true || TagFile.Step(Section) == false) + return false; + + std::string data; + #define APT_INRELEASE(TYPE, TAG, STORE) \ + data = Section.FindS(TAG); \ + if (data.empty() == false) \ + { \ + map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \ + STORE = storage; \ + } + APT_INRELEASE(MIXED, "Suite", File->Archive) + APT_INRELEASE(VERSIONNUMBER, "Version", File->Version) + APT_INRELEASE(MIXED, "Origin", File->Origin) + APT_INRELEASE(MIXED, "Codename", File->Codename) + APT_INRELEASE(MIXED, "Label", File->Label) + #undef APT_INRELEASE + Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic); + Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades); + + return !_error->PendingError(); +} + /*}}}*/ +// ReleaseIndex::FindInCache - Find this index /*{{{*/ +pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const +{ + std::string ReleaseFile; + bool const releaseExists = ReleaseFileName(this, ReleaseFile); + + pkgCache::RlsFileIterator File = Cache.RlsFileBegin(); + for (; File.end() == false; ++File) + { + if (File->FileName == 0 || ReleaseFile != File.FileName()) + continue; + + // empty means the file does not exist by "design" + if (releaseExists == false && File->Size == 0) + return File; + + struct stat St; + if (stat(File.FileName(),&St) != 0) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl; + return pkgCache::RlsFileIterator(Cache); + } + if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size + << ") or mtime (" << St.st_mtime << " <> " << File->mtime + << ") doesn't match for " << File.FileName() << std::endl; + return pkgCache::RlsFileIterator(Cache); + } + return File; + } + + return File; +} + /*}}}*/ + class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type { protected: -- cgit v1.2.3 From 3fd89e62e985c89b1f9a545ab72c20987b756aff Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 12 Jun 2015 12:06:29 +0200 Subject: implement default apt-get file --release-info mode Selecting targets based on the Release they belong to isn't to unrealistic. In fact, it is assumed to be the most used case so it is made the default especially as this allows to bundle another thing we have to be careful with: Filenames and only showing targets we have acquired. Closes: 752702 --- apt-pkg/deb/debmetaindex.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 994b95849..b328fcea8 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -422,7 +422,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/ } /*}}}*/ // ReleaseIndex::FindInCache - Find this index /*{{{*/ -pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const +pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool const ModifyCheck) const { std::string ReleaseFile; bool const releaseExists = ReleaseFileName(this, ReleaseFile); @@ -434,7 +434,7 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const continue; // empty means the file does not exist by "design" - if (releaseExists == false && File->Size == 0) + if (ModifyCheck == false || (releaseExists == false && File->Size == 0)) return File; struct stat St; -- cgit v1.2.3 From c8a4ce6cbed57ae108dc955d4a850f9b129a0693 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 16 Jun 2015 16:22:46 +0200 Subject: add d-pointer, virtual destructors and de-inline de/constructors To have a chance to keep the ABI for a while we need all three to team up. One of them missing and we might loose, so ensuring that they are available is a very tedious but needed task once in a while. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index b328fcea8..34fc98838 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -459,6 +459,8 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool con } /*}}}*/ +debDebFileMetaIndex::~debDebFileMetaIndex() {} + class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type { protected: -- cgit v1.2.3 From e8afd16892e87a6e2f17c1019ee455f5583387c2 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 17 Jun 2015 00:14:10 +0200 Subject: apply various style suggestions by cppcheck Some of them modify the ABI, but given that we prepare a big one already, these few hardly count for much. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 34fc98838..430a5021b 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -103,7 +103,7 @@ debReleaseIndex::~debReleaseIndex() { } template -void foreachTarget(std::string const URI, std::string const Dist, +void foreachTarget(std::string const &URI, std::string const &Dist, std::map > const &ArchEntries, CallC &Call) { -- cgit v1.2.3 From 6c55f07a5fa3612a5d59c61a17da5fe640eadc8b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 17 Jun 2015 09:29:00 +0200 Subject: make all d-pointer * const pointers Doing this disables the implicit copy assignment operator (among others) which would cause hovac if used on the classes as it would just copy the pointer, not the data the d-pointer points to. For most of the classes we don't need a copy assignment operator anyway and in many classes it was broken before as many contain a pointer of some sort. Only for our Cacheset Container interfaces we define an explicit copy assignment operator which could later be implemented to copy the data from one d-pointer to the other if we need it. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 430a5021b..026af077f 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -86,11 +86,11 @@ std::string debReleaseIndex::LocalFileName() const } debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) : - metaIndex(URI, Dist, "deb"), Trusted(CHECK_TRUST) + metaIndex(URI, Dist, "deb"), d(NULL), Trusted(CHECK_TRUST) {} debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist, bool const Trusted) : - metaIndex(URI, Dist, "deb") { + metaIndex(URI, Dist, "deb"), d(NULL) { SetTrusted(Trusted); } @@ -541,7 +541,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type }; debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) - : metaIndex(DebFile, "local-uri", "deb-dist"), DebFile(DebFile) + : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) { DebIndex = new debDebPkgFileIndex(DebFile); Indexes = new vector(); -- cgit v1.2.3 From 3d8232bf97ce11818fb07813a71136484ea1a44a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 18 Jun 2015 17:33:15 +0200 Subject: fix memory leaks reported by -fsanitize Various small leaks here and there. Nothing particularily big, but still good to fix. Found by the sanitizers while running our testcases. Reported-By: gcc -fsanitize Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 026af077f..5a517b290 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -270,7 +270,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const // special case for --print-uris std::vector const targets = GetIndexTargets(); #define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) - pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner, + pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), targets, iR); #undef APT_TARGET -- cgit v1.2.3 From 463c8d801595ce5ac94d7c032264820be7434232 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 21 Jun 2015 16:47:53 +0200 Subject: support lang= and target= sources.list options We support arch= for a while, now we finally add lang= as well and as a first simple way of controlling which targets to acquire also target=. This asked for a redesign of the internal API of parsing and storing information about 'deb' and 'deb-src' lines. As this API isn't visible to the outside no damage done through. Beside being a nice cleanup (= it actually does more in less lines) it also provides us with a predictable order of architectures as provides in the configuration rather than based on string sorting-order, so that now the native architecture is parsed/displayed first. Observeable e.g. in apt-get output. --- apt-pkg/deb/debmetaindex.cc | 430 +++++++++++++++++++------------------------- 1 file changed, 186 insertions(+), 244 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 5a517b290..f690a8d64 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -29,11 +29,25 @@ #include #include -using namespace std; - -string debReleaseIndex::MetaIndexInfo(const char *Type) const +class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ +{ + public: + struct APT_HIDDEN debSectionEntry + { + std::string Name; + std::vector Targets; + std::vector Architectures; + std::vector Languages; + }; + + std::vector DebEntries; + std::vector DebSrcEntries; +}; + /*}}}*/ +// ReleaseIndex::MetaIndex* - display helpers /*{{{*/ +std::string debReleaseIndex::MetaIndexInfo(const char *Type) const { - string Info = ::URI::ArchiveOnly(URI) + ' '; + std::string Info = ::URI::ArchiveOnly(URI) + ' '; if (Dist[Dist.size() - 1] == '/') { if (Dist != "/") @@ -50,15 +64,15 @@ std::string debReleaseIndex::Describe() const return MetaIndexInfo("Release"); } -string debReleaseIndex::MetaIndexFile(const char *Type) const +std::string debReleaseIndex::MetaIndexFile(const char *Type) const { return _config->FindDir("Dir::State::lists") + URItoFileName(MetaIndexURI(Type)); } -string debReleaseIndex::MetaIndexURI(const char *Type) const +std::string debReleaseIndex::MetaIndexURI(const char *Type) const { - string Res; + std::string Res; if (Dist == "/") Res = URI; @@ -70,8 +84,8 @@ string debReleaseIndex::MetaIndexURI(const char *Type) const Res += Type; return Res; } - -std::string debReleaseIndex::LocalFileName() const + /*}}}*/ +std::string debReleaseIndex::LocalFileName() const /*{{{*/ { // see if we have a InRelease file std::string PathInRelease = MetaIndexFile("InRelease"); @@ -84,28 +98,24 @@ std::string debReleaseIndex::LocalFileName() const return ""; } - -debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) : - metaIndex(URI, Dist, "deb"), d(NULL), Trusted(CHECK_TRUST) + /*}}}*/ +// ReleaseIndex Con- and Destructors /*{{{*/ +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()), Trusted(CHECK_TRUST) {} - -debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist, bool const Trusted) : - metaIndex(URI, Dist, "deb"), d(NULL) { +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) { SetTrusted(Trusted); } - debReleaseIndex::~debReleaseIndex() { - for (map >::const_iterator A = ArchEntries.begin(); - A != ArchEntries.end(); ++A) - for (vector::const_iterator S = A->second.begin(); - S != A->second.end(); ++S) - delete *S; + if (d != NULL) + delete d; } - -template -void foreachTarget(std::string const &URI, std::string const &Dist, - std::map > const &ArchEntries, - CallC &Call) + /*}}}*/ +// ReleaseIndex::GetIndexTargets /*{{{*/ +static void GetIndexTargetsFor(char const * const Type, std::string const &URI, std::string const &Dist, + std::vector const &entries, + std::vector &IndexTargets) { bool const flatArchive = (Dist[Dist.length() - 1] == '/'); std::string baseURI = URI; @@ -118,148 +128,101 @@ void foreachTarget(std::string const &URI, std::string const &Dist, baseURI += "dists/" + Dist + "/"; std::string const Release = (Dist == "/") ? "" : Dist; std::string const Site = ::URI::ArchiveOnly(URI); - std::vector lang = APT::Configuration::getLanguages(true); - if (lang.empty()) - lang.push_back("none"); - map >::const_iterator const src = ArchEntries.find("source"); - if (src != ArchEntries.end()) + + for (std::vector::const_iterator E = entries.begin(); E != entries.end(); ++E) { - std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true); - for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) + for (std::vector::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T) { -#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X)) - std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); - std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); - std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); +#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::" + (X)) + std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); + std::string const tplShortDesc = APT_T_CONFIG("ShortDescription"); + std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true); #undef APT_T_CONFIG - if (MetaKey.empty()) + if (tplMetaKey.empty()) continue; - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); - I != SectionEntries.end(); ++I) + for (std::vector::const_iterator L = E->Languages.begin(); L != E->Languages.end(); ++L) { - for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) + if (*L == "none" && tplMetaKey.find("$(LANGUAGE)") != std::string::npos) + continue; + + for (std::vector::const_iterator A = E->Architectures.begin(); A != E->Architectures.end(); ++A) { - if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos) - continue; std::map Options; Options.insert(std::make_pair("SITE", Site)); Options.insert(std::make_pair("RELEASE", Release)); - if (MetaKey.find("$(COMPONENT)") != std::string::npos) - Options.insert(std::make_pair("COMPONENT", (*I)->Section)); - if (MetaKey.find("$(LANGUAGE)") != std::string::npos) - Options.insert(std::make_pair("LANGUAGE", *l)); - Options.insert(std::make_pair("ARCHITECTURE", "source")); + if (tplMetaKey.find("$(COMPONENT)") != std::string::npos) + Options.insert(std::make_pair("COMPONENT", E->Name)); + if (tplMetaKey.find("$(LANGUAGE)") != std::string::npos) + Options.insert(std::make_pair("LANGUAGE", *L)); + if (tplMetaKey.find("$(ARCHITECTURE)") != std::string::npos) + Options.insert(std::make_pair("ARCHITECTURE", *A)); Options.insert(std::make_pair("BASE_URI", baseURI)); Options.insert(std::make_pair("REPO_URI", URI)); Options.insert(std::make_pair("TARGET_OF", "deb-src")); Options.insert(std::make_pair("CREATED_BY", *T)); - Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); - if (MetaKey.find("$(LANGUAGE)") == std::string::npos) + std::string MetaKey = tplMetaKey; + std::string ShortDesc = tplShortDesc; + std::string LongDesc = tplLongDesc; + for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O) + { + MetaKey = SubstVar(MetaKey, std::string("$(") + O->first + ")", O->second); + ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second); + LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second); + } + IndexTarget Target( + MetaKey, + ShortDesc, + LongDesc, + Options.find("BASE_URI")->second + MetaKey, + IsOptional, + Options + ); + IndexTargets.push_back(Target); + + if (tplMetaKey.find("$(ARCHITECTURE)") == std::string::npos) break; - } - - if (MetaKey.find("$(COMPONENT)") == std::string::npos) - break; - } - } - } - - std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true); - for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) - { -#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X)) - std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); - std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); - std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); - bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true); -#undef APT_T_CONFIG - if (MetaKey.empty()) - continue; - - for (map >::const_iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) - { - if (a->first == "source") - continue; - - for (vector ::const_iterator I = a->second.begin(); - I != a->second.end(); ++I) { - - for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) - { - if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos) - continue; - - std::map Options; - Options.insert(std::make_pair("SITE", Site)); - Options.insert(std::make_pair("RELEASE", Release)); - if (MetaKey.find("$(COMPONENT)") != std::string::npos) - Options.insert(std::make_pair("COMPONENT", (*I)->Section)); - if (MetaKey.find("$(LANGUAGE)") != std::string::npos) - Options.insert(std::make_pair("LANGUAGE", *l)); - if (MetaKey.find("$(ARCHITECTURE)") != std::string::npos) - Options.insert(std::make_pair("ARCHITECTURE", a->first)); - Options.insert(std::make_pair("BASE_URI", baseURI)); - Options.insert(std::make_pair("REPO_URI", URI)); - Options.insert(std::make_pair("TARGET_OF", "deb")); - Options.insert(std::make_pair("CREATED_BY", *T)); - Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); - if (MetaKey.find("$(LANGUAGE)") == std::string::npos) - break; } - if (MetaKey.find("$(COMPONENT)") == std::string::npos) + if (tplMetaKey.find("$(LANGUAGE)") == std::string::npos) break; + } - if (MetaKey.find("$(ARCHITECTURE)") == std::string::npos) - break; } } } - - -struct ComputeIndexTargetsClass -{ - vector IndexTargets; - - void operator()(std::string MetaKey, std::string ShortDesc, std::string LongDesc, - bool const IsOptional, std::map Options) - { - for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O) - { - MetaKey = SubstVar(MetaKey, std::string("$(") + O->first + ")", O->second); - ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second); - LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second); - } - IndexTarget Target( - MetaKey, - ShortDesc, - LongDesc, - Options.find("BASE_URI")->second + MetaKey, - IsOptional, - Options - ); - IndexTargets.push_back(Target); - } -}; - std::vector debReleaseIndex::GetIndexTargets() const { - ComputeIndexTargetsClass comp; - foreachTarget(URI, Dist, ArchEntries, comp); - return comp.IndexTargets; + std::vector IndexTargets; + GetIndexTargetsFor("deb-src", URI, Dist, d->DebSrcEntries, IndexTargets); + GetIndexTargetsFor("deb", URI, Dist, d->DebEntries, IndexTargets); + return IndexTargets; } + /*}}}*/ +void debReleaseIndex::AddComponent(bool const isSrc, std::string const &Name,/*{{{*/ + std::vector const &Targets, + std::vector const &Architectures, + std::vector Languages) +{ + if (Languages.empty() == true) + Languages.push_back("none"); + debReleaseIndexPrivate::debSectionEntry const entry = { + Name, Targets, Architectures, Languages + }; + if (isSrc) + d->DebSrcEntries.push_back(entry); + else + d->DebEntries.push_back(entry); +} + /*}}}*/ - /*}}}*/ -bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const +bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{{{*/ { indexRecords * const iR = new indexRecords(Dist); if (Trusted == ALWAYS_TRUSTED) @@ -282,7 +245,8 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const return true; } - + /*}}}*/ +// ReleaseIndex::*Trusted setters and checkers /*{{{*/ void debReleaseIndex::SetTrusted(bool const Trusted) { if (Trusted == true) @@ -290,7 +254,6 @@ void debReleaseIndex::SetTrusted(bool const Trusted) else this->Trusted = NEVER_TRUSTED; } - bool debReleaseIndex::IsTrusted() const { if (Trusted == ALWAYS_TRUSTED) @@ -303,19 +266,13 @@ bool debReleaseIndex::IsTrusted() const if(URI.substr(0,strlen("cdrom:")) == "cdrom:") return true; - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(MetaIndexURI("Release")) + ".gpg"; - - if (FileExists(VerifiedSigFile)) + if (FileExists(MetaIndexFile("Release.gpg"))) return true; - VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(MetaIndexURI("InRelease")); - - return FileExists(VerifiedSigFile); + return FileExists(MetaIndexFile("InRelease")); } - -std::vector *debReleaseIndex::GetIndexFiles() + /*}}}*/ +std::vector *debReleaseIndex::GetIndexFiles() /*{{{*/ { if (Indexes != NULL) return Indexes; @@ -335,23 +292,9 @@ std::vector *debReleaseIndex::GetIndexFiles() } return Indexes; } + /*}}}*/ -void debReleaseIndex::PushSectionEntry(vector const &Archs, const debSectionEntry *Entry) { - for (vector::const_iterator a = Archs.begin(); - a != Archs.end(); ++a) - ArchEntries[*a].push_back(new debSectionEntry(Entry->Section, Entry->IsSrc)); - delete Entry; -} - -void debReleaseIndex::PushSectionEntry(string const &Arch, const debSectionEntry *Entry) { - ArchEntries[Arch].push_back(Entry); -} - -debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section, - bool const &IsSrc): Section(Section), IsSrc(IsSrc) -{} - -static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile) +static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)/*{{{*/ { ReleaseFile = That->MetaIndexFile("InRelease"); bool releaseExists = false; @@ -365,7 +308,7 @@ static bool ReleaseFileName(debReleaseIndex const * const That, std::string &Rel } return releaseExists; } - + /*}}}*/ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/ { std::string ReleaseFile; @@ -459,46 +402,46 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool con } /*}}}*/ -debDebFileMetaIndex::~debDebFileMetaIndex() {} - -class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type +static std::vector parsePlusMinusOptions(std::string const &Name, /*{{{*/ + std::map const &Options, std::vector const &defaultValues) { - protected: + std::map::const_iterator val = Options.find(Name); + std::vector Values; + if (val != Options.end()) + Values = VectorizeString(val->second, ','); + else + Values = defaultValues; - bool CreateItemInternal(vector &List, string const &URI, - string const &Dist, string const &Section, - bool const &IsSrc, map const &Options) const + if ((val = Options.find(Name + "+")) != Options.end()) { - // parse arch=, arch+= and arch-= settings - map::const_iterator arch = Options.find("arch"); - vector Archs; - if (arch != Options.end()) - Archs = VectorizeString(arch->second, ','); - else - Archs = APT::Configuration::getArchitectures(); - - if ((arch = Options.find("arch+")) != Options.end()) - { - std::vector const plusArch = VectorizeString(arch->second, ','); - for (std::vector::const_iterator plus = plusArch.begin(); plus != plusArch.end(); ++plus) - if (std::find(Archs.begin(), Archs.end(), *plus) == Archs.end()) - Archs.push_back(*plus); - } - if ((arch = Options.find("arch-")) != Options.end()) + std::vector const plusArch = VectorizeString(val->second, ','); + for (std::vector::const_iterator plus = plusArch.begin(); plus != plusArch.end(); ++plus) + if (std::find(Values.begin(), Values.end(), *plus) == Values.end()) + Values.push_back(*plus); + } + if ((val = Options.find(Name + "-")) != Options.end()) + { + std::vector const minusArch = VectorizeString(val->second, ','); + for (std::vector::const_iterator minus = minusArch.begin(); minus != minusArch.end(); ++minus) { - std::vector const minusArch = VectorizeString(arch->second, ','); - for (std::vector::const_iterator minus = minusArch.begin(); minus != minusArch.end(); ++minus) - { - std::vector::iterator kill = std::find(Archs.begin(), Archs.end(), *minus); - if (kill != Archs.end()) - Archs.erase(kill); - } + std::vector::iterator kill = std::find(Values.begin(), Values.end(), *minus); + if (kill != Values.end()) + Values.erase(kill); } + } + return Values; +} + /*}}}*/ +class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ +{ + protected: - map::const_iterator const trusted = Options.find("trusted"); - + bool CreateItemInternal(std::vector &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + bool const &IsSrc, std::map const &Options) const + { debReleaseIndex *Deb = NULL; - for (vector::const_iterator I = List.begin(); + for (std::vector::const_iterator I = List.begin(); I != List.end(); ++I) { // We only worry about debian entries here @@ -523,87 +466,86 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type List.push_back(Deb); } - if (IsSrc == true) - Deb->PushSectionEntry ("source", new debReleaseIndex::debSectionEntry(Section, IsSrc)); - else - { - if (Dist[Dist.size() - 1] == '/') - Deb->PushSectionEntry ("any", new debReleaseIndex::debSectionEntry(Section, IsSrc)); - else - Deb->PushSectionEntry (Archs, new debReleaseIndex::debSectionEntry(Section, IsSrc)); - } + Deb->AddComponent( + IsSrc, + Section, + parsePlusMinusOptions("target", Options, _config->FindVector(std::string("APT::Acquire::Targets::") + Name, "", true)), + parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()), + parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)) + ); + std::map::const_iterator const trusted = Options.find("trusted"); if (trusted != Options.end()) Deb->SetTrusted(StringToBool(trusted->second, false)); return true; } -}; - -debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) - : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) -{ - DebIndex = new debDebPkgFileIndex(DebFile); - Indexes = new vector(); - Indexes->push_back(DebIndex); -} - -class APT_HIDDEN debSLTypeDeb : public debSLTypeDebian + debSLTypeDebian(char const * const Name, char const * const Label) : Type(Name, Label) + { + } +}; + /*}}}*/ +class APT_HIDDEN debSLTypeDeb : public debSLTypeDebian /*{{{*/ { public: - bool CreateItem(vector &List, string const &URI, - string const &Dist, string const &Section, - std::map const &Options) const + bool CreateItem(std::vector &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + std::map const &Options) const { return CreateItemInternal(List, URI, Dist, Section, false, Options); } - debSLTypeDeb() + debSLTypeDeb() : debSLTypeDebian("deb", "Debian binary tree") { - Name = "deb"; - Label = "Standard Debian binary tree"; - } + } }; - -class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian + /*}}}*/ +class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ { public: - bool CreateItem(vector &List, string const &URI, - string const &Dist, string const &Section, - std::map const &Options) const + bool CreateItem(std::vector &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + std::map const &Options) const { return CreateItemInternal(List, URI, Dist, Section, true, Options); } - - debSLTypeDebSrc() + + debSLTypeDebSrc() : debSLTypeDebian("deb-src", "Debian source tree") { - Name = "deb-src"; - Label = "Standard Debian source tree"; - } + } }; + /*}}}*/ -class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type +debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/ + : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) +{ + DebIndex = new debDebPkgFileIndex(DebFile); + Indexes = new std::vector(); + Indexes->push_back(DebIndex); +} +debDebFileMetaIndex::~debDebFileMetaIndex() {} + /*}}}*/ +class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ { public: - bool CreateItem(vector &List, string const &URI, - string const &/*Dist*/, string const &/*Section*/, - std::map const &/*Options*/) const + bool CreateItem(std::vector &List, std::string const &URI, + std::string const &/*Dist*/, std::string const &/*Section*/, + std::map const &/*Options*/) const { metaIndex *mi = new debDebFileMetaIndex(URI); List.push_back(mi); return true; } - - debSLTypeDebFile() + + debSLTypeDebFile() : Type("deb-file", "Debian local deb file") { - Name = "deb-file"; - Label = "Debian Deb File"; - } + } }; + /*}}}*/ APT_HIDDEN debSLTypeDeb _apt_DebType; APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; -- cgit v1.2.3 From 268ffcebb9ae4278b1e3c3f89f8167f229164dbd Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 22 Jun 2015 12:34:11 +0200 Subject: detect and error out on conflicting Trusted settings A specific trust state can be enforced via a sources.list option, but it effects all entries handled by the same Release file, not just the entry it was given on so we enforce acknowledgement of this by requiring the same value to be (not) set on all such entries. --- apt-pkg/deb/debmetaindex.cc | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index f690a8d64..1f725ba05 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -29,6 +29,8 @@ #include #include +#include + class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ { public: @@ -42,6 +44,11 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ std::vector DebEntries; std::vector DebSrcEntries; + + debReleaseIndex::TriState Trusted; + + debReleaseIndexPrivate() : Trusted(debReleaseIndex::TRI_UNSET) {} + debReleaseIndexPrivate(bool const pTrusted) : Trusted(pTrusted ? debReleaseIndex::TRI_YES : debReleaseIndex::TRI_NO) {} }; /*}}}*/ // ReleaseIndex::MetaIndex* - display helpers /*{{{*/ @@ -101,12 +108,11 @@ std::string debReleaseIndex::LocalFileName() const /*{{{*/ /*}}}*/ // ReleaseIndex Con- and Destructors /*{{{*/ debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist) : - metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()), Trusted(CHECK_TRUST) + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) {} debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted) : - metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) { - SetTrusted(Trusted); -} + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate(Trusted)) +{} debReleaseIndex::~debReleaseIndex() { if (d != NULL) delete d; @@ -225,9 +231,9 @@ void debReleaseIndex::AddComponent(bool const isSrc, std::string const &Name,/*{ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{{{*/ { indexRecords * const iR = new indexRecords(Dist); - if (Trusted == ALWAYS_TRUSTED) + if (d->Trusted == TRI_YES) iR->SetTrusted(true); - else if (Trusted == NEVER_TRUSTED) + else if (d->Trusted == TRI_NO) iR->SetTrusted(false); // special case for --print-uris @@ -246,19 +252,21 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{ return true; } /*}}}*/ -// ReleaseIndex::*Trusted setters and checkers /*{{{*/ -void debReleaseIndex::SetTrusted(bool const Trusted) +// ReleaseIndex::IsTrusted /*{{{*/ +bool debReleaseIndex::SetTrusted(TriState const Trusted) { - if (Trusted == true) - this->Trusted = ALWAYS_TRUSTED; - else - this->Trusted = NEVER_TRUSTED; + if (d->Trusted == TRI_UNSET) + d->Trusted = Trusted; + else if (d->Trusted != Trusted) + // TRANSLATOR: The first is an option name from sources.list manpage, the other two URI and Suite + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Trusted", URI.c_str(), Dist.c_str()); + return true; } bool debReleaseIndex::IsTrusted() const { - if (Trusted == ALWAYS_TRUSTED) + if (d->Trusted == TRI_YES) return true; - else if (Trusted == NEVER_TRUSTED) + else if (d->Trusted == TRI_NO) return false; @@ -476,7 +484,12 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ std::map::const_iterator const trusted = Options.find("trusted"); if (trusted != Options.end()) - Deb->SetTrusted(StringToBool(trusted->second, false)); + { + if (Deb->SetTrusted(StringToBool(trusted->second, false) ? debReleaseIndex::TRI_YES : debReleaseIndex::TRI_NO) == false) + return false; + } + else if (Deb->SetTrusted(debReleaseIndex::TRI_DONTCARE) == false) + return false; return true; } -- cgit v1.2.3 From 5ad0096a4e19e191b59634e8a8817995ec4045ad Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 23 Jun 2015 15:16:08 +0200 Subject: merge indexRecords into metaIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit indexRecords was used to parse the Release file – mostly the hashes – while metaIndex deals with downloading the Release file, storing all indexes coming from this release and … parsing the Release file, but this time mostly for the other fields. That wasn't a problem in metaIndex as this was done in the type specific subclass, but indexRecords while allowing to override the parsing method did expect by default a specific format. APT isn't really supporting different types at the moment, but this is a violation of the abstraction we have everywhere else and, which is the actual reason for this merge: Options e.g. coming from the sources.list come to metaIndex naturally, which needs to wrap them up and bring them into indexRecords, so the acquire system is told about it as they don't get to see the metaIndex, but they don't really belong in indexRecords as this is just for storing data loaded from the Release file… the result is a complete mess. I am not saying it is a lot prettier after the merge, but at least adding new options is now slightly easier and there is just one place responsible for parsing the Release file. That can't hurt. --- apt-pkg/deb/debmetaindex.cc | 231 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 195 insertions(+), 36 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 1f725ba05..f0b859eb4 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -45,10 +44,7 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ std::vector DebEntries; std::vector DebSrcEntries; - debReleaseIndex::TriState Trusted; - - debReleaseIndexPrivate() : Trusted(debReleaseIndex::TRI_UNSET) {} - debReleaseIndexPrivate(bool const pTrusted) : Trusted(pTrusted ? debReleaseIndex::TRI_YES : debReleaseIndex::TRI_NO) {} + debReleaseIndexPrivate() {} }; /*}}}*/ // ReleaseIndex::MetaIndex* - display helpers /*{{{*/ @@ -92,27 +88,15 @@ std::string debReleaseIndex::MetaIndexURI(const char *Type) const return Res; } /*}}}*/ -std::string debReleaseIndex::LocalFileName() const /*{{{*/ -{ - // see if we have a InRelease file - std::string PathInRelease = MetaIndexFile("InRelease"); - if (FileExists(PathInRelease)) - return PathInRelease; - - // and if not return the normal one - if (FileExists(PathInRelease)) - return MetaIndexFile("Release"); - - return ""; -} - /*}}}*/ // ReleaseIndex Con- and Destructors /*{{{*/ debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist) : metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) {} -debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted) : - metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate(Trusted)) -{} +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const pTrusted) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) +{ + Trusted = pTrusted ? TRI_YES : TRI_NO; +} debReleaseIndex::~debReleaseIndex() { if (d != NULL) delete d; @@ -227,22 +211,197 @@ void debReleaseIndex::AddComponent(bool const isSrc, std::string const &Name,/*{ } /*}}}*/ +bool debReleaseIndex::Load(std::string const &Filename, std::string * const ErrorText)/*{{{*/ +{ + LoadedSuccessfully = TRI_NO; + FileFd Fd; + if (OpenMaybeClearSignedFile(Filename, Fd) == false) + return false; + + pkgTagFile TagFile(&Fd, Fd.Size()); + if (_error->PendingError() == true) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Unable to parse Release file %s"),Filename.c_str()); + return false; + } + + pkgTagSection Section; + const char *Start, *End; + if (TagFile.Step(Section) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("No sections in Release file %s"), Filename.c_str()); + return false; + } + // FIXME: find better tag name + SupportsAcquireByHash = Section.FindB("Acquire-By-Hash", false); + + Suite = Section.FindS("Suite"); + Codename = Section.FindS("Codename"); + + bool FoundHashSum = false; + for (int i=0;HashString::SupportedHashes()[i] != NULL; i++) + { + if (!Section.Find(HashString::SupportedHashes()[i], Start, End)) + continue; + + std::string Name; + std::string Hash; + unsigned long long Size; + while (Start < End) + { + if (!parseSumData(Start, End, Name, Hash, Size)) + return false; + + if (Entries.find(Name) == Entries.end()) + { + metaIndex::checkSum *Sum = new metaIndex::checkSum; + Sum->MetaKeyFilename = Name; + Sum->Size = Size; + Sum->Hashes.FileSize(Size); + APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);) + Entries[Name] = Sum; + } + Entries[Name]->Hashes.push_back(HashString(HashString::SupportedHashes()[i],Hash)); + FoundHashSum = true; + } + } + + if(FoundHashSum == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("No Hash entry in Release file %s"), Filename.c_str()); + return false; + } + + std::string const StrDate = Section.FindS("Date"); + if (RFC1123StrToTime(StrDate.c_str(), Date) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid 'Date' entry in Release file %s"), Filename.c_str()); + return false; + } -bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{{{*/ + std::string const Label = Section.FindS("Label"); + std::string const StrValidUntil = Section.FindS("Valid-Until"); + + // if we have a Valid-Until header in the Release file, use it as default + if (StrValidUntil.empty() == false) + { + if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); + return false; + } + } + // get the user settings for this archive and use what expires earlier + int MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); + if (Label.empty() == false) + MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); + int MinAge = _config->FindI("Acquire::Min-ValidTime", 0); + if (Label.empty() == false) + MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); + + LoadedSuccessfully = TRI_YES; + if(MaxAge == 0 && + (MinAge == 0 || ValidUntil == 0)) // No user settings, use the one from the Release file + return true; + + if (MinAge != 0 && ValidUntil != 0) { + time_t const min_date = Date + MinAge; + if (ValidUntil < min_date) + ValidUntil = min_date; + } + if (MaxAge != 0) { + time_t const max_date = Date + MaxAge; + if (ValidUntil == 0 || ValidUntil > max_date) + ValidUntil = max_date; + } + + return true; +} + /*}}}*/ +metaIndex * debReleaseIndex::UnloadedClone() const /*{{{*/ +{ + if (Trusted == TRI_NO) + return new debReleaseIndex(URI, Dist, false); + else if (Trusted == TRI_YES) + return new debReleaseIndex(URI, Dist, true); + else + return new debReleaseIndex(URI, Dist); +} + /*}}}*/ +bool debReleaseIndex::parseSumData(const char *&Start, const char *End, /*{{{*/ + std::string &Name, std::string &Hash, unsigned long long &Size) { - indexRecords * const iR = new indexRecords(Dist); - if (d->Trusted == TRI_YES) - iR->SetTrusted(true); - else if (d->Trusted == TRI_NO) - iR->SetTrusted(false); + Name = ""; + Hash = ""; + Size = 0; + /* Skip over the first blank */ + while ((*Start == '\t' || *Start == ' ' || *Start == '\n' || *Start == '\r') + && Start < End) + Start++; + if (Start >= End) + return false; - // special case for --print-uris + /* Move EntryEnd to the end of the first entry (the hash) */ + const char *EntryEnd = Start; + while ((*EntryEnd != '\t' && *EntryEnd != ' ') + && EntryEnd < End) + EntryEnd++; + if (EntryEnd == End) + return false; + + Hash.append(Start, EntryEnd-Start); + + /* Skip over intermediate blanks */ + Start = EntryEnd; + while (*Start == '\t' || *Start == ' ') + Start++; + if (Start >= End) + return false; + + EntryEnd = Start; + /* Find the end of the second entry (the size) */ + while ((*EntryEnd != '\t' && *EntryEnd != ' ' ) + && EntryEnd < End) + EntryEnd++; + if (EntryEnd == End) + return false; + + Size = strtoull (Start, NULL, 10); + + /* Skip over intermediate blanks */ + Start = EntryEnd; + while (*Start == '\t' || *Start == ' ') + Start++; + if (Start >= End) + return false; + + EntryEnd = Start; + /* Find the end of the third entry (the filename) */ + while ((*EntryEnd != '\t' && *EntryEnd != ' ' && + *EntryEnd != '\n' && *EntryEnd != '\r') + && EntryEnd < End) + EntryEnd++; + + Name.append(Start, EntryEnd-Start); + Start = EntryEnd; //prepare for the next round + return true; +} + /*}}}*/ + +bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ +{ std::vector const targets = GetIndexTargets(); #define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), - targets, iR); + targets, this); #undef APT_TARGET + // special case for --print-uris if (GetAll) { for (std::vector::const_iterator Target = targets.begin(); Target != targets.end(); ++Target) @@ -253,20 +412,20 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{ } /*}}}*/ // ReleaseIndex::IsTrusted /*{{{*/ -bool debReleaseIndex::SetTrusted(TriState const Trusted) +bool debReleaseIndex::SetTrusted(TriState const pTrusted) { - if (d->Trusted == TRI_UNSET) - d->Trusted = Trusted; - else if (d->Trusted != Trusted) + if (Trusted == TRI_UNSET) + Trusted = pTrusted; + else if (Trusted != pTrusted) // TRANSLATOR: The first is an option name from sources.list manpage, the other two URI and Suite return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Trusted", URI.c_str(), Dist.c_str()); return true; } bool debReleaseIndex::IsTrusted() const { - if (d->Trusted == TRI_YES) + if (Trusted == TRI_YES) return true; - else if (d->Trusted == TRI_NO) + else if (Trusted == TRI_NO) return false; -- cgit v1.2.3 From 0741daeb7ab870b4dd62a93fa12a1cf6330f9a72 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 23 Jun 2015 17:26:57 +0200 Subject: add sources.list Check-Valid-Until and Valid-Until-{Max,Min} options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These options could be set via configuration before, but the connection to the actual sources is so strong that they should really be set in the sources.list instead – especially as this can be done a lot more specific rather than e.g. disabling Valid-Until for all sources at once. Valid-Until-* names are chosen instead of the Min/Max-ValidTime as this seems like a better name and their use in the wild is probably low enough that this isn't going to confuse anyone if we have to names for the same thing in different areas. In the longrun, the config options should be removed, but for now documentation hinting at the new options is good enough as these are the kind of options you set once across many systems with different apt versions, so the new way should work everywhere first before we deprecate the old way. --- apt-pkg/deb/debmetaindex.cc | 134 +++++++++++++++++++++++++++++++------------- 1 file changed, 95 insertions(+), 39 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index f0b859eb4..5d7e539c7 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -44,7 +44,11 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ std::vector DebEntries; std::vector DebSrcEntries; - debReleaseIndexPrivate() {} + metaIndex::TriState CheckValidUntil; + time_t ValidUntilMin; + time_t ValidUntilMax; + + debReleaseIndexPrivate() : CheckValidUntil(metaIndex::TRI_UNSET), ValidUntilMin(0), ValidUntilMax(0) {} }; /*}}}*/ // ReleaseIndex::MetaIndex* - display helpers /*{{{*/ @@ -283,43 +287,56 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro return false; } - std::string const Label = Section.FindS("Label"); - std::string const StrValidUntil = Section.FindS("Valid-Until"); + bool CheckValidUntil = _config->FindB("Acquire::Check-Valid-Until", true); + if (d->CheckValidUntil == metaIndex::TRI_NO) + CheckValidUntil = false; + else if (d->CheckValidUntil == metaIndex::TRI_YES) + CheckValidUntil = true; - // if we have a Valid-Until header in the Release file, use it as default - if (StrValidUntil.empty() == false) + if (CheckValidUntil == true) { - if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) + std::string const Label = Section.FindS("Label"); + std::string const StrValidUntil = Section.FindS("Valid-Until"); + + // if we have a Valid-Until header in the Release file, use it as default + if (StrValidUntil.empty() == false) { - if (ErrorText != NULL) - strprintf(*ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); - return false; + if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); + return false; + } + } + // get the user settings for this archive and use what expires earlier + time_t MaxAge = d->ValidUntilMax; + if (MaxAge == 0) + { + MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); + if (Label.empty() == false) + MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); + } + time_t MinAge = d->ValidUntilMin; + if (MinAge == 0) + { + MinAge = _config->FindI("Acquire::Min-ValidTime", 0); + if (Label.empty() == false) + MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); } - } - // get the user settings for this archive and use what expires earlier - int MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); - if (Label.empty() == false) - MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); - int MinAge = _config->FindI("Acquire::Min-ValidTime", 0); - if (Label.empty() == false) - MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); - - LoadedSuccessfully = TRI_YES; - if(MaxAge == 0 && - (MinAge == 0 || ValidUntil == 0)) // No user settings, use the one from the Release file - return true; - if (MinAge != 0 && ValidUntil != 0) { - time_t const min_date = Date + MinAge; - if (ValidUntil < min_date) - ValidUntil = min_date; - } - if (MaxAge != 0) { - time_t const max_date = Date + MaxAge; - if (ValidUntil == 0 || ValidUntil > max_date) - ValidUntil = max_date; + if (MinAge != 0 && ValidUntil != 0) { + time_t const min_date = Date + MinAge; + if (ValidUntil < min_date) + ValidUntil = min_date; + } + if (MaxAge != 0) { + time_t const max_date = Date + MaxAge; + if (ValidUntil == 0 || ValidUntil > max_date) + ValidUntil = max_date; + } } + LoadedSuccessfully = TRI_YES; return true; } /*}}}*/ @@ -411,7 +428,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ return true; } /*}}}*/ -// ReleaseIndex::IsTrusted /*{{{*/ +// ReleaseIndex::Set* TriState options /*{{{*/ bool debReleaseIndex::SetTrusted(TriState const pTrusted) { if (Trusted == TRI_UNSET) @@ -421,6 +438,32 @@ bool debReleaseIndex::SetTrusted(TriState const pTrusted) return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Trusted", URI.c_str(), Dist.c_str()); return true; } +bool debReleaseIndex::SetCheckValidUntil(TriState const pCheckValidUntil) +{ + if (d->CheckValidUntil == TRI_UNSET) + d->CheckValidUntil = pCheckValidUntil; + else if (d->CheckValidUntil != pCheckValidUntil) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Check-Valid-Until", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetValidUntilMin(time_t const Valid) +{ + if (d->ValidUntilMin == 0) + d->ValidUntilMin = Valid; + else if (d->ValidUntilMin != Valid) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Min-ValidTime", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetValidUntilMax(time_t const Valid) +{ + if (d->ValidUntilMax == 0) + d->ValidUntilMax = Valid; + else if (d->ValidUntilMax != Valid) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Max-ValidTime", URI.c_str(), Dist.c_str()); + return true; +} + /*}}}*/ +// ReleaseIndex::IsTrusted /*{{{*/ bool debReleaseIndex::IsTrusted() const { if (Trusted == TRI_YES) @@ -601,6 +644,22 @@ static std::vector parsePlusMinusOptions(std::string const &Name, / /*}}}*/ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ { + metaIndex::TriState GetTriStateOption(std::mapconst &Options, char const * const name) const + { + std::map::const_iterator const opt = Options.find(name); + if (opt != Options.end()) + return StringToBool(opt->second, false) ? metaIndex::TRI_YES : metaIndex::TRI_NO; + return metaIndex::TRI_DONTCARE; + } + + time_t GetTimeOption(std::mapconst &Options, char const * const name) const + { + std::map::const_iterator const opt = Options.find(name); + if (opt == Options.end()) + return 0; + return strtoull(opt->second.c_str(), NULL, 10); + } + protected: bool CreateItemInternal(std::vector &List, std::string const &URI, @@ -641,13 +700,10 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)) ); - std::map::const_iterator const trusted = Options.find("trusted"); - if (trusted != Options.end()) - { - if (Deb->SetTrusted(StringToBool(trusted->second, false) ? debReleaseIndex::TRI_YES : debReleaseIndex::TRI_NO) == false) - return false; - } - else if (Deb->SetTrusted(debReleaseIndex::TRI_DONTCARE) == false) + if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false || + Deb->SetCheckValidUntil(GetTriStateOption(Options, "check-valid-until")) == false || + Deb->SetValidUntilMax(GetTimeOption(Options, "valid-until-max")) == false || + Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false) return false; return true; -- cgit v1.2.3 From b0d408547734100bf86781615f546487ecf390d9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 24 Jun 2015 19:31:22 +0200 Subject: implement Signed-By option for sources.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limits which key(s) can be used to sign a repository. Not immensely useful from a security perspective all by itself, but if the user has additional measures in place to confine a repository (like pinning) an attacker who gets the key for such a repository is limited to its potential and can't use the key to sign its attacks for an other (maybe less limited) repository… (yes, this is as weak as it sounds, but having the capability might come in handy for implementing other stuff later). --- apt-pkg/deb/debmetaindex.cc | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 5d7e539c7..4bb03a942 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -461,6 +461,29 @@ bool debReleaseIndex::SetValidUntilMax(time_t const Valid) else if (d->ValidUntilMax != Valid) return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Max-ValidTime", URI.c_str(), Dist.c_str()); return true; +} +bool debReleaseIndex::SetSignedBy(std::string const &pSignedBy) +{ + if (SignedBy.empty() == true && pSignedBy.empty() == false) + { + if (pSignedBy[0] == '/') // no check for existence as we could be chrooting later or such things + ; // absolute path to a keyring file + else + { + // we could go all fancy and allow short/long/string matches as gpgv/apt-key does, + // but fingerprints are harder to fake than the others and this option is set once, + // not interactively all the time so easy to type is not really a concern. + std::string finger = pSignedBy; + finger.erase(std::remove(finger.begin(), finger.end(), ' '), finger.end()); + std::transform(finger.begin(), finger.end(), finger.begin(), ::toupper); + if (finger.length() != 40 || finger.find_first_not_of("0123456789ABCDEF") != std::string::npos) + return _error->Error(_("Invalid value set for option %s concerning source %s %s (%s)"), "Signed-By", URI.c_str(), Dist.c_str(), "not a fingerprint"); + } + SignedBy = pSignedBy; + } + else if (SignedBy != pSignedBy) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Signed-By", URI.c_str(), Dist.c_str()); + return true; } /*}}}*/ // ReleaseIndex::IsTrusted /*{{{*/ @@ -706,6 +729,18 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false) return false; + std::map::const_iterator const signedby = Options.find("signed-by"); + if (signedby == Options.end()) + { + if (Deb->SetSignedBy("") == false) + return false; + } + else + { + if (Deb->SetSignedBy(signedby->second) == false) + return false; + } + return true; } -- cgit v1.2.3 From 653ef26c70dc9c0e2cbfdd4e79117876bb63e87d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 30 Jun 2015 10:53:51 +0200 Subject: allow individual targets to be kept compressed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is an option to keep all targets (Packages, Sources, …) compressed for a while now, but the all-or-nothing approach is a bit limited for our purposes with additional targets as some of them are very big (Contents) and rarely used in comparison, so keeping them compressed by default can make sense, while others are still unpacked. Most interesting is the copy-change maybe: Copy is used by the acquire system as an uncompressor and it is hence expected that it returns the hashes for the "output", not the input. Now, in the case of keeping a file compressed, the output is never written to disk, but generated in memory and we should still validated it, so for compressed files copy is expected to return the hashes of the uncompressed file. We used to use the config option to enable on-the-fly decompress in the method, but in reality copy is never used in a way where it shouldn't decompress a compressed file to get its hashes, so we can save us the trouble of sending this information to the method and just do it always. --- apt-pkg/deb/debmetaindex.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 4bb03a942..10ab0f844 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -123,6 +123,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, std::string const Release = (Dist == "/") ? "" : Dist; std::string const Site = ::URI::ArchiveOnly(URI); + bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false); for (std::vector::const_iterator E = entries.begin(); E != entries.end(); ++E) { for (std::vector::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T) @@ -131,7 +132,8 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); std::string const tplShortDesc = APT_T_CONFIG("ShortDescription"); std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); - bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true); + bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::Optional", true); + bool const KeepCompressed = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::KeepCompressed", GzipIndex); #undef APT_T_CONFIG if (tplMetaKey.empty()) continue; @@ -173,6 +175,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, LongDesc, Options.find("BASE_URI")->second + MetaKey, IsOptional, + KeepCompressed, Options ); IndexTargets.push_back(Target); @@ -413,7 +416,7 @@ bool debReleaseIndex::parseSumData(const char *&Start, const char *End, /*{{{*/ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ { std::vector const targets = GetIndexTargets(); -#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) +#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, false, std::map()) pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), targets, this); -- cgit v1.2.3 From 3b3028467ceccca0b73a8f53051c0fa4de313111 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 9 Jul 2015 00:35:40 +0200 Subject: add c++11 override marker to overridden methods C++11 adds the 'override' specifier to mark that a method is overriding a base class method and error out if not. We hide it in the APT_OVERRIDE macro to ensure that we keep compiling in pre-c++11 standards. Reported-By: clang-modernize -add-override -override-macros Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 10ab0f844..46a7181fc 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -758,7 +758,7 @@ class APT_HIDDEN debSLTypeDeb : public debSLTypeDebian /*{{{*/ bool CreateItem(std::vector &List, std::string const &URI, std::string const &Dist, std::string const &Section, - std::map const &Options) const + std::map const &Options) const APT_OVERRIDE { return CreateItemInternal(List, URI, Dist, Section, false, Options); } @@ -774,7 +774,7 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ bool CreateItem(std::vector &List, std::string const &URI, std::string const &Dist, std::string const &Section, - std::map const &Options) const + std::map const &Options) const APT_OVERRIDE { return CreateItemInternal(List, URI, Dist, Section, true, Options); } @@ -800,7 +800,7 @@ class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ bool CreateItem(std::vector &List, std::string const &URI, std::string const &/*Dist*/, std::string const &/*Section*/, - std::map const &/*Options*/) const + std::map const &/*Options*/) const APT_OVERRIDE { metaIndex *mi = new debDebFileMetaIndex(URI); List.push_back(mi); -- cgit v1.2.3 From c2a4a8dded2dfb56dbcab9689b6cb4b96c9999b6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 10 Jul 2015 00:07:37 +0200 Subject: rename 'apt-get files' to 'apt-get indextargets' 'files' is a bit too generic as a name for a command usually only used programmatically (if at all) by developers, so instead of "wasting" this generic name for this we use "indextargets" which is actually the name of the datastructure the displayed data is stored in. Along with this rename the config options are renamed accordingly. --- apt-pkg/deb/debmetaindex.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 46a7181fc..480317db3 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -128,12 +128,12 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, { for (std::vector::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T) { -#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::" + (X)) +#define APT_T_CONFIG(X) _config->Find(std::string("Acquire::IndexTargets::") + Type + "::" + *T + "::" + (X)) std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); std::string const tplShortDesc = APT_T_CONFIG("ShortDescription"); - std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); - bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::Optional", true); - bool const KeepCompressed = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::KeepCompressed", GzipIndex); + std::string const tplLongDesc = "$(SITE) " + APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); + bool const IsOptional = _config->FindB(std::string("Acquire::IndexTargets::") + Type + "::" + *T + "::Optional", true); + bool const KeepCompressed = _config->FindB(std::string("Acquire::IndexTargets::") + Type + "::" + *T + "::KeepCompressed", GzipIndex); #undef APT_T_CONFIG if (tplMetaKey.empty()) continue; @@ -721,7 +721,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ Deb->AddComponent( IsSrc, Section, - parsePlusMinusOptions("target", Options, _config->FindVector(std::string("APT::Acquire::Targets::") + Name, "", true)), + parsePlusMinusOptions("target", Options, _config->FindVector(std::string("Acquire::IndexTargets::") + Name, "", true)), parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()), parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)) ); -- cgit v1.2.3 From 5465192b9aeb1ccea778950ccf2d1b7b32f2cd91 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Jul 2015 18:03:54 +0200 Subject: add volatile sources support in libapt-pkg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sources are usually defined in sources.list (and co) and are pretty stable, but once in a while a frontend might want to add an additional "source" like a local .deb file to install this package (No support for 'real' sources being added this way as this is a multistep process). We had a hack in place to allow apt-get and apt to pull this of for a short while now, but other frontends are either left in the cold by this and/or the code for it looks dirty with FIXMEs plastering it and has on top of this also some problems (like including these 'volatile' sources in the srcpkgcache.bin file). So the biggest part in this commit is actually the rewrite of the cache generation as it is now potentially a three step process. The biggest problem with adding support now through is that this makes a bunch of previously mostly unusable by externs and therefore hidden classes public, so a bit of further tuneing on this now public API is in order… --- apt-pkg/deb/debmetaindex.cc | 29 ----------------------------- 1 file changed, 29 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 480317db3..123c91648 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -785,34 +785,5 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ }; /*}}}*/ -debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/ - : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) -{ - DebIndex = new debDebPkgFileIndex(DebFile); - Indexes = new std::vector(); - Indexes->push_back(DebIndex); -} -debDebFileMetaIndex::~debDebFileMetaIndex() {} - /*}}}*/ -class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ -{ - public: - - bool CreateItem(std::vector &List, std::string const &URI, - std::string const &/*Dist*/, std::string const &/*Section*/, - std::map const &/*Options*/) const APT_OVERRIDE - { - metaIndex *mi = new debDebFileMetaIndex(URI); - List.push_back(mi); - return true; - } - - debSLTypeDebFile() : Type("deb-file", "Debian local deb file") - { - } -}; - /*}}}*/ - APT_HIDDEN debSLTypeDeb _apt_DebType; APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; -APT_HIDDEN debSLTypeDebFile _apt_DebFileType; -- cgit v1.2.3 From 88a8975f156e452d9f3ebe76822b236e8962ebba Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 17 Aug 2015 12:01:45 +0200 Subject: Cleanup includes after running iwyu --- apt-pkg/deb/debmetaindex.cc | 3 --- 1 file changed, 3 deletions(-) (limited to 'apt-pkg/deb/debmetaindex.cc') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 123c91648..b2e9eeb00 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -20,12 +20,9 @@ #include #include #include -#include #include -#include #include -#include #include #include -- cgit v1.2.3