configureable acquire targets to download additional files

First pass at making the acquire system capable of downloading files
based on configuration rather than hardcoded entries. It is now possible
to instruct 'deb' and 'deb-src' sources.list lines to download more than
just Packages/Translation-* and Sources files. Details on how to do that
can be found in the included documentation file.

5 files changed, 173 insertions, 187 deletions

diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index 42e35d32a..2500ab631 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -259,7 +259,7 @@ vector<string> Configuration::FindVector(const char *Name) const
    return FindVector(Name, "");
 }
 #endif
-vector<string> Configuration::FindVector(const char *Name, std::string const &Default) const
+vector<string> Configuration::FindVector(const char *Name, std::string const &Default, bool const Keys) const
 {
    vector<string> Vec;
    const Item *Top = Lookup(Name);
@@ -272,7 +272,7 @@ vector<string> Configuration::FindVector(const char *Name, std::string const &De
    Item *I = Top->Child;
    while(I != NULL)
    {
-      Vec.push_back(I->Value);
+      Vec.push_back(Keys ? I->Tag : I->Value);
       I = I->Next;
    }
    if (Vec.empty() == true)
diff --git a/apt-pkg/contrib/configuration.h b/apt-pkg/contrib/configuration.h
index 8d7d51037..eacc26fda 100644
--- a/apt-pkg/contrib/configuration.h
+++ b/apt-pkg/contrib/configuration.h
@@ -84,15 +84,8 @@ class Configuration
     *
     * \param Name of the parent node
     * \param Default list of values separated by commas */
-#if APT_PKG_ABI >= 413
-   std::vector<std::string> FindVector(const char *Name, std::string const &Default = "") const;
-   std::vector<std::string> FindVector(std::string const &Name, std::string const &Default = "") const { return FindVector(Name.c_str(), Default); };
-#else
-   std::vector<std::string> FindVector(const char *Name, std::string const &Default) const;
-   std::vector<std::string> FindVector(std::string const &Name, std::string const &Default) const { return FindVector(Name.c_str(), Default); };
-   std::vector<std::string> FindVector(const char *Name) const;
-   std::vector<std::string> FindVector(std::string const &Name) const { return FindVector(Name.c_str(), ""); };
-#endif
+   std::vector<std::string> FindVector(const char *Name, std::string const &Default = "", bool const Keys = false) const;
+   std::vector<std::string> FindVector(std::string const &Name, std::string const &Default = "", bool const Keys = false) const { return FindVector(Name.c_str(), Default, Keys); };
 
    int FindI(const char *Name,int const &Default = 0) const;
    int FindI(std::string const &Name,int const &Default = 0) const {return FindI(Name.c_str(),Default);};
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index eb5e78e3b..8fef05ab0 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -23,25 +23,6 @@
 
 using namespace std;
 
-string debReleaseIndex::Info(const char *Type, string const &Section, string const &Arch) const
-{
-   string Info = ::URI::SiteOnly(URI) + ' ';
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Info += Dist;
-   }
-   else
-   {
-      Info += Dist + '/' + Section;
-      if (Arch.empty() != true)
-	 Info += " " + Arch;
-   }
-   Info += " ";
-   Info += Type;
-   return Info;
-}
-
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
 {
    string Info = ::URI::SiteOnly(URI) + ' ';
@@ -92,81 +73,6 @@ std::string debReleaseIndex::LocalFileName() const
    return "";
 }
 
-string debReleaseIndex::IndexURISuffix(const char *Type, string const &Section, string const &Arch) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-   {
-      if (Arch == "native")
-	 Res += Section + "/binary-" + _config->Find("APT::Architecture") + '/';
-      else
-	 Res += Section + "/binary-" + Arch + '/';
-   }
-   return Res + Type;
-}
-   
-
-string debReleaseIndex::IndexURI(const char *Type, string const &Section, string const &Arch) const
-{
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      string Res;
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + '/' + IndexURISuffix(Type, Section, Arch);
- }
-
-string debReleaseIndex::SourceIndexURISuffix(const char *Type, const string &Section) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-      Res += Section + "/source/";
-   return Res + Type;
-}
-
-string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section) const
-{
-   string Res;
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section);
-}
-
-string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-      Res += Section + "/i18n/Translation-";
-   return Res + Type;
-}
-
-string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const
-{
-   string Res;
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section);
-}
-
 debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) :
 					metaIndex(URI, Dist, "deb"), Trusted(CHECK_TRUST)
 {}
@@ -184,73 +90,161 @@ debReleaseIndex::~debReleaseIndex() {
 			delete *S;
 }
 
-vector <IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
-	vector <IndexTarget *>* IndexTargets = new vector <IndexTarget *>;
+vector <IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const
+{
+   vector <IndexTarget *>* IndexTargets = new vector <IndexTarget *>;
 
-	map<string, vector<debSectionEntry const*> >::const_iterator const src = ArchEntries.find("source");
-	if (src != ArchEntries.end()) {
-		vector<debSectionEntry const*> const SectionEntries = src->second;
-		for (vector<debSectionEntry const*>::const_iterator I = SectionEntries.begin();
-		     I != SectionEntries.end(); ++I) {
-			char const * const ShortDesc = "Sources";
-			IndexTarget * const Target = new IndexTarget(
-			      SourceIndexURISuffix(ShortDesc, (*I)->Section),
-			      ShortDesc,
-			      Info(ShortDesc, (*I)->Section),
-			      SourceIndexURI(ShortDesc, (*I)->Section)
-			      );
-			IndexTargets->push_back (Target);
-		}
-	}
+   bool const flatArchive = (Dist[Dist.length() - 1] == '/');
+   std::string baseURI = URI;
+   if (flatArchive)
+   {
+      if (Dist != "/")
+         baseURI += Dist;
+   }
+   else
+      baseURI += "dists/" + Dist + "/";
+   std::string const Release = (Dist == "/") ? "" : Dist;
+   std::string const Site = ::URI::SiteOnly(URI);
+   std::vector<std::string> lang = APT::Configuration::getLanguages(true);
+   if (lang.empty())
+      lang.push_back("none");
+   map<string, vector<debSectionEntry const*> >::const_iterator const src = ArchEntries.find("source");
+   if (src != ArchEntries.end())
+   {
+      std::vector<std::string> const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true);
+      for (std::vector<std::string>::const_iterator T = targets.begin(); T != targets.end(); ++T)
+      {
+#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X))
+	 std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+	 std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
+	 std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
+	 bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
+#undef APT_T_CONFIG
+	 if (URI.empty())
+	    continue;
 
-	// Only source release
-	if (IndexTargets->empty() == false && ArchEntries.size() == 1)
-		return IndexTargets;
+	 vector<debSectionEntry const*> const SectionEntries = src->second;
+	 for (vector<debSectionEntry const*>::const_iterator I = SectionEntries.begin();
+	       I != SectionEntries.end(); ++I)
+	 {
+	    for (vector<std::string>::const_iterator l = lang.begin(); l != lang.end(); ++l)
+	    {
+	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+		  continue;
+
+	       struct SubstVar subst[] = {
+		  { "$(SITE)", &Site },
+		  { "$(RELEASE)", &Release },
+		  { "$(COMPONENT)", &((*I)->Section) },
+		  { "$(LANGUAGE)", &(*l) },
+		  { NULL, NULL }
+	       };
+	       std::string const name = SubstVar(URI, subst);
+	       IndexTarget * Target;
+	       if (IsOptional == true)
+	       {
+		  Target = new OptionalIndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       else
+	       {
+		  Target = new IndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       IndexTargets->push_back(Target);
+
+	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+		  break;
+	    }
 
-	std::set<std::string> sections;
-	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector <const debSectionEntry *>::const_iterator I = a->second.begin();
-		     I != a->second.end(); ++I) {
-			char const * const ShortDesc = "Packages";
-			IndexTarget * const Target = new IndexTarget(
-			      IndexURISuffix(ShortDesc, (*I)->Section, a->first),
-			      ShortDesc,
-			      Info (ShortDesc, (*I)->Section, a->first),
-			      IndexURI(ShortDesc, (*I)->Section, a->first)
-			      );
-			IndexTargets->push_back (Target);
-			sections.insert((*I)->Section);
-		}
-	}
+	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	       break;
+	 }
+      }
+   }
 
-	std::vector<std::string> lang = APT::Configuration::getLanguages(true);
-	std::vector<std::string>::iterator lend = std::remove(lang.begin(), lang.end(), "none");
-	if (lend != lang.end())
-		lang.erase(lend);
-
-	if (lang.empty() == true)
-		return IndexTargets;
-
-	// get the Translation-* files, later we will skip download of non-existent if we have an index
-	for (std::set<std::string>::const_iterator s = sections.begin();
-	     s != sections.end(); ++s) {
-		for (std::vector<std::string>::const_iterator l = lang.begin();
-		     l != lang.end(); ++l) {
-			std::string const ShortDesc = "Translation-" + *l;
-			IndexTarget * const Target = new OptionalIndexTarget(
-			      TranslationIndexURISuffix(l->c_str(), *s),
-			      ShortDesc,
-			      Info (ShortDesc.c_str(), *s),
-			      TranslationIndexURI(l->c_str(), *s)
-			      );
-			IndexTargets->push_back(Target);
-		}
-	}
+   // Only source release
+   if (IndexTargets->empty() == false && ArchEntries.size() == 1)
+      return IndexTargets;
 
-	return IndexTargets;
+   std::vector<std::string> const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true);
+   for (std::vector<std::string>::const_iterator T = targets.begin(); T != targets.end(); ++T)
+   {
+#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X))
+      std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+      std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
+      std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
+      bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true);
+#undef APT_T_CONFIG
+      if (URI.empty())
+	 continue;
+
+      for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
+	    a != ArchEntries.end(); ++a)
+      {
+	 if (a->first == "source")
+	    continue;
+
+	 for (vector <const debSectionEntry *>::const_iterator I = a->second.begin();
+	       I != a->second.end(); ++I) {
+
+	    for (vector<std::string>::const_iterator l = lang.begin(); l != lang.end(); ++l)
+	    {
+	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+		  continue;
+
+	       struct SubstVar subst[] = {
+		  { "$(SITE)", &Site },
+		  { "$(RELEASE)", &Release },
+		  { "$(COMPONENT)", &((*I)->Section) },
+		  { "$(LANGUAGE)", &(*l) },
+		  { "$(ARCHITECTURE)", &(a->first) },
+		  { NULL, NULL }
+	       };
+	       std::string const name = SubstVar(URI, subst);
+	       IndexTarget * Target;
+	       if (IsOptional == true)
+	       {
+		  Target = new OptionalIndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       else
+	       {
+		  Target = new IndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       IndexTargets->push_back(Target);
+
+	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+		  break;
+	    }
+
+	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	       break;
+	 }
+
+	 if (URI.find("$(ARCHITECTURE)") == std::string::npos)
+	    break;
+      }
+   }
+
+   return IndexTargets;
 }
 									/*}}}*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
@@ -361,17 +355,6 @@ void debReleaseIndex::PushSectionEntry(string const &Arch, const debSectionEntry
 	ArchEntries[Arch].push_back(Entry);
 }
 
-void debReleaseIndex::PushSectionEntry(const debSectionEntry *Entry) {
-	if (Entry->IsSrc == true)
-		PushSectionEntry("source", Entry);
-	else {
-		for (map<string, vector<const debSectionEntry *> >::iterator a = ArchEntries.begin();
-		     a != ArchEntries.end(); ++a) {
-			a->second.push_back(Entry);
-		}
-	}
-}
-
 debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 94d005760..e1c1d91ef 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -47,7 +47,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::string ArchiveURI(std::string const &File) const {return URI + File;};
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    std::vector <IndexTarget *>* ComputeIndexTargets() const;
-   std::string Info(const char *Type, std::string const &Section, std::string const &Arch="") const;
 
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
@@ -58,12 +57,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 #endif
    std::string LocalFileName() const;
 
-   std::string IndexURI(const char *Type, std::string const &Section, std::string const &Arch="native") const;
-   std::string IndexURISuffix(const char *Type, std::string const &Section, std::string const &Arch="native") const;
-   std::string SourceIndexURI(const char *Type, const std::string &Section) const;
-   std::string SourceIndexURISuffix(const char *Type, const std::string &Section) const;
-   std::string TranslationIndexURI(const char *Type, const std::string &Section) const;
-   std::string TranslationIndexURISuffix(const char *Type, const std::string &Section) const;
    virtual std::vector <pkgIndexFile *> *GetIndexFiles();
 
    void SetTrusted(bool const Trusted);
@@ -71,7 +64,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 
    void PushSectionEntry(std::vector<std::string> const &Archs, const debSectionEntry *Entry);
    void PushSectionEntry(std::string const &Arch, const debSectionEntry *Entry);
-   void PushSectionEntry(const debSectionEntry *Entry);
 };
 
 class APT_HIDDEN debDebFileMetaIndex : public metaIndex
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index f756eab26..50ea2b49e 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -101,8 +101,26 @@ bool pkgInitConfig(Configuration &Cnf)
    // The default user we drop to in the methods
    Cnf.CndSet("APT::Sandbox::User", "_apt");
 
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::URI", "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatURI", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::ShortDescription", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Description", "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatDescription", "$(SITE) $(RELEASE) Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Optional", false);
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::URI", "$(COMPONENT)/i18n/Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatURI", "$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::ShortDescription", "Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatDescription", "$(SITE) $(RELEASE) Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::URI", "$(COMPONENT)/source/Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatURI", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::ShortDescription", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Optional", false);
+
    bool Res = true;
-   
+
    // Read an alternate config file
    const char *Cfg = getenv("APT_CONFIG");
    if (Cfg != 0 && strlen(Cfg) != 0)