implement Fallback-Of for IndexTargets

Sometimes index files are in different locations in a repository as it
is currently the case for Contents files which are per-component in
Debian, but aren't in Ubuntu. This has historic reasons and is perhaps
changed soon, but such cases of transitions can always happen in the
future again, so we should prepare:

Introduced is a new field declaring that the current item should only be
downloaded if the mentioned item wasn't allowing for transitions without
a flagday in clients and archives.

This isn't implemented 'simpler' with multiple MetaKeys as items (could)
change their descriptions and perhaps also other configuration bits with
their location.

4 files changed, 46 insertions, 9 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 959627a01..8c45acddd 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1196,9 +1196,18 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
    // at this point the real Items are loaded in the fetcher
    ExpectedAdditionalItems = 0;
 
+   std::set<std::string> targetsSeen;
    bool const metaBaseSupportsByHash = TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
-   for (auto Target : TransactionManager->MetaIndexParser->GetIndexTargets())
+   for (auto &Target: TransactionManager->MetaIndexParser->GetIndexTargets())
    {
+      // if we have seen a target which is created-by a target this one here is declared a
+      // fallback to, we skip acquiring the fallback (but we make sure we clean up)
+      if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
+      {
+	 targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+	 new CleanupItem(Owner, TransactionManager, Target);
+	 continue;
+      }
       // all is an implementation detail. Users shouldn't use this as arch
       // We need this support trickery here as e.g. Debian has binary-all files already,
       // but arch:all packages are still in the arch:any files, so we would waste precious
@@ -1267,6 +1276,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	       else if (hashes.FileSize() == 0)
 	       {
 		  new CleanupItem(Owner, TransactionManager, Target);
+		  targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 		  continue;
 	       }
 	    }
@@ -1327,6 +1337,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	    std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
 	    if (FileExists(idxfilename))
 	       new NoActionItem(Owner, Target, idxfilename);
+	    targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 	    continue;
 	 }
 
@@ -1348,6 +1359,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
       }
 
       // Queue the Index file (Packages, Sources, Translation-$foo, …)
+      targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
       if (trypdiff)
          new pkgAcqDiffIndex(Owner, TransactionManager, Target);
       else
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 260686cc7..ea32e6235 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -33,13 +33,13 @@ class APT_HIDDEN debReleaseIndexPrivate					/*{{{*/
    public:
    struct APT_HIDDEN debSectionEntry
    {
-      std::string sourcesEntry;
-      std::string Name;
-      std::vector<std::string> Targets;
-      std::vector<std::string> Architectures;
-      std::vector<std::string> Languages;
-      bool UsePDiffs;
-      std::string UseByHash;
+      std::string const sourcesEntry;
+      std::string const Name;
+      std::vector<std::string> const Targets;
+      std::vector<std::string> const Architectures;
+      std::vector<std::string> const Languages;
+      bool const UsePDiffs;
+      std::string const UseByHash;
    };
 
    std::vector<debSectionEntry> DebEntries;
@@ -172,6 +172,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 	 std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
 	 std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 	 std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", "");
+	 std::string const FallbackOf = APT_T_CONFIG_STR("Fallback-Of", "");
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR
 	 if (tplMetaKey.empty())
@@ -292,6 +293,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 		  Options.insert(std::make_pair("REPO_URI", URI));
 		  Options.insert(std::make_pair("TARGET_OF", Type));
 		  Options.insert(std::make_pair("CREATED_BY", *T));
+		  Options.insert(std::make_pair("FALLBACK_OF", FallbackOf));
 		  Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
 		  Options.insert(std::make_pair("BY_HASH", UseByHash));
 		  Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
@@ -616,7 +618,8 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/
    // special case for --print-uris
    if (GetAll)
       for (auto const &Target: GetIndexTargets())
-	 new pkgAcqIndex(Owner, TransactionManager, Target);
+	 if (Target.Option(IndexTarget::FALLBACK_OF).empty())
+	    new pkgAcqIndex(Owner, TransactionManager, Target);
 
    return true;
 }
@@ -949,6 +952,26 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	 else if (optValue == false && tarItr != mytargets.end())
 	    mytargets.erase(std::remove(mytargets.begin(), mytargets.end(), target), mytargets.end());
       }
+      // if we can't order it in a 1000 steps we give up… probably a cycle
+      for (auto i = 0; i < 1000; ++i)
+      {
+	 bool Changed = false;
+	 for (auto t = mytargets.begin(); t != mytargets.end(); ++t)
+	 {
+	    std::string const fallback = _config->Find(std::string("Acquire::IndexTargets::") + Name + "::" + *t + "::Fallback-Of");
+	    if (fallback.empty())
+	       continue;
+	    auto const faller = std::find(mytargets.begin(), mytargets.end(), fallback);
+	    if (faller == mytargets.end() || faller < t)
+	       continue;
+	    Changed = true;
+	    auto const tv = *t;
+	    mytargets.erase(t);
+	    mytargets.emplace_back(tv);
+	 }
+	 if (Changed == false)
+	    break;
+      }
 
       bool UsePDiffs = _config->FindB("Acquire::PDiffs", true);
       {
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index a777c38ac..148ed5d98 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -139,6 +139,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(REPO_URI);
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
+      APT_CASE(FALLBACK_OF);
       APT_CASE(PDIFFS);
       APT_CASE(DEFAULTENABLED);
       APT_CASE(COMPRESSIONTYPES);
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 5c666671c..eaaded87a 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -91,6 +91,7 @@ class IndexTarget							/*{{{*/
       SOURCESENTRY,
       BY_HASH,
       KEEPCOMPRESSEDAS,
+      FALLBACK_OF,
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;