From e85b4cd500cc96a8ce0d35c5e63fe274bed5b917 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 12 Aug 2009 00:52:26 +0200 Subject: Add a Acquire::CompressionTypes config variable from there the acquire-items choose which compression file they should (try first to) download to easily add new or change the order of the compression types. And because it is easy now we directly add builtin lzma support. The compression ratio is better than bzip2 but we prefer the later for now as no (official) mirror uses lzma, so this would only generate useless hits on the servers. Maybe sometime in the future lzma will be the default... [apt-pkg/acquire-item.cc] - use configsettings for dynamic compression type use and order. Based on a patch by Jyrki Muukkonen, thanks! (LP: #71746) [apt-pkg/init.cc] - add default configuration for compression types and add lzma support. Order is now bzip2, lzma, gzip, none (Closes: #510526) [ftparchive/writer.cc] - add lzma support also here, patch for this (and inspiration for the one above) by Robert Millan, thanks! --- apt-pkg/acquire-item.cc | 79 ++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 30 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 39ae327cb..ffbe66d7d 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -551,13 +551,24 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, if(comprExt.empty()) { // autoselect the compression method - if(FileExists("/bin/bzip2")) - CompressionExtension = ".bz2"; - else - CompressionExtension = ".gz"; - } else { - CompressionExtension = (comprExt == "plain" ? "" : comprExt); + Configuration::Item const *Opts = _config->Tree("Acquire::CompressionTypes"); + if (Opts != 0) + Opts = Opts->Child; + + const char dirBin[] = "Dir::Bin::"; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Tag.empty() == true || Opts->Value.empty() == true) + continue; + const string bin = _config->FindFile(string(dirBin).append(Opts->Value).c_str(),""); + if (bin != "" && !FileExists(bin)) + continue; + comprExt = '.' + Opts->Tag; + break; + } } + CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt); + Desc.URI = URI + CompressionExtension; Desc.Description = URIDesc; @@ -584,24 +595,32 @@ string pkgAcqIndex::Custom600Headers() /*}}}*/ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - bool descChanged = false; - // no .bz2 found, retry with .gz - if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") { - Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz"; - - new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, - ExpectedHash, string(".gz")); - descChanged = true; - } - // no .gz found, retry with uncompressed - else if(Desc.URI.substr(Desc.URI.size()-2) == "gz") { - Desc.URI = Desc.URI.substr(0,Desc.URI.size()-2); + Configuration::Item const *Opts = _config->Tree("Acquire::CompressionTypes"); + if (Opts != 0) + Opts = Opts->Child; + + const char dirBin[] = "Dir::Bin::"; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Tag.empty() == true || Opts->Value.empty() == true) + continue; + + // jump over all already checked compression types + const unsigned int nameLen = Desc.URI.size() - Opts->Tag.size(); + if(Desc.URI.substr(nameLen) != Opts->Tag || Opts->Next == 0) + continue; + + // check if we need an external binary for this compression type + const string bin = _config->FindFile(string(dirBin).append(Opts->Next->Value).c_str(),""); + if (bin != "" && !FileExists(bin)) + continue; + + // retry with the next extension + Desc.URI = Desc.URI.substr(0, nameLen) + Opts->Next->Tag; + + new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc, + ExpectedHash, string(".").append(Opts->Next->Tag)); - new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, - ExpectedHash, string("plain")); - descChanged = true; - } - if (descChanged) { Status = StatDone; Complete = false; Dequeue(); @@ -698,11 +717,11 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Local = true; string compExt = flExtension(flNotDir(URI(Desc.URI).Path)); - const char *decompProg; - if(compExt == "bz2") - decompProg = "bzip2"; - else if(compExt == "gz") - decompProg = "gzip"; + string decompProg; + + // get the binary name for your used compression type + decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),""); + if(decompProg.empty() == false); // flExtensions returns the full name if no extension is found // this is why we have this complicated compare operation here // FIMXE: add a new flJustExtension() that return "" if no @@ -717,9 +736,9 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Decompression = true; DestFile += ".decomp"; - Desc.URI = string(decompProg) + ":" + FileName; + Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - Mode = decompProg; + Mode = decompProg.c_str(); } /*}}}*/ // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ -- cgit v1.2.3 From e878aedb8b53b311295a2df55ce5e865b1ad92b9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 25 Aug 2009 15:32:40 +0200 Subject: "backport" the APT::Configuration class to apt-sid We can use it to simplify the internal code to operate with Acquire::CompressionTypes group. This also made it possible to set this setting with the -o flag. --- apt-pkg/acquire-item.cc | 62 +++++++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 38 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index ffbe66d7d..94341c81a 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -15,6 +15,7 @@ // Include Files /*{{{*/ #include #include +#include #include #include #include @@ -551,21 +552,11 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, if(comprExt.empty()) { // autoselect the compression method - Configuration::Item const *Opts = _config->Tree("Acquire::CompressionTypes"); - if (Opts != 0) - Opts = Opts->Child; - - const char dirBin[] = "Dir::Bin::"; - for (; Opts != 0; Opts = Opts->Next) - { - if (Opts->Tag.empty() == true || Opts->Value.empty() == true) - continue; - const string bin = _config->FindFile(string(dirBin).append(Opts->Value).c_str(),""); - if (bin != "" && !FileExists(bin)) - continue; - comprExt = '.' + Opts->Tag; - break; - } + std::vector types = APT::Configuration::getCompressionTypes(); + if (types.empty() == true) + comprExt = "plain"; + else + comprExt = "." + types[0]; } CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt); @@ -595,36 +586,31 @@ string pkgAcqIndex::Custom600Headers() /*}}}*/ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - Configuration::Item const *Opts = _config->Tree("Acquire::CompressionTypes"); - if (Opts != 0) - Opts = Opts->Child; + std::vector types = APT::Configuration::getCompressionTypes(); - const char dirBin[] = "Dir::Bin::"; - for (; Opts != 0; Opts = Opts->Next) + for (std::vector::const_iterator t = types.begin(); + t != types.end(); t++) { - if (Opts->Tag.empty() == true || Opts->Value.empty() == true) + // jump over all already tried compression types + const unsigned int nameLen = Desc.URI.size() - (*t).size(); + if(Desc.URI.substr(nameLen) != *t) continue; - // jump over all already checked compression types - const unsigned int nameLen = Desc.URI.size() - Opts->Tag.size(); - if(Desc.URI.substr(nameLen) != Opts->Tag || Opts->Next == 0) - continue; - - // check if we need an external binary for this compression type - const string bin = _config->FindFile(string(dirBin).append(Opts->Next->Value).c_str(),""); - if (bin != "" && !FileExists(bin)) - continue; + // we want to try it with the next extension + t++; - // retry with the next extension - Desc.URI = Desc.URI.substr(0, nameLen) + Opts->Next->Tag; + if (t != types.end()) + { + Desc.URI = Desc.URI.substr(0, nameLen) + *t; - new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc, - ExpectedHash, string(".").append(Opts->Next->Tag)); + new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc, + ExpectedHash, string(".").append(*t)); - Status = StatDone; - Complete = false; - Dequeue(); - return; + Status = StatDone; + Complete = false; + Dequeue(); + return; + } } // on decompression failure, remove bad versions in partial/ -- cgit v1.2.3