From 124e6916b7b02984803ff8217e8163947aae2882 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 May 2016 12:08:32 +0200 Subject: fix and document on the fly compressor config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libapt allows to configure compressors to be used by its system via configuration implemented in 03bef78461c6f443187b60799402624326843396, but that was never really documented and also only partly working, which also explains why the tests weren't using it… --- apt-pkg/acquire.cc | 2 +- apt-pkg/aptconfiguration.cc | 25 +++++++++++----------- doc/apt-ftparchive.1.xml | 34 ++++++++++++++++-------------- doc/apt.conf.5.xml | 45 ++++++++++++++++++++++++++++++---------- doc/examples/apt-ftparchive.conf | 8 +++---- doc/examples/configure-index | 10 +++++++++ test/libapt/fileutl_test.cc | 34 +++++++++++++++++++----------- 7 files changed, 102 insertions(+), 56 deletions(-) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 7a44d8599..ff41246b8 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -466,7 +466,7 @@ void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet) void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) { RunFdsSane(RSet, WSet); -}; +} /*}}}*/ // Acquire::RunFdsSane - Deal with active FDs /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc index 01ae5a642..32778e273 100644 --- a/apt-pkg/aptconfiguration.cc +++ b/apt-pkg/aptconfiguration.cc @@ -35,6 +35,7 @@ namespace APT { // setDefaultConfigurationForCompressors /*{{{*/ static void setDefaultConfigurationForCompressors() { // Set default application paths to check for optional compression types + _config->CndSet("Dir::Bin::gzip", "/bin/gzip"); _config->CndSet("Dir::Bin::bzip2", "/bin/bzip2"); _config->CndSet("Dir::Bin::xz", "/usr/bin/xz"); _config->CndSet("Dir::Bin::lz4", "/usr/bin/lz4"); @@ -60,6 +61,12 @@ static void setDefaultConfigurationForCompressors() { _config->Set("APT::Compressor::lzma::UncompressArg::", "-d"); } } + // setup the defaults for the compressiontypes => method mapping + _config->CndSet("Acquire::CompressionTypes::xz","xz"); + _config->CndSet("Acquire::CompressionTypes::bz2","bzip2"); + _config->CndSet("Acquire::CompressionTypes::lzma","lzma"); + _config->CndSet("Acquire::CompressionTypes::gz","gzip"); + _config->CndSet("Acquire::CompressionTypes::lz4","lz4"); } /*}}}*/ // getCompressionTypes - Return Vector of usable compressiontypes /*{{{*/ @@ -75,14 +82,6 @@ const Configuration::getCompressionTypes(bool const &Cached) { types.clear(); } - // setup the defaults for the compressiontypes => method mapping - _config->CndSet("Acquire::CompressionTypes::xz","xz"); - _config->CndSet("Acquire::CompressionTypes::bz2","bzip2"); - _config->CndSet("Acquire::CompressionTypes::lzma","lzma"); - _config->CndSet("Acquire::CompressionTypes::gz","gzip"); - _config->CndSet("Acquire::CompressionTypes::lz4","lz4"); - - setDefaultConfigurationForCompressors(); std::vector const compressors = getCompressors(); // load the order setting into our vector @@ -398,12 +397,12 @@ const Configuration::getCompressors(bool const Cached) { compressors.push_back(Compressor("lzma",".lzma","false", NULL, NULL, 400)); #endif - std::vector const comp = _config->FindVector("APT::Compressor"); - for (std::vector::const_iterator c = comp.begin(); - c != comp.end(); ++c) { - if (c->empty() || *c == "." || *c == "gzip" || *c == "bzip2" || *c == "lzma" || *c == "xz") + std::vector const comp = _config->FindVector("APT::Compressor", "", true); + for (auto const &c: comp) + { + if (c.empty() || std::any_of(compressors.begin(), compressors.end(), [&c](Compressor const &ac) { return ac.Name == c; })) continue; - compressors.push_back(Compressor(c->c_str(), std::string(".").append(*c).c_str(), c->c_str(), "-9", "-d", 100)); + compressors.push_back(Compressor(c.c_str(), std::string(".").append(c).c_str(), c.c_str(), nullptr, nullptr, 1000)); } return compressors; diff --git a/doc/apt-ftparchive.1.xml b/doc/apt-ftparchive.1.xml index f478564d3..3dc6a6ceb 100644 --- a/doc/apt-ftparchive.1.xml +++ b/doc/apt-ftparchive.1.xml @@ -88,23 +88,26 @@ The release command generates a Release file from a - directory tree. It recursively searches the given directory for uncompressed - Packages and Sources files and ones - compressed with gzip, bzip2 or lzma - as well as Release and md5sum.txt files by default - (APT::FTPArchive::Release::Default-Patterns). Additional filename patterns - can be added by listing them in APT::FTPArchive::Release::Patterns. - It then writes to stdout a Release file containing (by default) an MD5, + directory tree. It recursively searches the given directory for + uncompressed and compressed Packages, + Sources, Contents, + Components and icons files as + well as Release, Index and + md5sum.txt files by default + (APT::FTPArchive::Release::Default-Patterns). + Additional filename patterns can be added by listing them in + APT::FTPArchive::Release::Patterns. It then writes to + stdout a Release file containing (by default) an MD5, SHA1, SHA256 and SHA512 digest for each file. Values for the additional metadata fields in the Release file are taken from the corresponding variables under APT::FTPArchive::Release, - e.g. APT::FTPArchive::Release::Origin. The supported fields - are: Origin, Label, Suite, + e.g. APT::FTPArchive::Release::Origin. The supported fields + are Origin, Label, Suite, Version, Codename, Date, - Valid-Until, Architectures, - Components, Description. + Valid-Until, Signed-By, Architectures, + Components and Description. @@ -177,10 +180,11 @@ - Sets the default compression schemes to use - for the package index files. It is a string that contains a space - separated list of at least one of: '.' (no compression), 'gzip' and - 'bzip2'. The default for all compression schemes is '. gzip'. + Sets the default compression schemes to use + for the package index files. It is a string that contains a space + separated list of at least one of the compressors configured via the + configuration scope. + The default for all compression schemes is '. gzip'. diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index 48ad9c026..215634e99 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -173,6 +173,29 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; + + + This scope defines which compression formats are supported, how compression + and decompression can be performed if support for this format isn't built + into apt directly and a cost-value indicating how costly it is to compress + something in this format. As an example the following configuration stanza + would allow apt to download and uncompress as well as create and store + files with the low-cost .reversed file extension which + it will pass to the command rev without additional + commandline parameters for compression and uncompression: + +APT::Compressor::rev { + Name "rev"; + Extension ".reversed"; + Binary "rev"; + CompressArg {}; + UncompressArg {}; + Cost "10"; +}; + + + + List of all build profiles enabled for build-dependency resolution, @@ -529,25 +552,25 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; List of compression types which are understood by the acquire methods. Files like Packages can be available in various compression formats. - By default the acquire methods can decompress bzip2, lzma - and gzip compressed files; with this setting more formats can be added - on the fly or the used method can be changed. The syntax for this is: + By default the acquire methods can decompress and recompress many common formats like xz and + gzip; with this scope the supported formats can be queried, modified + as well as support for more formats added (see also ). The syntax for this is: Acquire::CompressionTypes::FileExtension "Methodname"; Also, the Order subgroup can be used to define in which order the acquire system will try to download the compressed files. The acquire system will try the first and proceed with the next compression type in this list on error, so to prefer one over the other type - simply add the preferred type first - default types not already added will be implicitly appended + simply add the preferred type first - types not already added will be implicitly appended to the end of the list, so e.g. Acquire::CompressionTypes::Order:: "gz"; can - be used to prefer gzip compressed files over bzip2 and lzma. - If lzma should be preferred over gzip and bzip2 the - configure setting should look like this: Acquire::CompressionTypes::Order { "lzma"; "gz"; }; + be used to prefer gzip compressed files over all other compression formats. + If xz should be preferred over gzip and bzip2 the + configure setting should look like this: Acquire::CompressionTypes::Order { "xz"; "gz"; }; It is not needed to add bz2 to the list explicitly as it will be added automatically. Note that the Dir::Bin::Methodname - will be checked at run time. If this option has been set, the - method will only be used if this file exists; e.g. for the - bzip2 method (the inbuilt) setting is: - Dir::Bin::bzip2 "/bin/bzip2"; + will be checked at run time. If this option has been set and support for + this format isn't directly built into apt, the method will only be used if + this file exists; e.g. for the bzip2 method (the + inbuilt) setting is: Dir::Bin::bzip2 "/bin/bzip2"; Note also that list entries specified on the command line will be added at the end of the list specified in the configuration files, but before the default entries. To prefer a type in this case over the ones specified in the configuration files you can set the option direct - not in list style. diff --git a/doc/examples/apt-ftparchive.conf b/doc/examples/apt-ftparchive.conf index 0d245c0af..0e8bcb2ce 100644 --- a/doc/examples/apt-ftparchive.conf +++ b/doc/examples/apt-ftparchive.conf @@ -7,11 +7,11 @@ Dir { CacheDir "."; }; -// Create Packages, Packages.gz and Packages.bz2, remove what you don't need +// Create Packages, Packages.gz and Packages.xz, remove/add as needed Default { - Packages::Compress ". gzip bzip2"; - Sources::Compress ". gzip bzip2"; - Contents::Compress ". gzip bzip2"; + Packages::Compress ". gzip xz"; + Sources::Compress ". gzip xz"; + Contents::Compress ". gzip xz"; }; // Includes the main section. You can structure the directory tree under diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 8e8110d94..dc579c394 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -88,6 +88,16 @@ APT Post-Invoke {"touch /var/lib/apt/post-update-stamp"; }; }; + // define a new supported compressor on the fly + APT::Compressor::rev { + Name "rev"; + Extension ".reversed"; + Binary "rev"; + CompressArg {}; + UncompressArg {}; + Cost "10"; + }; + Authentication { TrustCDROM "false"; // consider the CD-ROM always trusted diff --git a/test/libapt/fileutl_test.cc b/test/libapt/fileutl_test.cc index 607c4a195..8cd5132ce 100644 --- a/test/libapt/fileutl_test.cc +++ b/test/libapt/fileutl_test.cc @@ -4,7 +4,9 @@ #include #include #include +#include +#include #include #include #include @@ -162,26 +164,34 @@ static void TestFileFd(mode_t const a_umask, mode_t const ExpectedFilePermission static void TestFileFd(unsigned int const filemode) { - std::vector compressors = APT::Configuration::getCompressors(); - - // testing the (un)compress via pipe, as the 'real' compressors are usually built in via libraries - compressors.push_back(APT::Configuration::Compressor("rev", ".reversed", "rev", NULL, NULL, 42)); - //compressors.push_back(APT::Configuration::Compressor("cat", ".ident", "cat", NULL, NULL, 42)); - - for (std::vector::const_iterator c = compressors.begin(); c != compressors.end(); ++c) + auto const compressors = APT::Configuration::getCompressors(); + EXPECT_EQ(7, compressors.size()); + bool atLeastOneWasTested = false; + for (auto const &c: compressors) { if ((filemode & FileFd::ReadWrite) == FileFd::ReadWrite && - (c->Name.empty() != true && c->Binary.empty() != true)) + (c.Name.empty() != true && c.Binary.empty() != true)) continue; - TestFileFd(0002, 0664, filemode, *c); - TestFileFd(0022, 0644, filemode, *c); - TestFileFd(0077, 0600, filemode, *c); - TestFileFd(0026, 0640, filemode, *c); + atLeastOneWasTested = true; + TestFileFd(0002, 0664, filemode, c); + TestFileFd(0022, 0644, filemode, c); + TestFileFd(0077, 0600, filemode, c); + TestFileFd(0026, 0640, filemode, c); } + EXPECT_TRUE(atLeastOneWasTested); } TEST(FileUtlTest, FileFD) { + // testing the (un)compress via pipe, as the 'real' compressors are usually built in via libraries + _config->Set("APT::Compressor::rev::Name", "rev"); + _config->Set("APT::Compressor::rev::Extension", ".reversed"); + _config->Set("APT::Compressor::rev::Binary", "rev"); + _config->Set("APT::Compressor::rev::Cost", 10); + auto const compressors = APT::Configuration::getCompressors(false); + EXPECT_EQ(7, compressors.size()); + EXPECT_TRUE(std::any_of(compressors.begin(), compressors.end(), [](APT::Configuration::Compressor const &c) { return c.Name == "rev"; })); + std::string const startdir = SafeGetCWD(); EXPECT_FALSE(startdir.empty()); std::string tempdir; -- cgit v1.2.3