[ Martin Pitt ]

* debian/rules:
  - Make DEB_BUILD_OPTIONS=noopt actually work by passing the right
    CXXFLAGS.
* apt-pkg/contrib/fileutl.{h,cc}:
  - Add support for reading of gzipped files with the new "ReadOnlyGzip"
    OpenMode. (Closes: #188407)
  - Link against zlib (in apt-pkg/makefile) and add zlib build dependency.
  - [ABI BREAK] This adds a new private member to FileFd, but its
    initialization is in the public header file.
* configure.in:
  - Check for zlib library and headers.
* apt-pkg/acquire-item.cc, apt-pkg/deb/debindexfile.cc,
  apt-pkg/deb/debrecords.cc, apt-pkg/deb/debsrcrecords.h,
  cmdline/apt-cache.cc:
  - Open Packages, Sources, and Translations indexes in "ReadOnlyGzip" mode.
* apt-pkg/deb/debindexfile.cc:
  - If we do not find uncompressed package/source/translation indexes, look
    for gzip compressed ones.
* apt-pkg/acquire-item.cc:
  - If the Acquire::GzipIndexes option is true and we download a gzipped
    index file, keep it as it is (and rename to .gz) instead of
    uncompressing it.
* doc/apt.conf.5.xml:
  - Document the new Acquire::GzipIndexes option.
* doc/po/apt-doc.pot, doc/po/de.po:
  - German translation of new Acquire::GzipIndexes option.
* Add test/test-indexes.sh:
  - Test behaviour of index retrieval and usage, in particular with
    uncompressed and gzip compressed indexes.
* methods/gzip.cc: With FileFd now being able to read gzipped files, there
  is no need for the gzip method any more to spawn an external gzip process.
  Rewrite it to use FileFd directly, which makes the code a lot simpler, and
  also using less memory and overhead.

7 files changed, 119 insertions, 28 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 58754a5c0..020efdfaa 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -280,7 +280,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
       ss >> ServerSha1 >> size;
       unsigned long const ServerSize = atol(size.c_str());
 
-      FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
+      FileFd fd(CurrentPackagesFile, FileFd::ReadOnlyGzip);
       SHA1Summation SHA1;
       SHA1.AddFD(fd.Fd(), fd.Size());
       string const local_sha1 = SHA1.Result();
@@ -511,7 +511,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff()					/*{{{*/
    string FinalFile = _config->FindDir("Dir::State::lists");
    FinalFile += URItoFileName(RealURI);
 
-   FileFd fd(FinalFile, FileFd::ReadOnly);
+   FileFd fd(FinalFile, FileFd::ReadOnlyGzip);
    SHA1Summation SHA1;
    SHA1.AddFD(fd.Fd(), fd.Size());
    string local_sha1 = string(SHA1.Result());
@@ -672,6 +672,8 @@ string pkgAcqIndex::Custom600Headers()
 {
    string Final = _config->FindDir("Dir::State::lists");
    Final += URItoFileName(RealURI);
+   if (_config->FindB("Acquire::GzipIndexes",false))
+      Final += ".gz";
    
    struct stat Buf;
    if (stat(Final.c_str(),&Buf) != 0)
@@ -789,18 +791,36 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
       ErrorText = "Method gave a blank filename";
    }
    
+   string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
+
    // The files timestamp matches
-   if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
+   if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
+       if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
+	  // Update DestFile for .gz suffix so that the clean operation keeps it
+	  DestFile += ".gz";
       return;
+    }
 
    if (FileName == DestFile)
       Erase = true;
    else
       Local = true;
    
-   string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
    string decompProg;
 
+   // If we enable compressed indexes and already have gzip, keep it
+   if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
+      string FinalFile = _config->FindDir("Dir::State::lists");
+      FinalFile += URItoFileName(RealURI) + ".gz";
+      Rename(DestFile,FinalFile);
+      chmod(FinalFile.c_str(),0644);
+      
+      // Update DestFile for .gz suffix so that the clean operation keeps it
+      DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+      DestFile += URItoFileName(RealURI) + ".gz";
+      return;
+    }
+
    // get the binary name for your used compression type
    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
    if(decompProg.empty() == false);
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index 8f7791a8a..49b2f3828 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -11,6 +11,7 @@
    Most of this source is placed in the Public Domain, do with it what 
    you will
    It was originally written by Jason Gunthorpe <jgg@debian.org>.
+   FileFd gzip support added by Martin Pitt <martin.pitt@canonical.com>
    
    The exception is RunScripts() it is under the GPLv2
 
@@ -656,6 +657,17 @@ bool FileFd::Open(string FileName,OpenMode Mode, unsigned long Perms)
       case ReadOnly:
       iFd = open(FileName.c_str(),O_RDONLY);
       break;
+
+      case ReadOnlyGzip:
+      iFd = open(FileName.c_str(),O_RDONLY);
+      if (iFd > 0) {
+	 gz = gzdopen (iFd, "r");
+	 if (gz == NULL) {
+	     close (iFd);
+	     iFd = -1;
+	 }
+      }
+      break;
       
       case WriteEmpty:
       {
@@ -711,7 +723,10 @@ bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual)
    
    do
    {
-      Res = read(iFd,To,Size);
+      if (gz != NULL)
+         Res = gzread(gz,To,Size);
+      else
+         Res = read(iFd,To,Size);
       if (Res < 0 && errno == EINTR)
 	 continue;
       if (Res < 0)
@@ -750,7 +765,10 @@ bool FileFd::Write(const void *From,unsigned long Size)
    errno = 0;
    do
    {
-      Res = write(iFd,From,Size);
+      if (gz != NULL)
+         Res = gzwrite(gz,From,Size);
+      else
+         Res = write(iFd,From,Size);
       if (Res < 0 && errno == EINTR)
 	 continue;
       if (Res < 0)
@@ -776,7 +794,12 @@ bool FileFd::Write(const void *From,unsigned long Size)
 /* */
 bool FileFd::Seek(unsigned long To)
 {
-   if (lseek(iFd,To,SEEK_SET) != (signed)To)
+   int res;
+   if (gz)
+      res = gzseek(gz,To,SEEK_SET);
+   else
+      res = lseek(iFd,To,SEEK_SET);
+   if (res != (signed)To)
    {
       Flags |= Fail;
       return _error->Error("Unable to seek to %lu",To);
@@ -790,7 +813,12 @@ bool FileFd::Seek(unsigned long To)
 /* */
 bool FileFd::Skip(unsigned long Over)
 {
-   if (lseek(iFd,Over,SEEK_CUR) < 0)
+   int res;
+   if (gz)
+      res = gzseek(gz,Over,SEEK_CUR);
+   else
+      res = lseek(iFd,Over,SEEK_CUR);
+   if (res < 0)
    {
       Flags |= Fail;
       return _error->Error("Unable to seek ahead %lu",Over);
@@ -804,6 +832,11 @@ bool FileFd::Skip(unsigned long Over)
 /* */
 bool FileFd::Truncate(unsigned long To)
 {
+   if (gz)
+   {
+      Flags |= Fail;
+      return _error->Error("Truncating gzipped files is not implemented (%s)", FileName.c_str());
+   }
    if (ftruncate(iFd,To) != 0)
    {
       Flags |= Fail;
@@ -818,7 +851,11 @@ bool FileFd::Truncate(unsigned long To)
 /* */
 unsigned long FileFd::Tell()
 {
-   off_t Res = lseek(iFd,0,SEEK_CUR);
+   off_t Res;
+   if (gz)
+     Res = gztell(gz);
+   else
+     Res = lseek(iFd,0,SEEK_CUR);
    if (Res == (off_t)-1)
       _error->Errno("lseek","Failed to determine the current file position");
    return Res;
@@ -829,6 +866,7 @@ unsigned long FileFd::Tell()
 /* */
 unsigned long FileFd::Size()
 {
+   //TODO: For gz, do we need the actual file size here or the uncompressed length?
    struct stat Buf;
    if (fstat(iFd,&Buf) != 0)
       return _error->Errno("fstat","Unable to determine the file size");
@@ -842,8 +880,16 @@ bool FileFd::Close()
 {
    bool Res = true;
    if ((Flags & AutoClose) == AutoClose)
-      if (iFd >= 0 && close(iFd) != 0)
-	 Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());
+   {
+      if (gz != NULL) {
+	 int const e = gzclose(gz);
+	 // gzdopen() on empty files always fails with "buffer error" here, ignore that
+	 if (e != 0 && e != Z_BUF_ERROR)
+	    Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
+      } else
+	 if (iFd > 0 && close(iFd) != 0)
+	    Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str());
+   }
 
    if ((Flags & Replace) == Replace && iFd >= 0) {
       if (rename(TemporaryFileName.c_str(), FileName.c_str()) != 0)
@@ -853,6 +899,7 @@ bool FileFd::Close()
    }
 
    iFd = -1;
+   gz = NULL;
 
    if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
        FileName.empty() == false)
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index 528725f89..0f70ab722 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -26,6 +26,8 @@
 #include <string>
 #include <vector>
 
+#include <zlib.h>
+
 using std::string;
 
 class FileFd
@@ -38,9 +40,10 @@ class FileFd
    unsigned long Flags;
    string FileName;
    string TemporaryFileName;
-   
+   gzFile gz;
+
    public:
-   enum OpenMode {ReadOnly,WriteEmpty,WriteExists,WriteAny,WriteTemp};
+   enum OpenMode {ReadOnly,WriteEmpty,WriteExists,WriteAny,WriteTemp,ReadOnlyGzip};
    
    inline bool Read(void *To,unsigned long Size,bool AllowEof)
    {
@@ -71,12 +74,12 @@ class FileFd
    inline string &Name() {return FileName;};
    
    FileFd(string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), 
-            Flags(0) 
+            Flags(0), gz(NULL)
    {
       Open(FileName,Mode,Perms);
    };
-   FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose) {};
-   FileFd(int Fd,bool) : iFd(Fd), Flags(0) {};
+   FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose), gz(NULL) {};
+   FileFd(int Fd,bool) : iFd(Fd), Flags(0), gz(NULL) {};
    virtual ~FileFd();
 };
 
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 5e6db3f38..7e9a973a4 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -63,9 +63,13 @@ string debSourcesIndex::SourceInfo(pkgSrcRecords::Parser const &Record,
 /* */
 pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const
 {
-   string SourcesURI = URItoFileName(IndexURI("Sources"));
-   return new debSrcRecordParser(_config->FindDir("Dir::State::lists") +
-				 SourcesURI,this);
+   string SourcesURI = _config->FindDir("Dir::State::lists") + 
+      URItoFileName(IndexURI("Sources"));
+   string SourcesURIgzip = SourcesURI + ".gz";
+   if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip))
+      SourcesURI = SourcesURIgzip;
+
+   return new debSrcRecordParser(SourcesURI,this);
 }
 									/*}}}*/
 // SourcesIndex::Describe - Give a descriptive path to the index	/*{{{*/
@@ -106,8 +110,14 @@ string debSourcesIndex::Info(const char *Type) const
 /* */
 inline string debSourcesIndex::IndexFile(const char *Type) const
 {
-   return URItoFileName(IndexURI(Type));
+   string s = URItoFileName(IndexURI(Type));
+   string sgzip = s + ".gz";
+   if (!FileExists(s) && FileExists(sgzip))
+       return sgzip;
+   else
+       return s;
 }
+
 string debSourcesIndex::IndexURI(const char *Type) const
 {
    string Res;
@@ -220,7 +230,12 @@ string debPackagesIndex::Info(const char *Type) const
 /* */
 inline string debPackagesIndex::IndexFile(const char *Type) const
 {
-   return _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
+   string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
+   string sgzip = s + ".gz";
+   if (!FileExists(s) && FileExists(sgzip))
+       return sgzip;
+   else
+       return s;
 }
 string debPackagesIndex::IndexURI(const char *Type) const
 {
@@ -265,8 +280,9 @@ unsigned long debPackagesIndex::Size() const
 bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 {
    string PackageFile = IndexFile("Packages");
-   FileFd Pkg(PackageFile,FileFd::ReadOnly);
+   FileFd Pkg(PackageFile,FileFd::ReadOnlyGzip);
    debListParser Parser(&Pkg, Architecture);
+
    if (_error->PendingError() == true)
       return _error->Error("Problem opening %s",PackageFile.c_str());
    if (Prog != NULL)
@@ -348,7 +364,12 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section
 /* */
 inline string debTranslationsIndex::IndexFile(const char *Type) const
 {
-   return _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
+   string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
+   string sgzip = s + ".gz";
+   if (!FileExists(s) && FileExists(sgzip))
+       return sgzip;
+   else
+       return s;
 }
 string debTranslationsIndex::IndexURI(const char *Type) const
 {
@@ -452,7 +473,7 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    string TranslationFile = IndexFile(Language);
    if (TranslationsAvailable() && FileExists(TranslationFile))
    {
-     FileFd Trans(TranslationFile,FileFd::ReadOnly);
+     FileFd Trans(TranslationFile,FileFd::ReadOnlyGzip);
      debListParser TransParser(&Trans);
      if (_error->PendingError() == true)
        return false;
@@ -533,7 +554,7 @@ unsigned long debStatusIndex::Size() const
 /* */
 bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 {
-   FileFd Pkg(File,FileFd::ReadOnly);
+   FileFd Pkg(File,FileFd::ReadOnlyGzip);
    if (_error->PendingError() == true)
       return false;
    debListParser Parser(&Pkg);
diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc
index 34ef0d8f2..ec9e395ef 100644
--- a/apt-pkg/deb/debrecords.cc
+++ b/apt-pkg/deb/debrecords.cc
@@ -19,7 +19,7 @@
 // ---------------------------------------------------------------------
 /* */
 debRecordParser::debRecordParser(string FileName,pkgCache &Cache) : 
-                  File(FileName,FileFd::ReadOnly), 
+                  File(FileName,FileFd::ReadOnlyGzip), 
                   Tags(&File, std::max(Cache.Head().MaxVerFileSize, 
 				       Cache.Head().MaxDescFileSize) + 200)
 {
diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h
index c39d78bae..905264daa 100644
--- a/apt-pkg/deb/debsrcrecords.h
+++ b/apt-pkg/deb/debsrcrecords.h
@@ -48,7 +48,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser
    virtual bool Files(vector<pkgSrcRecords::File> &F);
 
    debSrcRecordParser(string const &File,pkgIndexFile const *Index) 
-      : Parser(Index), Fd(File,FileFd::ReadOnly), Tags(&Fd,102400), 
+      : Parser(Index), Fd(File,FileFd::ReadOnlyGzip), Tags(&Fd,102400), 
         Buffer(0), BufSize(0) {}
    ~debSrcRecordParser();
 };
diff --git a/apt-pkg/makefile b/apt-pkg/makefile
index a5be462ce..2a7958536 100644
--- a/apt-pkg/makefile
+++ b/apt-pkg/makefile
@@ -14,7 +14,7 @@ include ../buildlib/libversion.mak
 LIBRARY=apt-pkg
 MAJOR=$(LIBAPTPKG_MAJOR)
 MINOR=$(LIBAPTPKG_RELEASE)
-SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil -ldl
+SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil -ldl -lz
 APT_DOMAIN:=libapt-pkg$(LIBAPTPKG_MAJOR)
 
 # Source code for the contributed non-core things