diff options
author | Michael Vogt <mvo@debian.org> | 2014-03-14 09:02:44 +0100 |
---|---|---|
committer | Michael Vogt <mvo@debian.org> | 2014-03-14 09:02:44 +0100 |
commit | 83b880c6505a20247239d897b7387bba37942993 (patch) | |
tree | 11a23275548175e301f6fefda20a6a8bb6fe6de8 /apt-pkg/contrib/fileutl.cc | |
parent | f98d9bd2adf4f24a72d973f5752b47987843984c (diff) | |
parent | 40f8a8ba8c2e7ff9ce80781250c863c07ac130dd (diff) |
fix test/integration/test-apt-helper
Diffstat (limited to 'apt-pkg/contrib/fileutl.cc')
-rw-r--r-- | apt-pkg/contrib/fileutl.cc | 475 |
1 files changed, 326 insertions, 149 deletions
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 79bcf112c..1eabf37f4 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1,6 +1,5 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: fileutl.cc,v 1.42 2002/09/14 05:29:22 jgg Exp $ /* ###################################################################### File Utilities @@ -26,16 +25,22 @@ #include <apt-pkg/sptr.h> #include <apt-pkg/aptconfiguration.h> #include <apt-pkg/configuration.h> - +#include <apt-pkg/macros.h> + +#include <ctype.h> +#include <stdarg.h> +#include <stddef.h> +#include <sys/select.h> +#include <time.h> +#include <string> +#include <vector> #include <cstdlib> #include <cstring> #include <cstdio> - #include <iostream> #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> -#include <sys/types.h> #include <sys/time.h> #include <sys/wait.h> #include <dirent.h> @@ -52,6 +57,10 @@ #ifdef HAVE_BZ2 #include <bzlib.h> #endif +#ifdef HAVE_LZMA + #include <stdint.h> + #include <lzma.h> +#endif #ifdef WORDS_BIGENDIAN #include <inttypes.h> @@ -62,54 +71,6 @@ using namespace std; -class FileFdPrivate { - public: -#ifdef HAVE_ZLIB - gzFile gz; -#else - void* gz; -#endif -#ifdef HAVE_BZ2 - BZFILE* bz2; -#else - void* bz2; -#endif - int compressed_fd; - pid_t compressor_pid; - bool pipe; - APT::Configuration::Compressor compressor; - unsigned int openmode; - unsigned long long seekpos; - FileFdPrivate() : gz(NULL), bz2(NULL), - compressed_fd(-1), compressor_pid(-1), pipe(false), - openmode(0), seekpos(0) {}; - bool CloseDown(std::string const &FileName) - { - bool Res = true; -#ifdef HAVE_ZLIB - if (gz != NULL) { - int const e = gzclose(gz); - gz = NULL; - // gzdclose() on empty files always fails with "buffer error" here, ignore that - if (e != 0 && e != Z_BUF_ERROR) - Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); - } -#endif -#ifdef HAVE_BZ2 - if (bz2 != NULL) { - BZ2_bzclose(bz2); - bz2 = NULL; - } -#endif - if (compressor_pid > 0) - ExecWait(compressor_pid, "FileFdCompressor", true); - compressor_pid = -1; - - return Res; - } - ~FileFdPrivate() { CloseDown(""); } -}; - // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -874,6 +835,122 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) } /*}}}*/ +class FileFdPrivate { /*{{{*/ + public: +#ifdef HAVE_ZLIB + gzFile gz; +#endif +#ifdef HAVE_BZ2 + BZFILE* bz2; +#endif +#ifdef HAVE_LZMA + struct LZMAFILE { + FILE* file; + uint8_t buffer[4096]; + lzma_stream stream; + lzma_ret err; + bool eof; + bool compressing; + + LZMAFILE() : file(NULL), eof(false), compressing(false) {} + ~LZMAFILE() { + if (compressing == true) + { + for (;;) { + stream.avail_out = sizeof(buffer)/sizeof(buffer[0]); + stream.next_out = buffer; + err = lzma_code(&stream, LZMA_FINISH); + if (err != LZMA_OK && err != LZMA_STREAM_END) + { + _error->Error("~LZMAFILE: Compress finalisation failed"); + break; + } + size_t const n = sizeof(buffer)/sizeof(buffer[0]) - stream.avail_out; + if (n && fwrite(buffer, 1, n, file) != n) + { + _error->Errno("~LZMAFILE",_("Write error")); + break; + } + if (err == LZMA_STREAM_END) + break; + } + } + lzma_end(&stream); + fclose(file); + } + }; + LZMAFILE* lzma; +#endif + int compressed_fd; + pid_t compressor_pid; + bool pipe; + APT::Configuration::Compressor compressor; + unsigned int openmode; + unsigned long long seekpos; + FileFdPrivate() : +#ifdef HAVE_ZLIB + gz(NULL), +#endif +#ifdef HAVE_BZ2 + bz2(NULL), +#endif +#ifdef HAVE_LZMA + lzma(NULL), +#endif + compressed_fd(-1), compressor_pid(-1), pipe(false), + openmode(0), seekpos(0) {}; + bool InternalClose(std::string const &FileName) + { + if (false) + /* dummy so that the rest can be 'else if's */; +#ifdef HAVE_ZLIB + else if (gz != NULL) { + int const e = gzclose(gz); + gz = NULL; + // gzdclose() on empty files always fails with "buffer error" here, ignore that + if (e != 0 && e != Z_BUF_ERROR) + return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); + } +#endif +#ifdef HAVE_BZ2 + else if (bz2 != NULL) { + BZ2_bzclose(bz2); + bz2 = NULL; + } +#endif +#ifdef HAVE_LZMA + else if (lzma != NULL) { + delete lzma; + lzma = NULL; + } +#endif + return true; + } + bool CloseDown(std::string const &FileName) + { + bool const Res = InternalClose(FileName); + + if (compressor_pid > 0) + ExecWait(compressor_pid, "FileFdCompressor", true); + compressor_pid = -1; + + return Res; + } + bool InternalStream() const { + return false +#ifdef HAVE_BZ2 + || bz2 != NULL +#endif +#ifdef HAVE_LZMA + || lzma != NULL +#endif + ; + } + + + ~FileFdPrivate() { CloseDown(""); } +}; + /*}}}*/ // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ @@ -891,7 +968,7 @@ bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, { for (; compressor != compressors.end(); ++compressor) { - std::string file = std::string(FileName).append(compressor->Extension); + std::string file = FileName + compressor->Extension; if (FileExists(file) == false) continue; FileName = file; @@ -1061,30 +1138,12 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration: { Close(); Flags = (AutoClose) ? FileFd::AutoClose : 0; - if (AutoClose == false && ( -#ifdef HAVE_ZLIB - compressor.Name == "gzip" || -#endif -#ifdef HAVE_BZ2 - compressor.Name == "bzip2" || -#endif - false)) - { - // Need to duplicate fd here or gzclose for cleanup will close the fd as well - iFd = dup(Fd); - } - else - iFd = Fd; + iFd = Fd; this->FileName = ""; - if (Fd == -1 || OpenInternDescriptor(Mode, compressor) == false) + if (OpenInternDescriptor(Mode, compressor) == false) { if (iFd != -1 && ( -#ifdef HAVE_ZLIB - compressor.Name == "gzip" || -#endif -#ifdef HAVE_BZ2 - compressor.Name == "bzip2" || -#endif + (Flags & Compressed) == Compressed || AutoClose == true)) { close (iFd); @@ -1096,52 +1155,121 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration: } bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) { + if (iFd == -1) + return false; if (compressor.Name == "." || compressor.Binary.empty() == true) return true; +#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA + // the API to open files is similar, so setup to avoid code duplicates later + // and while at it ensure that we close before opening (if its a reopen) + void* (*compress_open)(int, const char *) = NULL; + if (false) + /* dummy so that the rest can be 'else if's */; +#define APT_COMPRESS_INIT(NAME,OPEN) \ + else if (compressor.Name == NAME) \ + { \ + compress_open = (void*(*)(int, const char *)) OPEN; \ + if (d != NULL) d->InternalClose(FileName); \ + } +#ifdef HAVE_ZLIB + APT_COMPRESS_INIT("gzip", gzdopen) +#endif +#ifdef HAVE_BZ2 + APT_COMPRESS_INIT("bzip2", BZ2_bzdopen) +#endif +#ifdef HAVE_LZMA + APT_COMPRESS_INIT("xz", fdopen) + APT_COMPRESS_INIT("lzma", fdopen) +#endif +#undef APT_COMPRESS_INIT +#endif + if (d == NULL) { d = new FileFdPrivate(); d->openmode = Mode; d->compressor = compressor; +#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA + if (AutoClose == false && compress_open != NULL) + { + // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well + int const internFd = dup(iFd); + if (internFd == -1) + return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd); + iFd = internFd; + } +#endif } -#ifdef HAVE_ZLIB - if (compressor.Name == "gzip") +#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA + if (compress_open != NULL) { - if (d->gz != NULL) - { - gzclose(d->gz); - d->gz = NULL; - } + void* compress_struct = NULL; if ((Mode & ReadWrite) == ReadWrite) - d->gz = gzdopen(iFd, "r+"); + compress_struct = compress_open(iFd, "r+"); else if ((Mode & WriteOnly) == WriteOnly) - d->gz = gzdopen(iFd, "w"); + compress_struct = compress_open(iFd, "w"); else - d->gz = gzdopen(iFd, "r"); - if (d->gz == NULL) + compress_struct = compress_open(iFd, "r"); + if (compress_struct == NULL) return false; - Flags |= Compressed; - return true; - } + + if (false) + /* dummy so that the rest can be 'else if's */; +#ifdef HAVE_ZLIB + else if (compressor.Name == "gzip") + d->gz = (gzFile) compress_struct; #endif #ifdef HAVE_BZ2 - if (compressor.Name == "bzip2") - { - if (d->bz2 != NULL) + else if (compressor.Name == "bzip2") + d->bz2 = (BZFILE*) compress_struct; +#endif +#ifdef HAVE_LZMA + else if (compressor.Name == "xz" || compressor.Name == "lzma") { - BZ2_bzclose(d->bz2); - d->bz2 = NULL; + uint32_t const xzlevel = 6; + uint64_t const memlimit = UINT64_MAX; + if (d->lzma == NULL) + d->lzma = new FileFdPrivate::LZMAFILE; + d->lzma->file = (FILE*) compress_struct; + d->lzma->stream = LZMA_STREAM_INIT; + + if ((Mode & ReadWrite) == ReadWrite) + return FileFdError("ReadWrite mode is not supported for file %s", FileName.c_str()); + + if ((Mode & WriteOnly) == WriteOnly) + { + if (compressor.Name == "xz") + { + if (lzma_easy_encoder(&d->lzma->stream, xzlevel, LZMA_CHECK_CRC32) != LZMA_OK) + return false; + } + else + { + lzma_options_lzma options; + lzma_lzma_preset(&options, xzlevel); + if (lzma_alone_encoder(&d->lzma->stream, &options) != LZMA_OK) + return false; + } + d->lzma->compressing = true; + } + else + { + if (compressor.Name == "xz") + { + if (lzma_auto_decoder(&d->lzma->stream, memlimit, 0) != LZMA_OK) + return false; + } + else + { + if (lzma_alone_decoder(&d->lzma->stream, memlimit) != LZMA_OK) + return false; + } + d->lzma->compressing = false; + } } - if ((Mode & ReadWrite) == ReadWrite) - d->bz2 = BZ2_bzdopen(iFd, "r+"); - else if ((Mode & WriteOnly) == WriteOnly) - d->bz2 = BZ2_bzdopen(iFd, "w"); - else - d->bz2 = BZ2_bzdopen(iFd, "r"); - if (d->bz2 == NULL) - return false; +#endif Flags |= Compressed; return true; } @@ -1198,7 +1326,7 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C } else { - if (FileName.empty() == true) + if (d->compressed_fd != -1) dup2(d->compressed_fd,STDIN_FILENO); dup2(Pipe[1],STDOUT_FILENO); } @@ -1267,24 +1395,55 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) *((char *)To) = '\0'; do { + if (false) + /* dummy so that the rest can be 'else if's */; #ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) + else if (d != NULL && d->gz != NULL) Res = gzread(d->gz,To,Size); - else #endif #ifdef HAVE_BZ2 - if (d != NULL && d->bz2 != NULL) + else if (d != NULL && d->bz2 != NULL) Res = BZ2_bzread(d->bz2,To,Size); - else #endif +#ifdef HAVE_LZMA + else if (d != NULL && d->lzma != NULL) + { + if (d->lzma->eof == true) + break; + + d->lzma->stream.next_out = (uint8_t *) To; + d->lzma->stream.avail_out = Size; + if (d->lzma->stream.avail_in == 0) + { + d->lzma->stream.next_in = d->lzma->buffer; + d->lzma->stream.avail_in = fread(d->lzma->buffer, 1, sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]), d->lzma->file); + } + d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN); + if (d->lzma->err == LZMA_STREAM_END) + { + d->lzma->eof = true; + Res = Size - d->lzma->stream.avail_out; + } + else if (d->lzma->err != LZMA_OK) + { + Res = -1; + errno = 0; + } + else + Res = Size - d->lzma->stream.avail_out; + } +#endif + else Res = read(iFd,To,Size); if (Res < 0) { if (errno == EINTR) continue; + if (false) + /* dummy so that the rest can be 'else if's */; #ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) + else if (d != NULL && d->gz != NULL) { int err; char const * const errmsg = gzerror(d->gz, &err); @@ -1293,7 +1452,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) } #endif #ifdef HAVE_BZ2 - if (d != NULL && d->bz2 != NULL) + else if (d != NULL && d->bz2 != NULL) { int err; char const * const errmsg = BZ2_bzerror(d->bz2, &err); @@ -1301,6 +1460,10 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) return FileFdError("BZ2_bzread: %s (%d: %s)", _("Read error"), err, errmsg); } #endif +#ifdef HAVE_LZMA + else if (d != NULL && d->lzma != NULL) + return FileFdError("lzma_read: %s (%d)", _("Read error"), d->lzma->err); +#endif return FileFdErrno("read",_("Read error")); } @@ -1364,23 +1527,45 @@ bool FileFd::Write(const void *From,unsigned long long Size) errno = 0; do { + if (false) + /* dummy so that the rest can be 'else if's */; #ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) - Res = gzwrite(d->gz,From,Size); - else + else if (d != NULL && d->gz != NULL) + Res = gzwrite(d->gz,From,Size); #endif #ifdef HAVE_BZ2 - if (d != NULL && d->bz2 != NULL) - Res = BZ2_bzwrite(d->bz2,(void*)From,Size); - else + else if (d != NULL && d->bz2 != NULL) + Res = BZ2_bzwrite(d->bz2,(void*)From,Size); +#endif +#ifdef HAVE_LZMA + else if (d != NULL && d->lzma != NULL) + { + d->lzma->stream.next_in = (uint8_t *)From; + d->lzma->stream.avail_in = Size; + d->lzma->stream.next_out = d->lzma->buffer; + d->lzma->stream.avail_out = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]); + d->lzma->err = lzma_code(&d->lzma->stream, LZMA_RUN); + if (d->lzma->err != LZMA_OK) + return false; + size_t const n = sizeof(d->lzma->buffer)/sizeof(d->lzma->buffer[0]) - d->lzma->stream.avail_out; + size_t const m = (n == 0) ? 0 : fwrite(d->lzma->buffer, 1, n, d->lzma->file); + if (m != n) + Res = -1; + else + Res = Size - d->lzma->stream.avail_in; + } #endif - Res = write(iFd,From,Size); + else + Res = write(iFd,From,Size); + if (Res < 0 && errno == EINTR) continue; if (Res < 0) { + if (false) + /* dummy so that the rest can be 'else if's */; #ifdef HAVE_ZLIB - if (d != NULL && d->gz != NULL) + else if (d != NULL && d->gz != NULL) { int err; char const * const errmsg = gzerror(d->gz, &err); @@ -1389,7 +1574,7 @@ bool FileFd::Write(const void *From,unsigned long long Size) } #endif #ifdef HAVE_BZ2 - if (d != NULL && d->bz2 != NULL) + else if (d != NULL && d->bz2 != NULL) { int err; char const * const errmsg = BZ2_bzerror(d->bz2, &err); @@ -1397,10 +1582,14 @@ bool FileFd::Write(const void *From,unsigned long long Size) return FileFdError("BZ2_bzwrite: %s (%d: %s)", _("Write error"), err, errmsg); } #endif +#ifdef HAVE_LZMA + else if (d != NULL && d->lzma != NULL) + return FileFdErrno("lzma_fwrite", _("Write error")); +#endif return FileFdErrno("write",_("Write error")); } - From = (char *)From + Res; + From = (char const *)From + Res; Size -= Res; if (d != NULL) d->seekpos += Res; @@ -1424,7 +1613,7 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size) if (Res < 0) return _error->Errno("write",_("Write error")); - From = (char *)From + Res; + From = (char const *)From + Res; Size -= Res; } while (Res > 0 && Size > 0); @@ -1440,11 +1629,7 @@ bool FileFd::Write(int Fd, const void *From, unsigned long long Size) /* */ bool FileFd::Seek(unsigned long long To) { - if (d != NULL && (d->pipe == true -#ifdef HAVE_BZ2 - || d->bz2 != NULL -#endif - )) + if (d != NULL && (d->pipe == true || d->InternalStream() == true)) { // Our poor man seeking in pipes is costly, so try to avoid it unsigned long long seekpos = Tell(); @@ -1455,13 +1640,7 @@ bool FileFd::Seek(unsigned long long To) if ((d->openmode & ReadOnly) != ReadOnly) return FileFdError("Reopen is only implemented for read-only files!"); -#ifdef HAVE_BZ2 - if (d->bz2 != NULL) - { - BZ2_bzclose(d->bz2); - d->bz2 = NULL; - } -#endif + d->InternalClose(FileName); if (iFd != -1) close(iFd); iFd = -1; @@ -1507,11 +1686,7 @@ bool FileFd::Seek(unsigned long long To) /* */ bool FileFd::Skip(unsigned long long Over) { - if (d != NULL && (d->pipe == true -#ifdef HAVE_BZ2 - || d->bz2 != NULL -#endif - )) + if (d != NULL && (d->pipe == true || d->InternalStream() == true)) { d->seekpos += Over; char buffer[1024]; @@ -1548,8 +1723,12 @@ bool FileFd::Truncate(unsigned long long To) // truncating /dev/null is always successful - as we get an error otherwise if (To == 0 && FileName == "/dev/null") return true; -#if defined HAVE_ZLIB || defined HAVE_BZ2 - if (d != NULL && (d->gz != NULL || d->bz2 != NULL)) +#if defined HAVE_ZLIB || defined HAVE_BZ2 || defined HAVE_LZMA + if (d != NULL && (d->InternalStream() == true +#ifdef HAVE_ZLIB + || d->gz != NULL +#endif + )) return FileFdError("Truncating compressed files is not implemented (%s)", FileName.c_str()); #endif if (ftruncate(iFd,To) != 0) @@ -1567,11 +1746,7 @@ unsigned long long FileFd::Tell() // seeking around, but not all users of FileFd use always Seek() and co // so d->seekpos isn't always true and we can just use it as a hint if // we have nothing else, but not always as an authority… - if (d != NULL && (d->pipe == true -#ifdef HAVE_BZ2 - || d->bz2 != NULL -#endif - )) + if (d != NULL && (d->pipe == true || d->InternalStream() == true)) return d->seekpos; off_t Res; @@ -1646,11 +1821,7 @@ unsigned long long FileFd::Size() // for compressor pipes st_size is undefined and at 'best' zero, // so we 'read' the content and 'seek' back - see there - if (d != NULL && (d->pipe == true -#ifdef HAVE_BZ2 - || (d->bz2 && size > 0) -#endif - )) + if (d != NULL && (d->pipe == true || (d->InternalStream() == true && size > 0))) { unsigned long long const oldSeek = Tell(); char ignore[1000]; @@ -1793,7 +1964,13 @@ bool FileFd::FileFdError(const char *Description,...) { } /*}}}*/ -gzFile FileFd::gzFd() { return (gzFile) d->gz; } +APT_DEPRECATED gzFile FileFd::gzFd() { +#ifdef HAVE_ZLIB + return d->gz; +#else + return NULL; +#endif +} // Glob - wrapper around "glob()" /*{{{*/ |