diff options
Diffstat (limited to 'apt-pkg')
-rw-r--r-- | apt-pkg/contrib/fileutl.cc | 302 | ||||
-rw-r--r-- | apt-pkg/contrib/fileutl.h | 10 | ||||
-rw-r--r-- | apt-pkg/contrib/mmap.cc | 9 | ||||
-rw-r--r-- | apt-pkg/deb/deblistparser.cc | 1 | ||||
-rw-r--r-- | apt-pkg/indexcopy.cc | 97 |
5 files changed, 316 insertions, 103 deletions
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 58cd6dceb..727d3ddb5 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -44,7 +44,13 @@ #include <set> #include <algorithm> +// FIXME: Compressor Fds have some speed disadvantages and are a bit buggy currently, +// so while the current implementation satisfies the testcases it is not a real option +// to disable it for now +#define APT_USE_ZLIB 1 +#ifdef APT_USE_ZLIB #include <zlib.h> +#endif #ifdef WORDS_BIGENDIAN #include <inttypes.h> @@ -57,8 +63,16 @@ using namespace std; class FileFdPrivate { public: +#ifdef APT_USE_ZLIB gzFile gz; - FileFdPrivate() : gz(NULL) {}; +#else + void* gz; +#endif + pid_t compressor_pid; + bool pipe; + APT::Configuration::Compressor compressor; + FileFd::OpenMode openmode; + FileFdPrivate() : gz(NULL), compressor_pid(-1), pipe(false) {}; }; // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ @@ -724,6 +738,175 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) } /*}}}*/ +// ExecCompressor - Open a de/compressor pipe /*{{{*/ +// --------------------------------------------------------------------- +/* This opens the compressor, either in compress mode or decompress + mode. FileFd is always the compressor input/output file, + OutFd is the created pipe, Input for Compress, Output for Decompress. */ +bool ExecCompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, int const FileFd, int &OutFd, bool const Comp) +{ + if (Pid != NULL) + *Pid = -1; + + // No compression + if (Prog.Binary.empty() == true) + { + OutFd = dup(FileFd); + return true; + } + + // Handle 'decompression' of empty files + if (Comp == false) + { + struct stat Buf; + fstat(FileFd, &Buf); + if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) + { + OutFd = FileFd; + return true; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + if (Comp == true) + OutFd = Pipe[1]; + else + OutFd = Pipe[0]; + + // The child.. + pid_t child = ExecFork(); + if (Pid != NULL) + *Pid = child; + if (child == 0) + { + if (Comp == true) + { + dup2(FileFd,STDOUT_FILENO); + dup2(Pipe[0],STDIN_FILENO); + } + else + { + dup2(FileFd,STDIN_FILENO); + dup2(Pipe[1],STDOUT_FILENO); + } + + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + + std::vector<char const*> Args; + Args.push_back(Prog.Binary.c_str()); + std::vector<std::string> const * const addArgs = + (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); + for (std::vector<std::string>::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + + if (Pid == NULL) + ExecWait(child, Prog.Binary.c_str(), true); + + return true; +} +bool ExecCompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, std::string const &FileName, int &OutFd, bool const Comp) +{ + if (Pid != NULL) + *Pid = -1; + + // No compression + if (Prog.Binary.empty() == true) + { + if (Comp == true) + OutFd = open(FileName.c_str(), O_WRONLY, 0666); + else + OutFd = open(FileName.c_str(), O_RDONLY); + return true; + } + + // Handle 'decompression' of empty files + if (Comp == false) + { + struct stat Buf; + stat(FileName.c_str(), &Buf); + if (Buf.st_size == 0) + { + OutFd = open(FileName.c_str(), O_RDONLY); + return true; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + if (Comp == true) + OutFd = Pipe[1]; + else + OutFd = Pipe[0]; + + // The child.. + pid_t child = ExecFork(); + if (Pid != NULL) + *Pid = child; + if (child == 0) + { + if (Comp == true) + { + dup2(Pipe[0],STDIN_FILENO); + SetCloseExec(STDIN_FILENO,false); + } + else + { + dup2(Pipe[1],STDOUT_FILENO); + SetCloseExec(STDOUT_FILENO,false); + } + + std::vector<char const*> Args; + Args.push_back(Prog.Binary.c_str()); + std::vector<std::string> const * const addArgs = + (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); + for (std::vector<std::string>::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + Args.push_back("--stdout"); + Args.push_back(FileName.c_str()); + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + + if (Pid == NULL) + ExecWait(child, Prog.Binary.c_str(), false); + + return true; +} + /*}}}*/ + // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ @@ -733,6 +916,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned return Open(FileName, ReadOnly, Gzip, Perms); Close(); d = new FileFdPrivate; + d->openmode = Mode; Flags = AutoClose; if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) @@ -805,12 +989,15 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned // if we have them, use inbuilt compressors instead of forking if (compressor != compressors.end()) { +#ifdef APT_USE_ZLIB if (compressor->Name == "gzip") { Compress = Gzip; compressor = compressors.end(); } - else if (compressor->Name == "." || Compress == None) + else +#endif + if (compressor->Name == ".") { Compress = None; compressor = compressors.end(); @@ -839,9 +1026,12 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned if (compressor != compressors.end()) { if ((Mode & ReadWrite) == ReadWrite) - _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); + return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); - _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + if (ExecCompressor(*compressor, NULL /*d->compressor_pid*/, FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + d->pipe = true; + d->compressor = *compressor; } else { @@ -875,6 +1065,7 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A { Close(); d = new FileFdPrivate; + d->openmode = Mode; Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; if (OpenInternDescriptor(Mode, Compress) == false) @@ -890,6 +1081,7 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) { if (Compress == None) return true; +#ifdef APT_USE_ZLIB else if (Compress == Gzip) { if ((Mode & ReadWrite) == ReadWrite) @@ -902,8 +1094,29 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) return false; Flags |= Compressed; } +#endif else - return false; + { + std::string name; + switch (Compress) + { + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + default: return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + } + std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors(); + std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin(); + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end() || + ExecCompressor(*compressor, NULL /*&(d->compressor_pid)*/, + FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", name.c_str(), FileName.c_str()); + d->pipe = true; + } return true; } /*}}}*/ @@ -926,12 +1139,14 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) errno = 0; if (Actual != 0) *Actual = 0; - + *((char *)To) = '\0'; do { +#ifdef APT_USE_ZLIB if (d->gz != NULL) Res = gzread(d->gz,To,Size); else +#endif Res = read(iFd,To,Size); if (Res < 0 && errno == EINTR) continue; @@ -968,8 +1183,11 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) files because of the naive implementation! */ char* FileFd::ReadLine(char *To, unsigned long long const Size) { + *To = '\0'; +#ifdef APT_USE_ZLIB if (d->gz != NULL) return gzgets(d->gz, To, Size); +#endif unsigned long long read = 0; if (Read(To, Size, &read) == false) @@ -993,9 +1211,11 @@ bool FileFd::Write(const void *From,unsigned long long Size) errno = 0; do { +#ifdef APT_USE_ZLIB if (d->gz != NULL) Res = gzwrite(d->gz,From,Size); else +#endif Res = write(iFd,From,Size); if (Res < 0 && errno == EINTR) continue; @@ -1022,10 +1242,21 @@ bool FileFd::Write(const void *From,unsigned long long Size) /* */ bool FileFd::Seek(unsigned long long To) { + if (d->pipe == true) + { + // FIXME: What about OpenDescriptor() stuff here? + close(iFd); + bool result = ExecCompressor(d->compressor, NULL, FileName, iFd, (d->openmode & ReadOnly) != ReadOnly); + if (result == true && To != 0) + result &= Skip(To); + return result; + } int res; +#ifdef USE_ZLIB if (d->gz) res = gzseek(d->gz,To,SEEK_SET); else +#endif res = lseek(iFd,To,SEEK_SET); if (res != (signed)To) { @@ -1042,9 +1273,11 @@ bool FileFd::Seek(unsigned long long To) bool FileFd::Skip(unsigned long long Over) { int res; +#ifdef USE_ZLIB if (d->gz != NULL) res = gzseek(d->gz,Over,SEEK_CUR); else +#endif res = lseek(iFd,Over,SEEK_CUR); if (res < 0) { @@ -1080,9 +1313,11 @@ bool FileFd::Truncate(unsigned long long To) unsigned long long FileFd::Tell() { off_t Res; +#ifdef USE_ZLIB if (d->gz != NULL) Res = gztell(d->gz); else +#endif Res = lseek(iFd,0,SEEK_CUR); if (Res == (off_t)-1) _error->Errno("lseek","Failed to determine the current file position"); @@ -1095,9 +1330,19 @@ unsigned long long FileFd::Tell() unsigned long long FileFd::FileSize() { struct stat Buf; - - if (fstat(iFd,&Buf) != 0) + if (d->pipe == false && fstat(iFd,&Buf) != 0) return _error->Errno("fstat","Unable to determine the file size"); + + // for compressor pipes st_size is undefined and at 'best' zero + if (d->pipe == true || S_ISFIFO(Buf.st_mode)) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + d->pipe = true; + if (stat(FileName.c_str(), &Buf) != 0) + return _error->Errno("stat","Unable to determine the file size"); + } + return Buf.st_size; } /*}}}*/ @@ -1108,10 +1353,25 @@ unsigned long long FileFd::Size() { unsigned long long size = FileSize(); + // for compressor pipes st_size is undefined and at 'best' zero, + // so we 'read' the content and 'seek' back - see there + if (d->pipe == true) + { + // FIXME: If we have read first and then FileSize() the report is wrong + size = 0; + char ignore[1000]; + unsigned long long read = 0; + do { + Read(ignore, sizeof(ignore), &read); + size += read; + } while(read != 0); + Seek(0); + } +#ifdef USE_ZLIB // only check gzsize if we are actually a gzip file, just checking for // "gz" is not sufficient as uncompressed files could be opened with // gzopen in "direct" mode as well - if (d->gz && !gzdirect(d->gz) && size > 0) + else if (d->gz && !gzdirect(d->gz) && size > 0) { /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do * this ourselves; the original (uncompressed) file size is the last 32 @@ -1135,6 +1395,7 @@ unsigned long long FileFd::Size() return _error->Errno("lseek","Unable to seek in gzipped file"); return size; } +#endif return size; } @@ -1145,11 +1406,25 @@ unsigned long long FileFd::Size() time_t FileFd::ModificationTime() { struct stat Buf; - if (fstat(iFd,&Buf) != 0) + if (d->pipe == false && fstat(iFd,&Buf) != 0) { _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); return 0; } + + // for compressor pipes st_size is undefined and at 'best' zero + if (d->pipe == true || S_ISFIFO(Buf.st_mode)) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + d->pipe = true; + if (stat(FileName.c_str(), &Buf) != 0) + { + _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); + return 0; + } + } + return Buf.st_mtime; } /*}}}*/ @@ -1164,12 +1439,14 @@ bool FileFd::Close() bool Res = true; if ((Flags & AutoClose) == AutoClose) { +#ifdef USE_ZLIB if (d != NULL && d->gz != NULL) { int const e = gzclose(d->gz); // gzdopen() on empty files always fails with "buffer error" here, ignore that if (e != 0 && e != Z_BUF_ERROR) Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); } else +#endif if (iFd > 0 && close(iFd) != 0) Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str()); } @@ -1191,6 +1468,8 @@ bool FileFd::Close() if (d != NULL) { +// if (d->compressor_pid != -1) +// ExecWait(d->compressor_pid, "FileFdCompressor", true); delete d; d = NULL; } @@ -1210,4 +1489,5 @@ bool FileFd::Sync() return true; } /*}}}*/ -gzFile FileFd::gzFd() {return d->gz;}; + +gzFile FileFd::gzFd() { return (gzFile) d->gz; } diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 209ca91e7..f96dc72dc 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -22,6 +22,7 @@ #define PKGLIB_FILEUTL_H #include <apt-pkg/macros.h> +#include <apt-pkg/aptconfiguration.h> #include <string> #include <vector> @@ -110,6 +111,7 @@ class FileFd inline int Fd() {return iFd;}; inline void Fd(int fd) {iFd = fd;}; __deprecated gzFile gzFd(); + inline bool IsOpen() {return iFd >= 0;}; inline bool Failed() {return (Flags & Fail) == Fail;}; inline void EraseOnFailure() {Flags |= DelOnFail;}; @@ -170,6 +172,14 @@ bool WaitFd(int Fd,bool write = false,unsigned long timeout = 0); pid_t ExecFork(); bool ExecWait(pid_t Pid,const char *Name,bool Reap = false); +bool ExecCompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, int const FileFd, int &OutFd, bool const Comp = true); +inline bool ExecDecompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, int const FileFd, int &OutFd) +{ + return ExecCompressor(Prog, Pid, FileFd, OutFd, true); +} + // File string manipulators std::string flNotDir(std::string File); std::string flNotFile(std::string File); diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 1fb84b0af..a67ab3698 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -66,7 +66,7 @@ MMap::~MMap() bool MMap::Map(FileFd &Fd) { iSize = Fd.Size(); - + // Set the permissions. int Prot = PROT_READ; int Map = MAP_SHARED; @@ -97,6 +97,13 @@ bool MMap::Map(FileFd &Fd) { // The filesystem doesn't support this particular kind of mmap. // So we allocate a buffer and read the whole file into it. + if ((Flags & ReadOnly) == ReadOnly) + { + // for readonly, we don't need sync, so make it simple + Base = new unsigned char[iSize]; + return Fd.Read(Base, iSize); + } + // FIXME: Writing to compressed fd's ? int const dupped_fd = dup(Fd.Fd()); if (dupped_fd == -1) return _error->Errno("mmap", _("Couldn't duplicate file descriptor %i"), Fd.Fd()); diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 28568d5e3..bdb50f6bf 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -773,6 +773,7 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, // file. to provide Component pinning we use the section name now FileI->Component = WriteUniqString(component); + // FIXME: Code depends on the fact that Release files aren't compressed FILE* release = fdopen(dup(File.Fd()), "r"); if (release == NULL) return false; diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index f6457aa39..3747e3570 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -38,65 +38,6 @@ using namespace std; -// DecompressFile - wrapper for decompressing compressed files /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool DecompressFile(string Filename, int *fd, off_t *FileSize) -{ - struct stat Buf; - *fd = -1; - - std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); - std::vector<APT::Configuration::Compressor>::const_iterator UnCompress; - std::string file = std::string(Filename).append(UnCompress->Extension); - for (UnCompress = compressor.begin(); UnCompress != compressor.end(); ++UnCompress) - { - if (stat(file.c_str(), &Buf) == 0) - break; - } - - if (UnCompress == compressor.end()) - return _error->Errno("decompressor", "Unable to parse file"); - - *FileSize = Buf.st_size; - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - *fd = Pipe[1]; - - // The child.. - pid_t Pid = ExecFork(); - if (Pid == 0) - { - dup2(Pipe[1],STDOUT_FILENO); - SetCloseExec(STDOUT_FILENO, false); - - std::vector<char const*> Args; - Args.push_back(UnCompress->Binary.c_str()); - for (std::vector<std::string>::const_iterator a = UnCompress->UncompressArgs.begin(); - a != UnCompress->UncompressArgs.end(); ++a) - Args.push_back(a->c_str()); - Args.push_back("--stdout"); - Args.push_back(file.c_str()); - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - } - - // Wait for decompress to finish - if (ExecWait(Pid, UnCompress->Binary.c_str(), false) == false) - return false; - - return true; -} - /*}}}*/ // IndexCopy::CopyPackages - Copy the package files from the CD /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -142,24 +83,10 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) { string OrigPath = string(*I,CDROM.length()); - off_t FileSize = 0; // Open the package file - FileFd Pkg; - if (RealFileExists(*I + GetFileName()) == true) - { - Pkg.Open(*I + GetFileName(),FileFd::ReadOnly); - FileSize = Pkg.Size(); - } - else - { - int fd; - if (!DecompressFile(string(*I + GetFileName()), &fd, &FileSize)) - return _error->Errno("decompress","Decompress failed for %s", - string(*I + GetFileName()).c_str()); - Pkg.Fd(dup(fd)); - Pkg.Seek(0); - } + FileFd Pkg(*I + GetFileName(), FileFd::ReadOnly, FileFd::Extension); + off_t const FileSize = Pkg.Size(); pkgTagFile Parser(&Pkg); if (_error->PendingError() == true) @@ -868,23 +795,11 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) { string OrigPath = string(*I,CDROM.length()); - off_t FileSize = 0; - + // Open the package file - FileFd Pkg; - if (RealFileExists(*I) == true) - { - Pkg.Open(*I,FileFd::ReadOnly); - FileSize = Pkg.Size(); - } - else - { - int fd; - if (!DecompressFile(*I, &fd, &FileSize)) - return _error->Errno("decompress","Decompress failed for %s", (*I).c_str()); - Pkg.Fd(dup(fd)); - Pkg.Seek(0); - } + FileFd Pkg(*I, FileFd::ReadOnly, FileFd::Extension); + off_t const FileSize = Pkg.Size(); + pkgTagFile Parser(&Pkg); if (_error->PendingError() == true) return false; |