From 01366a44bf85a61d0a533e867922b6097bfb216d Mon Sep 17 00:00:00 2001 From: Steve McIntyre Date: Wed, 23 Nov 2011 19:34:58 +0100 Subject: factored out the decompressor code in IndexCopy::CopyPackages() and TranslationsCopy::CopyTranslations() into a single common function --- apt-pkg/cdrom.cc | 2 +- apt-pkg/indexcopy.cc | 174 ++++++++++++++++++++++++++++----------------------- 2 files changed, 95 insertions(+), 81 deletions(-) diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index 07983e44f..872879752 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -116,7 +116,7 @@ bool pkgCdrom::FindPackages(string CD, } } - // see if we find translatin indexes + // see if we find translation indices if (stat("i18n",&Buf) == 0) { D = opendir("i18n"); diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 4df018ef4..84f9fd420 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -37,8 +37,83 @@ using namespace std; - - +// DecompressFile - wrapper for decompressing gzip/bzip2/xz compressed files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool DecompressFile(string Filename, int *fd, off_t *FileSize) +{ + string CompressProg; + string CompressProgFind; + FileFd From; + struct stat Buf; + *fd = -1; + + if (stat((Filename + ".gz").c_str(), &Buf) == 0) + { + CompressProg = "gzip"; + CompressProgFind = "Dir::bin::gzip"; + From.Open(Filename + ".gz",FileFd::ReadOnly); + } + else if (stat((Filename + ".bz2").c_str(), &Buf) == 0) + { + CompressProg = "bzip2"; + CompressProgFind = "Dir::bin::bzip2"; + From.Open(Filename + ".bz2",FileFd::ReadOnly); + } + else if (stat((Filename + ".xz").c_str(), &Buf) == 0) + { + CompressProg = "xz"; + CompressProgFind = "Dir::bin::xz"; + From.Open(Filename + ".xz",FileFd::ReadOnly); + } + else + { + return _error->Errno("decompressor", "Unable to parse file"); + } + + if (_error->PendingError() == true) + return -1; + + *FileSize = Buf.st_size; + + // Get a temp file + FILE *tmp = tmpfile(); + if (tmp == 0) + return _error->Errno("tmpfile","Unable to create a tmp file"); + *fd = dup(fileno(tmp)); + fclose(tmp); + + // Fork decompressor + pid_t Process = fork(); + if (Process < 0) + return _error->Errno("fork","Couldn't fork to run decompressor"); + + // The child + if (Process == 0) + { + dup2(From.Fd(),STDIN_FILENO); + dup2(*fd,STDOUT_FILENO); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDOUT_FILENO,false); + + const char *Args[3]; + string Tmp = _config->Find(CompressProgFind, CompressProg); + Args[0] = Tmp.c_str(); + Args[1] = "-d"; + Args[2] = 0; + if(execvp(Args[0],(char **)Args)) + return(_error->Errno("decompressor","decompress failed")); + /* Should never get here */ + exit(100); + } + + // Wait for decompress to finish + if (ExecWait(Process,CompressProg.c_str(),false) == false) + return false; + + return true; +} + /*}}}*/ // IndexCopy::CopyPackages - Copy the package files from the CD /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -61,7 +136,9 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List, { struct stat Buf; if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 && - stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0) + stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0 && + stat(string(*I + GetFileName() + ".xz").c_str(),&Buf) != 0 && + stat(string(*I + GetFileName() + ".bz2").c_str(),&Buf) != 0) return _error->Errno("stat","Stat failed for %s", string(*I + GetFileName()).c_str()); TotalSize += Buf.st_size; @@ -85,46 +162,14 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List, } else { - FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - FileSize = From.Size(); - - // Get a temp file - FILE *tmp = tmpfile(); - if (tmp == 0) - return _error->Errno("tmpfile","Unable to create a tmp file"); - Pkg.Fd(dup(fileno(tmp))); - fclose(tmp); - - // Fork gzip - pid_t Process = fork(); - if (Process < 0) - return _error->Errno("fork","Couldn't fork gzip"); - - // The child - if (Process == 0) - { - dup2(From.Fd(),STDIN_FILENO); - dup2(Pkg.Fd(),STDOUT_FILENO); - SetCloseExec(STDIN_FILENO,false); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[3]; - string Tmp = _config->Find("Dir::bin::gzip","gzip"); - Args[0] = Tmp.c_str(); - Args[1] = "-d"; - Args[2] = 0; - execvp(Args[0],(char **)Args); - exit(100); - } - - // Wait for gzip to finish - if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false) - return _error->Error("gzip failed, perhaps the disk is full."); - + int fd; + if (!DecompressFile(string(*I + GetFileName()), &fd, &FileSize)) + return _error->Errno("decompress","Decompress failed for %s", + string(*I + GetFileName()).c_str()); + Pkg.Fd(dup(fd)); Pkg.Seek(0); } + pkgTagFile Parser(&Pkg); if (_error->PendingError() == true) return false; @@ -792,8 +837,11 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ for (vector::iterator I = List.begin(); I != List.end(); ++I) { struct stat Buf; + if (stat(string(*I).c_str(),&Buf) != 0 && - stat(string(*I + ".gz").c_str(),&Buf) != 0) + stat(string(*I + ".gz").c_str(),&Buf) != 0 && + stat(string(*I + ".bz2").c_str(),&Buf) != 0 && + stat(string(*I + ".xz").c_str(),&Buf) != 0) return _error->Errno("stat","Stat failed for %s", string(*I).c_str()); TotalSize += Buf.st_size; @@ -817,44 +865,10 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ } else { - FileFd From(*I + ".gz",FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - FileSize = From.Size(); - - // Get a temp file - FILE *tmp = tmpfile(); - if (tmp == 0) - return _error->Errno("tmpfile","Unable to create a tmp file"); - Pkg.Fd(dup(fileno(tmp))); - fclose(tmp); - - // Fork gzip - pid_t Process = fork(); - if (Process < 0) - return _error->Errno("fork","Couldn't fork gzip"); - - // The child - if (Process == 0) - { - dup2(From.Fd(),STDIN_FILENO); - dup2(Pkg.Fd(),STDOUT_FILENO); - SetCloseExec(STDIN_FILENO,false); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[3]; - string Tmp = _config->Find("Dir::bin::gzip","gzip"); - Args[0] = Tmp.c_str(); - Args[1] = "-d"; - Args[2] = 0; - execvp(Args[0],(char **)Args); - exit(100); - } - - // Wait for gzip to finish - if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false) - return _error->Error("gzip failed, perhaps the disk is full."); - + int fd; + if (!DecompressFile(*I, &fd, &FileSize)) + return _error->Errno("decompress","Decompress failed for %s", (*I).c_str()); + Pkg.Fd(dup(fd)); Pkg.Seek(0); } pkgTagFile Parser(&Pkg); -- cgit v1.2.3 From 1e67a553e86625052b140e9868c3b51759a71b1f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 23 Nov 2011 19:48:07 +0100 Subject: fix another hickup in the changelog --- debian/changelog | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/debian/changelog b/debian/changelog index 8b45cef39..f95edd60c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -33,6 +33,8 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low apt (0.8.16~exp8) experimental; urgency=low [ David Kalnischkies ] + * algorithms.cc: + - show a debug why a package was kept by ResolveByKeep() * apt-pkg/packagemanager.cc: - do not fail on unpacked packages in SmartUnPack, just don't shedule them for unpack, but do all checks and configure them @@ -228,14 +230,6 @@ apt (0.8.16~exp1) experimental; urgency=low -- Michael Vogt Wed, 29 Jun 2011 12:40:31 +0200 -apt (1.8.15.9+nmu1) unstable; urgency=low - - [ David Kalnischkies ] - * algorithms.cc: - - show a debug why a package was kept by ResolveByKeep() - - -- David Kalnischkies Mon, 17 Oct 2011 16:36:22 +0200 - apt (0.8.15.9) unstable; urgency=low [ David Kalnischkies ] -- cgit v1.2.3 From 78c9276d29172cbe72fc65ac56bde1627a8f86d1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 24 Nov 2011 00:53:47 +0100 Subject: use getCompressors() instead of getCompressorTypes() and use it everywhere to replace hardcoding of compressiontypes and compressors --- apt-pkg/cdrom.cc | 125 ++++++++++++++++++++++--------------------- apt-pkg/indexcopy.cc | 146 +++++++++++++++++++++++++-------------------------- 2 files changed, 137 insertions(+), 134 deletions(-) diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index 872879752..2c40c731d 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -82,66 +82,68 @@ bool pkgCdrom::FindPackages(string CD, /* Aha! We found some package files. We assume that everything under this dir is controlled by those package files so we don't look down anymore */ - std::vector types = APT::Configuration::getCompressionTypes(); - types.push_back(""); - for (std::vector::const_iterator t = types.begin(); - t != types.end(); ++t) - { - std::string filename = std::string("Packages"); - if ((*t).size() > 0) - filename.append("."+*t); - if (stat(filename.c_str(), &Buf) == 0) - { - List.push_back(CD); - - // Continue down if thorough is given - if (_config->FindB("APT::CDROM::Thorough",false) == false) - return true; - break; - } - } - for (std::vector::const_iterator t = types.begin(); - t != types.end(); ++t) - { - std::string filename = std::string("Sources"); - if ((*t).size() > 0) - filename.append("."+*t); - { - SList.push_back(CD); - - // Continue down if thorough is given - if (_config->FindB("APT::CDROM::Thorough",false) == false) - return true; - break; - } - } + std::vector const compressor = APT::Configuration::getCompressors(); + for (std::vector::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat(std::string("Packages").append(c->Extension).c_str(), &Buf) != 0) + continue; + + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found Packages in " << CD << std::endl; + List.push_back(CD); + + // Continue down if thorough is given + if (_config->FindB("APT::CDROM::Thorough",false) == false) + return true; + break; + } + for (std::vector::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat(std::string("Sources").append(c->Extension).c_str(), &Buf) != 0) + continue; + + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found Sources in " << CD << std::endl; + SList.push_back(CD); + + // Continue down if thorough is given + if (_config->FindB("APT::CDROM::Thorough",false) == false) + return true; + break; + } // see if we find translation indices - if (stat("i18n",&Buf) == 0) + if (DirectoryExists("i18n") == true) { D = opendir("i18n"); for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) { - if(strstr(Dir->d_name,"Translation") != NULL) + if(strncmp(Dir->d_name, "Translation-", strlen("Translation-")) != 0) + continue; + string file = Dir->d_name; + for (std::vector::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) { - if (_config->FindB("Debug::aptcdrom",false) == true) - std::clog << "found translations: " << Dir->d_name << "\n"; - string file = Dir->d_name; - for (std::vector::const_iterator t = types.begin(); - t != types.end(); ++t) - { - std::string needle = "." + *t; - if(file.substr(file.size()-needle.size()) == needle) - file = file.substr(0, file.size()-needle.size()); - TransList.push_back(CD+"i18n/"+ file); - break; - } + string fileext = flExtension(file); + if (file == fileext) + fileext.clear(); + else if (fileext.empty() == false) + fileext = "." + fileext; + + if (c->Extension == fileext) + { + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found translation " << Dir->d_name << " in " << CD << "i18n/" << std::endl; + TransList.push_back(CD + "i18n/" + file); + break; + } } } closedir(D); } - D = opendir("."); if (D == 0) return _error->Errno("opendir","Unable to read %s",CD.c_str()); @@ -278,24 +280,27 @@ bool pkgCdrom::DropRepeats(vector &List,const char *Name) { // Get a list of all the inodes ino_t *Inodes = new ino_t[List.size()]; - for (unsigned int I = 0; I != List.size(); I++) + for (unsigned int I = 0; I != List.size(); ++I) { struct stat Buf; - std::vector types = APT::Configuration::getCompressionTypes(); - types.push_back(""); - for (std::vector::const_iterator t = types.begin(); - t != types.end(); ++t) + bool found = false; + + std::vector const compressor = APT::Configuration::getCompressors(); + for (std::vector::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) { - std::string filename = List[I] + Name; - if ((*t).size() > 0) - filename.append("." + *t); + std::string filename = std::string(List[I]).append(Name).append(c->Extension); if (stat(filename.c_str(), &Buf) != 0) - _error->Errno("stat","Failed to stat %s%s",List[I].c_str(), - Name); - Inodes[I] = Buf.st_ino; + continue; + Inodes[I] = Buf.st_ino; + found = true; + break; } + + if (found == false) + _error->Errno("stat","Failed to stat %s%s",List[I].c_str(), Name); } - + if (_error->PendingError() == true) { delete[] Inodes; return false; diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 84f9fd420..e38fe3e45 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -37,80 +38,62 @@ using namespace std; -// DecompressFile - wrapper for decompressing gzip/bzip2/xz compressed files /*{{{*/ +// DecompressFile - wrapper for decompressing compressed files /*{{{*/ // --------------------------------------------------------------------- /* */ bool DecompressFile(string Filename, int *fd, off_t *FileSize) { - string CompressProg; - string CompressProgFind; - FileFd From; struct stat Buf; - *fd = -1; + *fd = -1; - if (stat((Filename + ".gz").c_str(), &Buf) == 0) + std::vector const compressor = APT::Configuration::getCompressors(); + std::vector::const_iterator UnCompress; + std::string file = std::string(Filename).append(UnCompress->Extension); + for (UnCompress = compressor.begin(); UnCompress != compressor.end(); ++UnCompress) { - CompressProg = "gzip"; - CompressProgFind = "Dir::bin::gzip"; - From.Open(Filename + ".gz",FileFd::ReadOnly); + if (stat(file.c_str(), &Buf) == 0) + break; } - else if (stat((Filename + ".bz2").c_str(), &Buf) == 0) - { - CompressProg = "bzip2"; - CompressProgFind = "Dir::bin::bzip2"; - From.Open(Filename + ".bz2",FileFd::ReadOnly); - } - else if (stat((Filename + ".xz").c_str(), &Buf) == 0) - { - CompressProg = "xz"; - CompressProgFind = "Dir::bin::xz"; - From.Open(Filename + ".xz",FileFd::ReadOnly); - } - else - { + + if (UnCompress == compressor.end()) return _error->Errno("decompressor", "Unable to parse file"); - } - if (_error->PendingError() == true) - return -1; - *FileSize = Buf.st_size; - - // Get a temp file - FILE *tmp = tmpfile(); - if (tmp == 0) - return _error->Errno("tmpfile","Unable to create a tmp file"); - *fd = dup(fileno(tmp)); - fclose(tmp); - - // Fork decompressor - pid_t Process = fork(); - if (Process < 0) - return _error->Errno("fork","Couldn't fork to run decompressor"); - - // The child - if (Process == 0) - { - dup2(From.Fd(),STDIN_FILENO); - dup2(*fd,STDOUT_FILENO); - SetCloseExec(STDIN_FILENO,false); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[3]; - string Tmp = _config->Find(CompressProgFind, CompressProg); - Args[0] = Tmp.c_str(); - Args[1] = "-d"; - Args[2] = 0; - if(execvp(Args[0],(char **)Args)) - return(_error->Errno("decompressor","decompress failed")); - /* Should never get here */ - exit(100); + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + *fd = Pipe[1]; + + // The child.. + pid_t Pid = ExecFork(); + if (Pid == 0) + { + dup2(Pipe[1],STDOUT_FILENO); + SetCloseExec(STDOUT_FILENO, false); + + std::vector Args; + Args.push_back(UnCompress->Binary.c_str()); + for (std::vector::const_iterator a = UnCompress->UncompressArgs.begin(); + a != UnCompress->UncompressArgs.end(); ++a) + Args.push_back(a->c_str()); + Args.push_back("--stdout"); + Args.push_back(file.c_str()); + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); } // Wait for decompress to finish - if (ExecWait(Process,CompressProg.c_str(),false) == false) + if (ExecWait(Pid, UnCompress->Binary.c_str(), false) == false) return false; - + return true; } /*}}}*/ @@ -132,17 +115,25 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List, // Prepare the progress indicator off_t TotalSize = 0; + std::vector const compressor = APT::Configuration::getCompressors(); for (vector::iterator I = List.begin(); I != List.end(); ++I) { struct stat Buf; - if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 && - stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0 && - stat(string(*I + GetFileName() + ".xz").c_str(),&Buf) != 0 && - stat(string(*I + GetFileName() + ".bz2").c_str(),&Buf) != 0) - return _error->Errno("stat","Stat failed for %s", - string(*I + GetFileName()).c_str()); + bool found = false; + std::string file = std::string(*I).append(GetFileName()); + for (std::vector::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat(std::string(file + c->Extension).c_str(), &Buf) != 0) + continue; + found = true; + break; + } + + if (found == false) + return _error->Errno("stat", "Stat failed for %s", file.c_str()); TotalSize += Buf.st_size; - } + } off_t CurrentSize = 0; unsigned int NotFound = 0; @@ -834,18 +825,25 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ // Prepare the progress indicator off_t TotalSize = 0; + std::vector const compressor = APT::Configuration::getCompressors(); for (vector::iterator I = List.begin(); I != List.end(); ++I) { struct stat Buf; - - if (stat(string(*I).c_str(),&Buf) != 0 && - stat(string(*I + ".gz").c_str(),&Buf) != 0 && - stat(string(*I + ".bz2").c_str(),&Buf) != 0 && - stat(string(*I + ".xz").c_str(),&Buf) != 0) - return _error->Errno("stat","Stat failed for %s", - string(*I).c_str()); + bool found = false; + std::string file = *I; + for (std::vector::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat(std::string(file + c->Extension).c_str(), &Buf) != 0) + continue; + found = true; + break; + } + + if (found == false) + return _error->Errno("stat", "Stat failed for %s", file.c_str()); TotalSize += Buf.st_size; - } + } off_t CurrentSize = 0; unsigned int NotFound = 0; -- cgit v1.2.3 From 257e8d668c044fb0e9ad8e4b53afd32e07404831 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 29 Nov 2011 12:14:31 +0100 Subject: split up the OpenMode into OpenMode and CompressionMode and provide ReadOnly, WriteOnly and ReadWrite as flags alongside the additional flags as decompression will be one-way later, but certain parts really depend on Write* openmodes being ReadWrite opens, so we will have to fail for those. --- apt-pkg/contrib/fileutl.cc | 102 +++++++++++++++++++++++---------------------- apt-pkg/contrib/fileutl.h | 30 +++++++++++-- 2 files changed, 79 insertions(+), 53 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 95058cbde..d5e7192e3 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -718,69 +718,72 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ -bool FileFd::Open(string FileName,OpenMode Mode, unsigned long Perms) +bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long Perms) { + if (Mode == ReadOnlyGzip) + return Open(FileName, ReadOnly, Gzip, Perms); Close(); Flags = AutoClose; - switch (Mode) + + int fileflags = 0; +#define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE + if_FLAGGED_SET(ReadWrite, O_RDWR); + else if_FLAGGED_SET(ReadOnly, O_RDONLY); + else if_FLAGGED_SET(WriteOnly, O_WRONLY); + else return _error->Error("No openmode provided in FileFd::Open for %s", FileName.c_str()); + + if_FLAGGED_SET(Create, O_CREAT); + if_FLAGGED_SET(Exclusive, O_EXCL); + else if_FLAGGED_SET(Atomic, O_EXCL); + if_FLAGGED_SET(Empty, O_TRUNC); +#undef if_FLAGGED_SET + + if ((Mode & Atomic) == Atomic) + { + Flags |= Replace; + char *name = strdup((FileName + ".XXXXXX").c_str()); + TemporaryFileName = string(mktemp(name)); + free(name); + } + else if ((Mode & (Exclusive | Create)) == (Exclusive | Create)) + { + // for atomic, this will be done by rename in Close() + unlink(FileName.c_str()); + } + if ((Mode & Empty) == Empty) { - case ReadOnly: - iFd = open(FileName.c_str(),O_RDONLY); - break; + struct stat Buf; + if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) + unlink(FileName.c_str()); + } - case ReadOnlyGzip: - iFd = open(FileName.c_str(),O_RDONLY); - if (iFd > 0) { - gz = gzdopen (iFd, "r"); - if (gz == NULL) { - close (iFd); - iFd = -1; - } - } - break; - - case WriteAtomic: - { - Flags |= Replace; - char *name = strdup((FileName + ".XXXXXX").c_str()); - TemporaryFileName = string(mktemp(name)); - iFd = open(TemporaryFileName.c_str(),O_RDWR | O_CREAT | O_EXCL,Perms); - free(name); - break; - } + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), fileflags, Perms); + else + iFd = open(FileName.c_str(), fileflags, Perms); - case WriteEmpty: + if (iFd != -1 && Compress == Gzip) + { + gz = gzdopen (iFd, "r"); + if (gz == NULL) { - struct stat Buf; - if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) - unlink(FileName.c_str()); - iFd = open(FileName.c_str(),O_RDWR | O_CREAT | O_TRUNC,Perms); - break; + close (iFd); + iFd = -1; } - - case WriteExists: - iFd = open(FileName.c_str(),O_RDWR); - break; - - case WriteAny: - iFd = open(FileName.c_str(),O_RDWR | O_CREAT,Perms); - break; - - case WriteTemp: - unlink(FileName.c_str()); - iFd = open(FileName.c_str(),O_RDWR | O_CREAT | O_EXCL,Perms); - break; - } + } - if (iFd < 0) + if (iFd == -1) return _error->Errno("open",_("Could not open file %s"),FileName.c_str()); - + this->FileName = FileName; SetCloseExec(iFd,true); return true; } - -bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose) + /*}}}*/ +// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose) { Close(); Flags = (AutoClose) ? FileFd::AutoClose : 0; @@ -1031,6 +1034,7 @@ bool FileFd::Close() Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str()); FileName = TemporaryFileName; // for the unlink() below. + TemporaryFileName.clear(); } iFd = -1; diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 0d0451a46..fa8f92272 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -44,8 +44,24 @@ class FileFd gzFile gz; public: - enum OpenMode {ReadOnly,WriteEmpty,WriteExists,WriteAny,WriteTemp,ReadOnlyGzip, - WriteAtomic}; + enum OpenMode { + ReadOnly = (1 << 0), + WriteOnly = (1 << 1), + ReadWrite = ReadOnly | WriteOnly, + + Create = (1 << 2), + Exclusive = (1 << 3), + Atomic = Exclusive | (1 << 4), + Empty = (1 << 5), + + WriteEmpty = ReadWrite | Create | Empty, + WriteExists = ReadWrite, + WriteAny = ReadWrite | Create, + WriteTemp = ReadWrite | Create | Exclusive, + ReadOnlyGzip, + WriteAtomic = ReadWrite | Create | Atomic + }; + enum CompressMode { Auto, None, Gzip, Bzip2, Lzma, Xz }; inline bool Read(void *To,unsigned long long Size,bool AllowEof) { @@ -77,8 +93,14 @@ class FileFd return T; } - bool Open(std::string FileName,OpenMode Mode,unsigned long Perms = 0666); - bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false); + bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long Perms = 0666); + inline bool Open(std::string const &FileName,OpenMode Mode,unsigned long Perms = 0666) { + return Open(FileName, Mode, None, Perms); + }; + bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); + inline bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false) { + return OpenDescriptor(Fd, Mode, None, AutoClose); + }; bool Close(); bool Sync(); -- cgit v1.2.3 From a7e052eae64486a31595bc9b3f4a1c02c13256e3 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 30 Nov 2011 16:37:09 +0100 Subject: ensure that we exit instead of proceed in execution after the trap (the execution leads to hard failures anyway) --- test/integration/framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework b/test/integration/framework index b55f793a4..2ea1844f0 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -102,7 +102,7 @@ aptitude() { addtrap() { CURRENTTRAP="$CURRENTTRAP $1" - trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM + trap "$CURRENTTRAP exit;" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM } setupenvironment() { -- cgit v1.2.3 From 8e16d8c39447809506a8cd8e6f88cae3c168f82d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 Dec 2011 17:35:54 +0100 Subject: * cmdline/apt-config.cc: - dump the APT::Compressor settings correctly and completely --- cmdline/apt-config.cc | 16 ++++++++++++++++ debian/changelog | 11 +++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc index 94f6ee9b0..47bedfe3f 100644 --- a/cmdline/apt-config.cc +++ b/cmdline/apt-config.cc @@ -134,6 +134,22 @@ int main(int argc,const char *argv[]) /*{{{*/ for (std::vector::const_iterator a = archs.begin(); a != archs.end(); ++a) _config->Set("APT::Architectures::", *a); + std::vector const compressors = APT::Configuration::getCompressors(); + _config->Clear("APT::Compressor"); + string conf = "APT::Compressor::"; + for (std::vector::const_iterator c = compressors.begin(); c != compressors.end(); ++c) + { + string comp = conf + c->Name + "::"; + _config->Set(comp + "Name", c->Name); + _config->Set(comp + "Extension", c->Extension); + _config->Set(comp + "Binary", c->Binary); + _config->Set(std::string(comp + "Cost").c_str(), c->Cost); + for (std::vector::const_iterator a = c->CompressArgs.begin(); a != c->CompressArgs.end(); ++a) + _config->Set(comp + "CompressArg::", *a); + for (std::vector::const_iterator a = c->UncompressArgs.begin(); a != c->UncompressArgs.end(); ++a) + _config->Set(comp + "UncompressArg::", *a); + } + // Match the operation CmdL.DispatchArg(Cmds); diff --git a/debian/changelog b/debian/changelog index f95edd60c..685774a8f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,16 +1,17 @@ + apt (0.8.16~exp9) UNRELEASED; urgency=low [ Julian Andres Klode ] * apt-pkg/cdrom.cc: - Accept .bz2, .xz files in addition to .gz files (Closes: #649451) - + [ Michael Vogt ] * apt-pkg/cdrom.cc: - use aptconfiguration to get the supported compression types * debian/control: - bump debhelper build-dep to debhelper (>= 8.1.3~) - set libapt-pkg-dev to multi-arch: same too - + [ Colin Watson ] * Convert libapt-pkg4.12 and libapt-inst1.4 to Multi-Arch: same. @@ -27,8 +28,10 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low * cmdline/apt-get.cc: - ignore foreign architectures if we check if a provides has only one resolver as it's basically the same for the user, so no need to choose - - -- Julian Andres Klode Mon, 21 Nov 2011 18:46:36 +0100 + * cmdline/apt-config.cc: + - dump the APT::Compressor settings correctly and completely + + -- David Kalnischkies Sat, 10 Dec 2011 17:34:50 +0100 apt (0.8.16~exp8) experimental; urgency=low -- cgit v1.2.3 From 468720c59fcf48b20332cdb7b601b2b0d7cbbfbb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 Dec 2011 19:31:36 +0100 Subject: enable FileFd to guess the compressor based on the filename if requested or to search for compressed silbings of the given filename and use this guessing instead of hardcoding Gzip compression --- apt-pkg/contrib/fileutl.cc | 129 ++++++++++++++++++++++++++++++++++++++------ apt-pkg/contrib/fileutl.h | 14 +++-- apt-pkg/deb/debindexfile.cc | 13 ++--- apt-pkg/deb/debrecords.cc | 2 +- apt-pkg/deb/debsrcrecords.h | 2 +- cmdline/apt-cache.cc | 2 +- methods/gzip.cc | 2 +- methods/rred.cc | 2 +- 8 files changed, 135 insertions(+), 31 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index d5e7192e3..1cb3fab1e 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -725,6 +726,11 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned Close(); Flags = AutoClose; + if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) + return _error->Error("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); + if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) + return _error->Error("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); + int fileflags = 0; #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE if_FLAGGED_SET(ReadWrite, O_RDWR); @@ -738,6 +744,70 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned if_FLAGGED_SET(Empty, O_TRUNC); #undef if_FLAGGED_SET + // FIXME: Denote inbuilt compressors somehow - as we don't need to have the binaries for them + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + if (Compress == Auto) + { + Compress = None; + for (; compressor != compressors.end(); ++compressor) + { + std::string file = std::string(FileName).append(compressor->Extension); + if (FileExists(file) == false) + continue; + FileName = file; + if (compressor->Binary == ".") + Compress = None; + else + Compress = Extension; + break; + } + } + else if (Compress == Extension) + { + Compress = None; + std::string ext = flExtension(FileName); + if (ext != FileName) + { + ext = "." + ext; + for (; compressor != compressors.end(); ++compressor) + if (ext == compressor->Extension) + break; + } + } + else if (Compress != None) + { + std::string name; + switch (Compress) + { + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + default: return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end() && name != "gzip") + return _error->Error("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + + // if we have them, use inbuilt compressors instead of forking + if (compressor != compressors.end()) + { + if (compressor->Name == "gzip") + { + Compress = Gzip; + compressor = compressors.end(); + } + else if (compressor->Name == "." || Compress == None) + { + Compress = None; + compressor = compressors.end(); + } + } + if ((Mode & Atomic) == Atomic) { Flags |= Replace; @@ -757,18 +827,27 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned unlink(FileName.c_str()); } - if (TemporaryFileName.empty() == false) - iFd = open(TemporaryFileName.c_str(), fileflags, Perms); - else - iFd = open(FileName.c_str(), fileflags, Perms); + if (compressor != compressors.end()) + { + if ((Mode & ReadWrite) == ReadWrite) + _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); - if (iFd != -1 && Compress == Gzip) + _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + } + else { - gz = gzdopen (iFd, "r"); - if (gz == NULL) + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), fileflags, Perms); + else + iFd = open(FileName.c_str(), fileflags, Perms); + + if (iFd != -1) { - close (iFd); - iFd = -1; + if (OpenInternDescriptor(Mode, Compress) == false) + { + close (iFd); + iFd = -1; + } } } @@ -788,17 +867,33 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A Close(); Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; - if (Mode == ReadOnlyGzip) { - gz = gzdopen (iFd, "r"); - if (gz == NULL) { - if (AutoClose) - close (iFd); - return _error->Errno("gzdopen",_("Could not open file descriptor %d"), - Fd); - } + if (OpenInternDescriptor(Mode, Compress) == false) + { + if (AutoClose) + close (iFd); + return _error->Errno("gzdopen",_("Could not open file descriptor %d"), Fd); } this->FileName = ""; return true; +} +bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) +{ + if (Compress == None) + return true; + else if (Compress == Gzip) + { + if ((Mode & ReadWrite) == ReadWrite) + gz = gzdopen(iFd, "r+"); + else if ((Mode & WriteOnly) == WriteOnly) + gz = gzdopen(iFd, "w"); + else + gz = gzdopen (iFd, "r"); + if (gz == NULL) + return false; + } + else + return false; + return true; } /*}}}*/ // FileFd::~File - Closes the file /*{{{*/ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index fa8f92272..59a9d97e3 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -61,7 +61,7 @@ class FileFd ReadOnlyGzip, WriteAtomic = ReadWrite | Create | Atomic }; - enum CompressMode { Auto, None, Gzip, Bzip2, Lzma, Xz }; + enum CompressMode { Auto = 'A', None = 'N', Extension = 'E', Gzip = 'G', Bzip2 = 'B', Lzma = 'L', Xz = 'X' }; inline bool Read(void *To,unsigned long long Size,bool AllowEof) { @@ -94,7 +94,7 @@ class FileFd } bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long Perms = 0666); - inline bool Open(std::string const &FileName,OpenMode Mode,unsigned long Perms = 0666) { + inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long Perms = 0666) { return Open(FileName, Mode, None, Perms); }; bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); @@ -118,11 +118,19 @@ class FileFd FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), gz(NULL) { - Open(FileName,Mode,Perms); + Open(FileName,Mode, None, Perms); + }; + FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : + iFd(-1), Flags(0), gz(NULL) + { + Open(FileName,Mode, Compress, Perms); }; FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose), gz(NULL) {}; FileFd(int Fd,bool) : iFd(Fd), Flags(0), gz(NULL) {}; virtual ~FileFd(); + + private: + bool OpenInternDescriptor(OpenMode Mode, CompressMode Compress); }; bool RunScripts(const char *Cnf); diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 27c1f7f32..d9c448598 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -159,7 +159,7 @@ unsigned long debSourcesIndex::Size() const /* we need to ignore errors here; if the lists are absent, just return 0 */ _error->PushToStack(); - FileFd f = FileFd (IndexFile("Sources"), FileFd::ReadOnlyGzip); + FileFd f = FileFd (IndexFile("Sources"), FileFd::ReadOnly, FileFd::Extension); if (!f.Failed()) size = f.Size(); @@ -288,7 +288,7 @@ unsigned long debPackagesIndex::Size() const /* we need to ignore errors here; if the lists are absent, just return 0 */ _error->PushToStack(); - FileFd f = FileFd (IndexFile("Packages"), FileFd::ReadOnlyGzip); + FileFd f = FileFd (IndexFile("Packages"), FileFd::ReadOnly, FileFd::Extension); if (!f.Failed()) size = f.Size(); @@ -305,7 +305,7 @@ unsigned long debPackagesIndex::Size() const bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { string PackageFile = IndexFile("Packages"); - FileFd Pkg(PackageFile,FileFd::ReadOnlyGzip); + FileFd Pkg(PackageFile,FileFd::ReadOnly, FileFd::Extension); debListParser Parser(&Pkg, Architecture); if (_error->PendingError() == true) @@ -319,6 +319,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator File = Gen.GetCurFile(); pkgCacheGenerator::Dynamic DynFile(File); + // FIXME: Get this info from FileFd instead struct stat St; if (fstat(Pkg.Fd(),&St) != 0) return _error->Errno("fstat","Failed to stat"); @@ -489,7 +490,7 @@ unsigned long debTranslationsIndex::Size() const /* we need to ignore errors here; if the lists are absent, just return 0 */ _error->PushToStack(); - FileFd f = FileFd (IndexFile(Language), FileFd::ReadOnlyGzip); + FileFd f = FileFd (IndexFile(Language), FileFd::ReadOnly, FileFd::Extension); if (!f.Failed()) size = f.Size(); @@ -509,7 +510,7 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const string TranslationFile = IndexFile(Language); if (FileExists(TranslationFile)) { - FileFd Trans(TranslationFile,FileFd::ReadOnlyGzip); + FileFd Trans(TranslationFile,FileFd::ReadOnly, FileFd::Extension); debListParser TransParser(&Trans); if (_error->PendingError() == true) return false; @@ -590,7 +591,7 @@ unsigned long debStatusIndex::Size() const /* */ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { - FileFd Pkg(File,FileFd::ReadOnlyGzip); + FileFd Pkg(File,FileFd::ReadOnly, FileFd::Extension); if (_error->PendingError() == true) return false; debListParser Parser(&Pkg); diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc index 1afa7b74d..184c07c33 100644 --- a/apt-pkg/deb/debrecords.cc +++ b/apt-pkg/deb/debrecords.cc @@ -25,7 +25,7 @@ using std::string; // --------------------------------------------------------------------- /* */ debRecordParser::debRecordParser(string FileName,pkgCache &Cache) : - File(FileName,FileFd::ReadOnlyGzip), + File(FileName,FileFd::ReadOnly, FileFd::Extension), Tags(&File, std::max(Cache.Head().MaxVerFileSize, Cache.Head().MaxDescFileSize) + 200) { diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index 4c8d03224..5d2a67f4f 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -50,7 +50,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser virtual bool Files(std::vector &F); debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) - : Parser(Index), Fd(File,FileFd::ReadOnlyGzip), Tags(&Fd,102400), + : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400), Buffer(NULL) {} virtual ~debSrcRecordParser(); }; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 65d7b0ccd..1cd5080cc 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1143,7 +1143,7 @@ bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V) return _error->Error(_("Package file %s is out of sync."),I.FileName()); FileFd PkgF; - if (PkgF.Open(I.FileName(), FileFd::ReadOnlyGzip) == false) + if (PkgF.Open(I.FileName(), FileFd::ReadOnly, FileFd::Extension) == false) return false; // Read the record diff --git a/methods/gzip.cc b/methods/gzip.cc index a51497948..6ab6548ef 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -48,7 +48,7 @@ bool GzipMethod::Fetch(FetchItem *Itm) URIStart(Res); // Open the source and destination files - FileFd From(Path,FileFd::ReadOnlyGzip); + FileFd From(Path,FileFd::ReadOnly, FileFd::Gzip); if(From.FileSize() == 0) return _error->Error(_("Empty files can't be valid archives")); diff --git a/methods/rred.cc b/methods/rred.cc index ef00fcaa3..56ad8220b 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -489,7 +489,7 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ // Open the source and destination files (the d'tor of FileFd will do // the cleanup/closing of the fds) FileFd From(Path,FileFd::ReadOnly); - FileFd Patch(Path+".ed",FileFd::ReadOnlyGzip); + FileFd Patch(Path+".ed",FileFd::ReadOnly, FileFd::Gzip); FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) -- cgit v1.2.3 From 76a763e1f842543a53bc28db681d963d0fc4ae12 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 Dec 2011 20:03:49 +0100 Subject: * apt-pkg/contrib/fileutl.{h,cc}: - implement a ModificationTime method for FileFd --- apt-pkg/contrib/fileutl.cc | 14 ++++++++++++++ apt-pkg/contrib/fileutl.h | 1 + apt-pkg/deb/debindexfile.cc | 19 ++++++------------- apt-pkg/edsp/edspindexfile.cc | 7 ++----- debian/changelog | 5 +++-- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 1cb3fab1e..83b68e796 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1106,6 +1106,20 @@ unsigned long long FileFd::Size() return size; } /*}}}*/ +// FileFd::ModificationTime - Return the time of last touch /*{{{*/ +// --------------------------------------------------------------------- +/* */ +time_t FileFd::ModificationTime() +{ + struct stat Buf; + if (fstat(iFd,&Buf) != 0) + { + _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); + return 0; + } + return Buf.st_mtime; +} + /*}}}*/ // FileFd::Close - Close the file if the close flag is set /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 59a9d97e3..8f2d7a0a0 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -78,6 +78,7 @@ class FileFd unsigned long long Tell(); unsigned long long Size(); unsigned long long FileSize(); + time_t ModificationTime(); /* You want to use 'unsigned long long' if you are talking about a file to be able to support large files (>2 or >4 GB) properly. diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index d9c448598..2635d52c8 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -319,12 +319,8 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator File = Gen.GetCurFile(); pkgCacheGenerator::Dynamic DynFile(File); - // FIXME: Get this info from FileFd instead - struct stat St; - if (fstat(Pkg.Fd(),&St) != 0) - return _error->Errno("fstat","Failed to stat"); - File->Size = St.st_size; - File->mtime = St.st_mtime; + File->Size = Pkg.FileSize(); + File->mtime = Pkg.ModificationTime(); if (Gen.MergeList(Parser) == false) return _error->Error("Problem with MergeList %s",PackageFile.c_str()); @@ -522,11 +518,8 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator TransFile = Gen.GetCurFile(); - struct stat TransSt; - if (fstat(Trans.Fd(),&TransSt) != 0) - return _error->Errno("fstat","Failed to stat"); - TransFile->Size = TransSt.st_size; - TransFile->mtime = TransSt.st_mtime; + TransFile->Size = Trans.FileSize(); + TransFile->mtime = Trans.ModificationTime(); if (Gen.MergeList(TransParser) == false) return _error->Error("Problem with MergeList %s",TranslationFile.c_str()); @@ -608,8 +601,8 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const struct stat St; if (fstat(Pkg.Fd(),&St) != 0) return _error->Errno("fstat","Failed to stat"); - CFile->Size = St.st_size; - CFile->mtime = St.st_mtime; + CFile->Size = Pkg.FileSize(); + CFile->mtime = Pkg.ModificationTime(); CFile->Archive = Gen.WriteUniqString("now"); if (Gen.MergeList(Parser) == false) diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc index 058cef636..5d824f9cb 100644 --- a/apt-pkg/edsp/edspindexfile.cc +++ b/apt-pkg/edsp/edspindexfile.cc @@ -49,11 +49,8 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); - struct stat St; - if (fstat(Pkg.Fd(),&St) != 0) - return _error->Errno("fstat","Failed to stat"); - CFile->Size = St.st_size; - CFile->mtime = St.st_mtime; + CFile->Size = Pkg.FileSize(); + CFile->mtime = Pkg.ModificationTime(); CFile->Archive = Gen.WriteUniqString("edsp::scenario"); if (Gen.MergeList(Parser) == false) diff --git a/debian/changelog b/debian/changelog index 685774a8f..a401d61a0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,3 @@ - apt (0.8.16~exp9) UNRELEASED; urgency=low [ Julian Andres Klode ] @@ -30,8 +29,10 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low resolver as it's basically the same for the user, so no need to choose * cmdline/apt-config.cc: - dump the APT::Compressor settings correctly and completely + * apt-pkg/contrib/fileutl.{h,cc}: + - implement a ModificationTime method for FileFd - -- David Kalnischkies Sat, 10 Dec 2011 17:34:50 +0100 + -- David Kalnischkies Sat, 10 Dec 2011 20:02:29 +0100 apt (0.8.16~exp8) experimental; urgency=low -- cgit v1.2.3 From 4e86b003bad2e6146c9fe8be492bdc9d212bcd74 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 00:41:50 +0100 Subject: strip the extension of the translation file before storing it in the list (regression from compression rewrite; found by Steve McIntyre, thanks!) --- apt-pkg/cdrom.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index 2c40c731d..026094a6e 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -136,6 +136,7 @@ bool pkgCdrom::FindPackages(string CD, { if (_config->FindB("Debug::aptcdrom",false) == true) std::clog << "Found translation " << Dir->d_name << " in " << CD << "i18n/" << std::endl; + file.erase(file.size() - fileext.size()); TransList.push_back(CD + "i18n/" + file); break; } -- cgit v1.2.3 From 711078ae18df09ca4f0c371c071c59458fad3918 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 00:58:35 +0100 Subject: use fileutl exists-functions instead of doing the stat'ing by hand --- apt-pkg/cdrom.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index 026094a6e..f5c19a4d6 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -58,15 +58,14 @@ bool pkgCdrom::FindPackages(string CD, return _error->Errno("chdir","Unable to change to %s",CD.c_str()); // Look for a .disk subdirectory - struct stat Buf; - if (stat(".disk",&Buf) == 0) + if (DirectoryExists(".disk") == true) { if (InfoDir.empty() == true) InfoDir = CD + ".disk/"; } // Don't look into directories that have been marked to ingore. - if (stat(".aptignr",&Buf) == 0) + if (RealFileExists(".aptignr") == true) return true; @@ -74,7 +73,7 @@ bool pkgCdrom::FindPackages(string CD, under a Packages/Source file are in control of that file and stops the scanning */ - if (stat("Release.gpg",&Buf) == 0) + if (RealFileExists("Release.gpg") == true) { SigList.push_back(CD); } @@ -86,7 +85,7 @@ bool pkgCdrom::FindPackages(string CD, for (std::vector::const_iterator c = compressor.begin(); c != compressor.end(); ++c) { - if (stat(std::string("Packages").append(c->Extension).c_str(), &Buf) != 0) + if (RealFileExists(std::string("Packages").append(c->Extension).c_str()) == false) continue; if (_config->FindB("Debug::aptcdrom",false) == true) @@ -101,7 +100,7 @@ bool pkgCdrom::FindPackages(string CD, for (std::vector::const_iterator c = compressor.begin(); c != compressor.end(); ++c) { - if (stat(std::string("Sources").append(c->Extension).c_str(), &Buf) != 0) + if (RealFileExists(std::string("Sources").append(c->Extension).c_str()) == false) continue; if (_config->FindB("Debug::aptcdrom",false) == true) -- cgit v1.2.3 From 212080b87daa25944259287a5a625e63dd696ff0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 01:30:45 +0100 Subject: * apt-pkg/cdrom.cc: - support InRelease files on cdrom --- apt-pkg/cdrom.cc | 4 ++-- apt-pkg/indexcopy.cc | 26 +++++++++++++++++++++----- debian/changelog | 4 +++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index f5c19a4d6..d9ecdf4f6 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -68,12 +68,11 @@ bool pkgCdrom::FindPackages(string CD, if (RealFileExists(".aptignr") == true) return true; - /* Check _first_ for a signature file as apt-cdrom assumes that all files under a Packages/Source file are in control of that file and stops the scanning */ - if (RealFileExists("Release.gpg") == true) + if (RealFileExists("Release.gpg") == true || RealFileExists("InRelease") == true) { SigList.push_back(CD); } @@ -718,6 +717,7 @@ bool pkgCdrom::Add(pkgCdromStatus *log) /*{{{*/ DropRepeats(List,"Packages"); DropRepeats(SourceList,"Sources"); DropRepeats(SigList,"Release.gpg"); + DropRepeats(SigList,"InRelease"); DropRepeats(TransList,""); if(log != NULL) { msg.str(""); diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index e38fe3e45..f6457aa39 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -638,13 +638,19 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, string const releasegpg = *I+"Release.gpg"; string const release = *I+"Release"; + string const inrelease = *I+"InRelease"; + bool useInRelease = true; // a Release.gpg without a Release should never happen - if(RealFileExists(release) == false) + if (RealFileExists(inrelease) == true) + ; + else if(RealFileExists(release) == false || RealFileExists(releasegpg) == false) { delete MetaIndex; continue; } + else + useInRelease = false; pid_t pid = ExecFork(); if(pid < 0) { @@ -652,11 +658,16 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, return false; } if(pid == 0) - RunGPGV(release, releasegpg); + { + if (useInRelease == true) + RunGPGV(inrelease, inrelease); + else + RunGPGV(release, releasegpg); + } if(!ExecWait(pid, "gpgv")) { _error->Warning("Signature verification failed for: %s", - releasegpg.c_str()); + (useInRelease ? inrelease.c_str() : releasegpg.c_str())); // something went wrong, don't copy the Release.gpg // FIXME: delete any existing gpg file? continue; @@ -686,8 +697,13 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, delete MetaIndex; // everything was fine, copy the Release and Release.gpg file - CopyMetaIndex(CDROM, Name, prefix, "Release"); - CopyMetaIndex(CDROM, Name, prefix, "Release.gpg"); + if (useInRelease == true) + CopyMetaIndex(CDROM, Name, prefix, "InRelease"); + else + { + CopyMetaIndex(CDROM, Name, prefix, "Release"); + CopyMetaIndex(CDROM, Name, prefix, "Release.gpg"); + } } return true; diff --git a/debian/changelog b/debian/changelog index a401d61a0..3e4a8fa3c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -31,8 +31,10 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low - dump the APT::Compressor settings correctly and completely * apt-pkg/contrib/fileutl.{h,cc}: - implement a ModificationTime method for FileFd + * apt-pkg/cdrom.cc: + - support InRelease files on cdrom - -- David Kalnischkies Sat, 10 Dec 2011 20:02:29 +0100 + -- David Kalnischkies Sun, 11 Dec 2011 01:30:12 +0100 apt (0.8.16~exp8) experimental; urgency=low -- cgit v1.2.3 From 2c405a44a0e4ff4c6f40e2521a55811179c87ec3 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 02:55:20 +0100 Subject: add a testcase for FindPackages() to better validate that cdrom should work. Unfortunately it's hard to do an automated integration test with cd, so we test this method in isolation which tries to find Indexes and dropping of duplications with DropRepeats() --- apt-pkg/cdrom.cc | 23 ++++++---- test/libapt/cdromfindpackages_test.cc | 86 +++++++++++++++++++++++++++++++++++ test/libapt/makefile | 6 +++ test/libapt/run-tests | 29 ++++++++++++ 4 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 test/libapt/cdromfindpackages_test.cc diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index d9ecdf4f6..4462d4e24 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -277,6 +277,7 @@ bool pkgCdrom::DropBinaryArch(vector &List) /* Here we go and stat every file that we found and strip dup inodes. */ bool pkgCdrom::DropRepeats(vector &List,const char *Name) { + bool couldFindAllFiles = true; // Get a list of all the inodes ino_t *Inodes = new ino_t[List.size()]; for (unsigned int I = 0; I != List.size(); ++I) @@ -297,21 +298,22 @@ bool pkgCdrom::DropRepeats(vector &List,const char *Name) } if (found == false) - _error->Errno("stat","Failed to stat %s%s",List[I].c_str(), Name); + { + _error->Errno("stat","Failed to stat %s%s",List[I].c_str(), Name); + couldFindAllFiles = false; + Inodes[I] = 0; + } } - if (_error->PendingError() == true) { - delete[] Inodes; - return false; - } - // Look for dups for (unsigned int I = 0; I != List.size(); I++) { + if (Inodes[I] == 0) + continue; for (unsigned int J = I+1; J < List.size(); J++) { // No match - if (Inodes[J] != Inodes[I]) + if (Inodes[J] == 0 || Inodes[J] != Inodes[I]) continue; // We score the two paths.. and erase one @@ -337,7 +339,7 @@ bool pkgCdrom::DropRepeats(vector &List,const char *Name) List.erase(List.begin()+I); } - return true; + return couldFindAllFiles; } /*}}}*/ // ReduceSourceList - Takes the path list and reduces it /*{{{*/ @@ -716,8 +718,13 @@ bool pkgCdrom::Add(pkgCdromStatus *log) /*{{{*/ DropBinaryArch(List); DropRepeats(List,"Packages"); DropRepeats(SourceList,"Sources"); + // FIXME: We ignore stat() errors here as we usually have only one of those in use + // This has little potencial to drop 'valid' stat() errors as we know that one of these + // files need to exist, but it would be better if we would check it here + _error->PushToStack(); DropRepeats(SigList,"Release.gpg"); DropRepeats(SigList,"InRelease"); + _error->RevertToStack(); DropRepeats(TransList,""); if(log != NULL) { msg.str(""); diff --git a/test/libapt/cdromfindpackages_test.cc b/test/libapt/cdromfindpackages_test.cc new file mode 100644 index 000000000..e9f5a51b0 --- /dev/null +++ b/test/libapt/cdromfindpackages_test.cc @@ -0,0 +1,86 @@ +#include +#include + +#include +#include +#include + +#include "assert.h" + +class Cdrom : public pkgCdrom { + public: + bool FindPackages(std::string const &CD, + std::vector &List, + std::vector &SList, + std::vector &SigList, + std::vector &TransList, + std::string &InfoDir) { + bool const result = pkgCdrom::FindPackages(CD, List, SList, SigList, TransList, InfoDir, NULL, 0); + std::sort(List.begin(), List.end()); + std::sort(SList.begin(), SList.end()); + std::sort(SigList.begin(), SigList.end()); + std::sort(TransList.begin(), TransList.end()); + return result; + } + + bool DropRepeats(std::vector &List, char const *Name) { + return pkgCdrom::DropRepeats(List, Name); + } +}; + +int main(int argc, char const *argv[]) { + if (argc != 2) { + std::cout << "One parameter expected - given " << argc << std::endl; + return 100; + } + + Cdrom cd; + std::vector Packages, Sources, Signatur, Translation; + std::string InfoDir; + std::string path = argv[1]; + equals(true, cd.FindPackages(path, Packages, Sources, Signatur, Translation, InfoDir)); + equals(4, Packages.size()); + equals(path + "/dists/sid/main/binary-i386/", Packages[0]); + equals(path + "/dists/stable/contrib/binary-amd64/", Packages[1]); + equals(path + "/dists/stable/main/binary-i386/", Packages[2]); + equals(path + "/dists/unstable/main/binary-i386/", Packages[3]); + equals(3, Sources.size()); + equals(path + "/dists/sid/main/source/", Sources[0]); + equals(path + "/dists/stable/main/source/", Sources[1]); + equals(path + "/dists/unstable/main/source/", Sources[2]); + equals(3, Signatur.size()); + equals(path + "/dists/sid/", Signatur[0]); + equals(path + "/dists/stable/", Signatur[1]); + equals(path + "/dists/unstable/", Signatur[2]); + equals(4, Translation.size()); + equals(path + "/dists/sid/main/i18n/Translation-de", Translation[0]); + equals(path + "/dists/sid/main/i18n/Translation-en", Translation[1]); + equals(path + "/dists/unstable/main/i18n/Translation-de", Translation[2]); + equals(path + "/dists/unstable/main/i18n/Translation-en", Translation[3]); + equals(path + "/.disk/", InfoDir); + + cd.DropRepeats(Packages, "Packages"); + cd.DropRepeats(Sources, "Sources"); + _error->PushToStack(); + cd.DropRepeats(Signatur, "InRelease"); + cd.DropRepeats(Signatur, "Release.gpg"); + _error->RevertToStack(); + _error->DumpErrors(); + cd.DropRepeats(Translation, ""); + + equals(3, Packages.size()); + equals(path + "/dists/stable/contrib/binary-amd64/", Packages[0]); + equals(path + "/dists/stable/main/binary-i386/", Packages[1]); + equals(path + "/dists/unstable/main/binary-i386/", Packages[2]); + equals(2, Sources.size()); + equals(path + "/dists/stable/main/source/", Sources[0]); + equals(path + "/dists/unstable/main/source/", Sources[1]); + equals(2, Signatur.size()); + equals(path + "/dists/stable/", Signatur[0]); + equals(path + "/dists/unstable/", Signatur[1]); + equals(2, Translation.size()); + equals(path + "/dists/unstable/main/i18n/Translation-de", Translation[0]); + equals(path + "/dists/unstable/main/i18n/Translation-en", Translation[1]); + + return 0; +} diff --git a/test/libapt/makefile b/test/libapt/makefile index d3dddaeed..1952051e2 100644 --- a/test/libapt/makefile +++ b/test/libapt/makefile @@ -74,3 +74,9 @@ PROGRAM = Configuration${BASENAME} SLIBS = -lapt-pkg SOURCE = configuration_test.cc include $(PROGRAM_H) + +# test cdroms core FindPackages +PROGRAM = CdromFindPackages${BASENAME} +SLIBS = -lapt-pkg +SOURCE = cdromfindpackages_test.cc +include $(PROGRAM_H) diff --git a/test/libapt/run-tests b/test/libapt/run-tests index ada2dc38b..bd47d4e0d 100755 --- a/test/libapt/run-tests +++ b/test/libapt/run-tests @@ -75,6 +75,35 @@ do continue elif [ $name = "CompareVersion${EXT}" ]; then tmppath="${DIR}/versions.lst" + elif [ $name = "CdromFindPackages${EXT}" ]; then + tmppath=$(mktemp -d) + mkdir -p "${tmppath}/.disk" "${tmppath}/pool" \ + "${tmppath}/dists/stable/main/binary-i386" \ + "${tmppath}/dists/stable/main/source" \ + "${tmppath}/dists/stable/contrib/binary-amd64" \ + "${tmppath}/dists/stable/contrib/binary-all" \ + "${tmppath}/dists/unstable/main/binary-i386" \ + "${tmppath}/dists/unstable/main/i18n" \ + "${tmppath}/dists/unstable/main/source" \ + "${tmppath}/dists/broken/non-free/source" + touch "${tmppath}/dists/broken/.aptignr" \ + "${tmppath}/dists/stable/main/binary-i386/Packages" \ + "${tmppath}/dists/stable/main/binary-i386/Packages.gz" \ + "${tmppath}/dists/stable/main/source/Sources.gz" \ + "${tmppath}/dists/stable/contrib/binary-amd64/Packages" \ + "${tmppath}/dists/stable/contrib/binary-amd64/Packages.gz" \ + "${tmppath}/dists/stable/contrib/binary-all/Packages" \ + "${tmppath}/dists/unstable/main/binary-i386/Packages.gz" \ + "${tmppath}/dists/unstable/main/binary-i386/Packages.gz" \ + "${tmppath}/dists/unstable/main/i18n/Translation-en" \ + "${tmppath}/dists/unstable/main/i18n/Translation-de.gz" \ + "${tmppath}/dists/unstable/main/source/Sources.gz" \ + "${tmppath}/dists/broken/non-free/source/Sources.gz" \ + "${tmppath}/dists/stable/Release.gpg" \ + "${tmppath}/dists/stable/Release" \ + "${tmppath}/dists/unstable/InRelease" \ + "${tmppath}/dists/broken/Release.gpg" + ln -s "${tmppath}/dists/unstable" "${tmppath}/dists/sid" fi echo -n "Testing with ${NAME} " -- cgit v1.2.3 From eed564b853a0e5778080c5c3012517a814041d5a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 13:08:21 +0100 Subject: use different compression types in the test now that we support them --- test/libapt/run-tests | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/libapt/run-tests b/test/libapt/run-tests index bd47d4e0d..5fff4ecca 100755 --- a/test/libapt/run-tests +++ b/test/libapt/run-tests @@ -88,16 +88,16 @@ do "${tmppath}/dists/broken/non-free/source" touch "${tmppath}/dists/broken/.aptignr" \ "${tmppath}/dists/stable/main/binary-i386/Packages" \ - "${tmppath}/dists/stable/main/binary-i386/Packages.gz" \ - "${tmppath}/dists/stable/main/source/Sources.gz" \ + "${tmppath}/dists/stable/main/binary-i386/Packages.bz2" \ + "${tmppath}/dists/stable/main/source/Sources.xz" \ "${tmppath}/dists/stable/contrib/binary-amd64/Packages" \ "${tmppath}/dists/stable/contrib/binary-amd64/Packages.gz" \ "${tmppath}/dists/stable/contrib/binary-all/Packages" \ - "${tmppath}/dists/unstable/main/binary-i386/Packages.gz" \ - "${tmppath}/dists/unstable/main/binary-i386/Packages.gz" \ + "${tmppath}/dists/unstable/main/binary-i386/Packages.xz" \ + "${tmppath}/dists/unstable/main/binary-i386/Packages.lzma" \ "${tmppath}/dists/unstable/main/i18n/Translation-en" \ - "${tmppath}/dists/unstable/main/i18n/Translation-de.gz" \ - "${tmppath}/dists/unstable/main/source/Sources.gz" \ + "${tmppath}/dists/unstable/main/i18n/Translation-de.bz2" \ + "${tmppath}/dists/unstable/main/source/Sources.xz" \ "${tmppath}/dists/broken/non-free/source/Sources.gz" \ "${tmppath}/dists/stable/Release.gpg" \ "${tmppath}/dists/stable/Release" \ -- cgit v1.2.3 From 032bd56ff86166fd4b6a8f69bd9d5d1bc57b886e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 19:46:59 +0100 Subject: - add a ReadLine method - drop the explicit export of gz-compression handling --- apt-pkg/contrib/fileutl.cc | 83 ++++++++++++++++++++++++++++++++++------------ apt-pkg/contrib/fileutl.h | 30 +++++++++++------ apt-pkg/contrib/mmap.cc | 13 +++++++- debian/changelog | 4 ++- methods/rred.cc | 42 ++++++++--------------- 5 files changed, 110 insertions(+), 62 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 83b68e796..58cd6dceb 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -44,6 +44,8 @@ #include #include +#include + #ifdef WORDS_BIGENDIAN #include #endif @@ -53,6 +55,12 @@ using namespace std; +class FileFdPrivate { + public: + gzFile gz; + FileFdPrivate() : gz(NULL) {}; +}; + // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -719,11 +727,12 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ -bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long Perms) +bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long const Perms) { if (Mode == ReadOnlyGzip) return Open(FileName, ReadOnly, Gzip, Perms); Close(); + d = new FileFdPrivate; Flags = AutoClose; if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) @@ -865,6 +874,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose) { Close(); + d = new FileFdPrivate; Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; if (OpenInternDescriptor(Mode, Compress) == false) @@ -883,13 +893,14 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) else if (Compress == Gzip) { if ((Mode & ReadWrite) == ReadWrite) - gz = gzdopen(iFd, "r+"); + d->gz = gzdopen(iFd, "r+"); else if ((Mode & WriteOnly) == WriteOnly) - gz = gzdopen(iFd, "w"); + d->gz = gzdopen(iFd, "w"); else - gz = gzdopen (iFd, "r"); - if (gz == NULL) + d->gz = gzdopen (iFd, "r"); + if (d->gz == NULL) return false; + Flags |= Compressed; } else return false; @@ -918,8 +929,8 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) do { - if (gz != NULL) - Res = gzread(gz,To,Size); + if (d->gz != NULL) + Res = gzread(d->gz,To,Size); else Res = read(iFd,To,Size); if (Res < 0 && errno == EINTR) @@ -951,6 +962,28 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) return _error->Error(_("read, still have %llu to read but none left"), Size); } /*}}}*/ +// FileFd::ReadLine - Read a complete line from the file /*{{{*/ +// --------------------------------------------------------------------- +/* Beware: This method can be quiet slow for big buffers on UNcompressed + files because of the naive implementation! */ +char* FileFd::ReadLine(char *To, unsigned long long const Size) +{ + if (d->gz != NULL) + return gzgets(d->gz, To, Size); + + unsigned long long read = 0; + if (Read(To, Size, &read) == false) + return NULL; + char* c = To; + for (; *c != '\n' && *c != '\0' && read != 0; --read, ++c) + ; // find the end of the line + if (*c != '\0') + *c = '\0'; + if (read != 0) + Seek(Tell() - read); + return To; +} + /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -960,8 +993,8 @@ bool FileFd::Write(const void *From,unsigned long long Size) errno = 0; do { - if (gz != NULL) - Res = gzwrite(gz,From,Size); + if (d->gz != NULL) + Res = gzwrite(d->gz,From,Size); else Res = write(iFd,From,Size); if (Res < 0 && errno == EINTR) @@ -990,8 +1023,8 @@ bool FileFd::Write(const void *From,unsigned long long Size) bool FileFd::Seek(unsigned long long To) { int res; - if (gz) - res = gzseek(gz,To,SEEK_SET); + if (d->gz) + res = gzseek(d->gz,To,SEEK_SET); else res = lseek(iFd,To,SEEK_SET); if (res != (signed)To) @@ -1009,8 +1042,8 @@ bool FileFd::Seek(unsigned long long To) bool FileFd::Skip(unsigned long long Over) { int res; - if (gz) - res = gzseek(gz,Over,SEEK_CUR); + if (d->gz != NULL) + res = gzseek(d->gz,Over,SEEK_CUR); else res = lseek(iFd,Over,SEEK_CUR); if (res < 0) @@ -1027,7 +1060,7 @@ bool FileFd::Skip(unsigned long long Over) /* */ bool FileFd::Truncate(unsigned long long To) { - if (gz) + if (d->gz != NULL) { Flags |= Fail; return _error->Error("Truncating gzipped files is not implemented (%s)", FileName.c_str()); @@ -1047,8 +1080,8 @@ bool FileFd::Truncate(unsigned long long To) unsigned long long FileFd::Tell() { off_t Res; - if (gz) - Res = gztell(gz); + if (d->gz != NULL) + Res = gztell(d->gz); else Res = lseek(iFd,0,SEEK_CUR); if (Res == (off_t)-1) @@ -1076,9 +1109,9 @@ unsigned long long FileFd::Size() unsigned long long size = FileSize(); // only check gzsize if we are actually a gzip file, just checking for - // "gz" is not sufficient as uncompressed files will be opened with + // "gz" is not sufficient as uncompressed files could be opened with // gzopen in "direct" mode as well - if (gz && !gzdirect(gz) && size > 0) + if (d->gz && !gzdirect(d->gz) && size > 0) { /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do * this ourselves; the original (uncompressed) file size is the last 32 @@ -1125,11 +1158,14 @@ time_t FileFd::ModificationTime() /* */ bool FileFd::Close() { + if (iFd == -1) + return true; + bool Res = true; if ((Flags & AutoClose) == AutoClose) { - if (gz != NULL) { - int const e = gzclose(gz); + if (d != NULL && d->gz != NULL) { + int const e = gzclose(d->gz); // gzdopen() on empty files always fails with "buffer error" here, ignore that if (e != 0 && e != Z_BUF_ERROR) Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); @@ -1147,13 +1183,17 @@ bool FileFd::Close() } iFd = -1; - gz = NULL; if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail && FileName.empty() == false) if (unlink(FileName.c_str()) != 0) Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str()); + if (d != NULL) + { + delete d; + d = NULL; + } return Res; } @@ -1170,3 +1210,4 @@ bool FileFd::Sync() return true; } /*}}}*/ +gzFile FileFd::gzFd() {return d->gz;}; diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 8f2d7a0a0..209ca91e7 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -31,17 +31,17 @@ /* Define this for python-apt */ #define APT_HAS_GZIP 1 +class FileFdPrivate; class FileFd { protected: int iFd; enum LocalFlags {AutoClose = (1<<0),Fail = (1<<1),DelOnFail = (1<<2), - HitEof = (1<<3), Replace = (1<<4) }; + HitEof = (1<<3), Replace = (1<<4), Compressed = (1<<5) }; unsigned long Flags; std::string FileName; std::string TemporaryFileName; - gzFile gz; public: enum OpenMode { @@ -71,6 +71,7 @@ class FileFd return Read(To,Size); } bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0); + char* ReadLine(char *To, unsigned long long const Size); bool Write(const void *From,unsigned long long Size); bool Seek(unsigned long long To); bool Skip(unsigned long long To); @@ -94,8 +95,8 @@ class FileFd return T; } - bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long Perms = 0666); - inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long Perms = 0666) { + bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long const Perms = 0666); + inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long const Perms = 0666) { return Open(FileName, Mode, None, Perms); }; bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); @@ -108,29 +109,36 @@ class FileFd // Simple manipulators inline int Fd() {return iFd;}; inline void Fd(int fd) {iFd = fd;}; - inline gzFile gzFd() {return gz;}; + __deprecated gzFile gzFd(); inline bool IsOpen() {return iFd >= 0;}; inline bool Failed() {return (Flags & Fail) == Fail;}; inline void EraseOnFailure() {Flags |= DelOnFail;}; inline void OpFail() {Flags |= Fail;}; inline bool Eof() {return (Flags & HitEof) == HitEof;}; + inline bool IsCompressed() {return (Flags & Compressed) == Compressed;}; inline std::string &Name() {return FileName;}; - FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), - Flags(0), gz(NULL) + FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) { Open(FileName,Mode, None, Perms); }; - FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : - iFd(-1), Flags(0), gz(NULL) + FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) { Open(FileName,Mode, Compress, Perms); }; - FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose), gz(NULL) {}; - FileFd(int Fd,bool) : iFd(Fd), Flags(0), gz(NULL) {}; + FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {}; + FileFd(int const Fd, OpenMode Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL) + { + OpenDescriptor(Fd, Mode, Compress); + }; + FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL) + { + OpenDescriptor(Fd, ReadWrite, None, AutoClose); + }; virtual ~FileFd(); private: + FileFdPrivate* d; bool OpenInternDescriptor(OpenMode Mode, CompressMode Compress); }; diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index f76169a92..1fb84b0af 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -77,7 +77,18 @@ bool MMap::Map(FileFd &Fd) if (iSize == 0) return _error->Error(_("Can't mmap an empty file")); - + + // We can't mmap compressed fd's directly, so we need to read it completely + if (Fd.IsCompressed() == true) + { + if ((Flags & ReadOnly) != ReadOnly) + return _error->Error("Compressed file %s can only be mapped readonly", Fd.Name().c_str()); + Base = new unsigned char[iSize]; + if (Fd.Seek(0L) == false || Fd.Read(Base, iSize) == false) + return false; + return true; + } + // Map it. Base = mmap(0,iSize,Prot,Map,Fd.Fd(),0); if (Base == (void *)-1) diff --git a/debian/changelog b/debian/changelog index 3e4a8fa3c..197e27a65 100644 --- a/debian/changelog +++ b/debian/changelog @@ -31,10 +31,12 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low - dump the APT::Compressor settings correctly and completely * apt-pkg/contrib/fileutl.{h,cc}: - implement a ModificationTime method for FileFd + - add a ReadLine method + - drop the explicit export of gz-compression handling * apt-pkg/cdrom.cc: - support InRelease files on cdrom - -- David Kalnischkies Sun, 11 Dec 2011 01:30:12 +0100 + -- David Kalnischkies Sun, 11 Dec 2011 19:34:58 +0100 apt (0.8.16~exp8) experimental; urgency=low diff --git a/methods/rred.cc b/methods/rred.cc index 56ad8220b..2a70a9f91 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -37,13 +37,13 @@ class RredMethod : public pkgAcqMethod { // return values enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE, MMAP_FAILED}; - State applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file, + State applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file, unsigned long &line, char *buffer, Hashes *hash) const; void ignoreLineInFile(FILE *fin, char *buffer) const; - void ignoreLineInFile(gzFile &fin, char *buffer) const; + void ignoreLineInFile(FileFd &fin, char *buffer) const; void copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lines, Hashes *hash, char *buffer) const; - void copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines, + void copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines, Hashes *hash, char *buffer) const; State patchFile(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const; @@ -72,10 +72,10 @@ public: * \param hash the created file for correctness * \return the success State of the ed command executor */ -RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file, +RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file, unsigned long &line, char *buffer, Hashes *hash) const { // get the current command and parse it - if (gzgets(ed_cmds, buffer, BUF_SIZE) == NULL) { + if (ed_cmds.ReadLine(buffer, BUF_SIZE) == NULL) { if (Debug == true) std::clog << "rred: encounter end of file - we can start patching now." << std::endl; line = 0; @@ -130,7 +130,7 @@ RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *ou unsigned char mode = *idx; // save the current position - unsigned const long pos = gztell(ed_cmds); + unsigned const long long pos = ed_cmds.Tell(); // if this is add or change then go to the next full stop unsigned int data_length = 0; @@ -164,7 +164,7 @@ RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *ou // include data from ed script if (mode == MODE_CHANGED || mode == MODE_ADDED) { - gzseek(ed_cmds, pos, SEEK_SET); + ed_cmds.Seek(pos); copyLinesFromFileToFile(ed_cmds, out_file, data_length, hash, buffer); } @@ -190,11 +190,11 @@ void RredMethod::copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lin } } /*}}}*/ -void RredMethod::copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines,/*{{{*/ +void RredMethod::copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines,/*{{{*/ Hashes *hash, char *buffer) const { while (0 < lines--) { do { - gzgets(fin, buffer, BUF_SIZE); + fin.ReadLine(buffer, BUF_SIZE); size_t const written = fwrite(buffer, 1, strlen(buffer), fout); hash->Add((unsigned char*)buffer, written); } while (strlen(buffer) == (BUF_SIZE - 1) && @@ -211,11 +211,11 @@ void RredMethod::ignoreLineInFile(FILE *fin, char *buffer) const { /*{{{*/ } } /*}}}*/ -void RredMethod::ignoreLineInFile(gzFile &fin, char *buffer) const { /*{{{*/ - gzgets(fin, buffer, BUF_SIZE); +void RredMethod::ignoreLineInFile(FileFd &fin, char *buffer) const { /*{{{*/ + fin.ReadLine(buffer, BUF_SIZE); while (strlen(buffer) == (BUF_SIZE - 1) && buffer[BUF_SIZE - 2] != '\n') { - gzgets(fin, buffer, BUF_SIZE); + fin.ReadLine(buffer, BUF_SIZE); buffer[0] = ' '; } } @@ -224,12 +224,11 @@ RredMethod::State RredMethod::patchFile(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { char buffer[BUF_SIZE]; FILE* fFrom = fdopen(From.Fd(), "r"); - gzFile fPatch = Patch.gzFd(); FILE* fTo = fdopen(out_file.Fd(), "w"); /* we do a tail recursion to read the commands in the right order */ unsigned long line = -1; // assign highest possible value - State const result = applyFile(fPatch, fFrom, fTo, line, buffer, hash); + State const result = applyFile(Patch, fFrom, fTo, line, buffer, hash); /* read the rest from infile */ if (result == ED_OK) { @@ -258,20 +257,7 @@ struct EdCommand { RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { #ifdef _POSIX_MAPPED_FILES - MMap ed_cmds(MMap::ReadOnly); - if (Patch.gzFd() != NULL) { - unsigned long long mapSize = Patch.Size(); - DynamicMMap* dyn = new DynamicMMap(0, mapSize, 0); - if (dyn->validData() == false) { - delete dyn; - return MMAP_FAILED; - } - dyn->AddSize(mapSize); - gzread(Patch.gzFd(), dyn->Data(), mapSize); - ed_cmds = *dyn; - } else - ed_cmds = MMap(Patch, MMap::ReadOnly); - + MMap ed_cmds(Patch, MMap::ReadOnly); MMap in_file(From, MMap::ReadOnly); if (ed_cmds.Size() == 0 || in_file.Size() == 0) -- cgit v1.2.3 From 699b209e5122f8fcd85fc4666c9b7020286ab0d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 00:17:30 +0100 Subject: Allow the FileFd to use an external Compressor to uncompress a given file internally so that it is exported and can be used like a "normal" uncompressed file with FileFd This allows us to hide th zlib usage in the implementation and use gzip instead if we don't have zlib builtin (the same for other compressors). The code includes quiet a few FIXME's so while all tests are working it shouldn't be used just yet outside of libapt as it might break. --- apt-pkg/contrib/fileutl.cc | 302 +++++++++++++++++++++++++++++++++++++++++-- apt-pkg/contrib/fileutl.h | 10 ++ apt-pkg/contrib/mmap.cc | 9 +- apt-pkg/deb/deblistparser.cc | 1 + apt-pkg/indexcopy.cc | 97 +------------- ftparchive/multicompress.cc | 74 +---------- ftparchive/multicompress.h | 2 - 7 files changed, 320 insertions(+), 175 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 58cd6dceb..727d3ddb5 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -44,7 +44,13 @@ #include #include +// FIXME: Compressor Fds have some speed disadvantages and are a bit buggy currently, +// so while the current implementation satisfies the testcases it is not a real option +// to disable it for now +#define APT_USE_ZLIB 1 +#ifdef APT_USE_ZLIB #include +#endif #ifdef WORDS_BIGENDIAN #include @@ -57,8 +63,16 @@ using namespace std; class FileFdPrivate { public: +#ifdef APT_USE_ZLIB gzFile gz; - FileFdPrivate() : gz(NULL) {}; +#else + void* gz; +#endif + pid_t compressor_pid; + bool pipe; + APT::Configuration::Compressor compressor; + FileFd::OpenMode openmode; + FileFdPrivate() : gz(NULL), compressor_pid(-1), pipe(false) {}; }; // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ @@ -724,6 +738,175 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) } /*}}}*/ +// ExecCompressor - Open a de/compressor pipe /*{{{*/ +// --------------------------------------------------------------------- +/* This opens the compressor, either in compress mode or decompress + mode. FileFd is always the compressor input/output file, + OutFd is the created pipe, Input for Compress, Output for Decompress. */ +bool ExecCompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, int const FileFd, int &OutFd, bool const Comp) +{ + if (Pid != NULL) + *Pid = -1; + + // No compression + if (Prog.Binary.empty() == true) + { + OutFd = dup(FileFd); + return true; + } + + // Handle 'decompression' of empty files + if (Comp == false) + { + struct stat Buf; + fstat(FileFd, &Buf); + if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) + { + OutFd = FileFd; + return true; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + if (Comp == true) + OutFd = Pipe[1]; + else + OutFd = Pipe[0]; + + // The child.. + pid_t child = ExecFork(); + if (Pid != NULL) + *Pid = child; + if (child == 0) + { + if (Comp == true) + { + dup2(FileFd,STDOUT_FILENO); + dup2(Pipe[0],STDIN_FILENO); + } + else + { + dup2(FileFd,STDIN_FILENO); + dup2(Pipe[1],STDOUT_FILENO); + } + + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + + std::vector Args; + Args.push_back(Prog.Binary.c_str()); + std::vector const * const addArgs = + (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); + for (std::vector::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + + if (Pid == NULL) + ExecWait(child, Prog.Binary.c_str(), true); + + return true; +} +bool ExecCompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, std::string const &FileName, int &OutFd, bool const Comp) +{ + if (Pid != NULL) + *Pid = -1; + + // No compression + if (Prog.Binary.empty() == true) + { + if (Comp == true) + OutFd = open(FileName.c_str(), O_WRONLY, 0666); + else + OutFd = open(FileName.c_str(), O_RDONLY); + return true; + } + + // Handle 'decompression' of empty files + if (Comp == false) + { + struct stat Buf; + stat(FileName.c_str(), &Buf); + if (Buf.st_size == 0) + { + OutFd = open(FileName.c_str(), O_RDONLY); + return true; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + if (Comp == true) + OutFd = Pipe[1]; + else + OutFd = Pipe[0]; + + // The child.. + pid_t child = ExecFork(); + if (Pid != NULL) + *Pid = child; + if (child == 0) + { + if (Comp == true) + { + dup2(Pipe[0],STDIN_FILENO); + SetCloseExec(STDIN_FILENO,false); + } + else + { + dup2(Pipe[1],STDOUT_FILENO); + SetCloseExec(STDOUT_FILENO,false); + } + + std::vector Args; + Args.push_back(Prog.Binary.c_str()); + std::vector const * const addArgs = + (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); + for (std::vector::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + Args.push_back("--stdout"); + Args.push_back(FileName.c_str()); + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + + if (Pid == NULL) + ExecWait(child, Prog.Binary.c_str(), false); + + return true; +} + /*}}}*/ + // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ @@ -733,6 +916,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned return Open(FileName, ReadOnly, Gzip, Perms); Close(); d = new FileFdPrivate; + d->openmode = Mode; Flags = AutoClose; if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) @@ -805,12 +989,15 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned // if we have them, use inbuilt compressors instead of forking if (compressor != compressors.end()) { +#ifdef APT_USE_ZLIB if (compressor->Name == "gzip") { Compress = Gzip; compressor = compressors.end(); } - else if (compressor->Name == "." || Compress == None) + else +#endif + if (compressor->Name == ".") { Compress = None; compressor = compressors.end(); @@ -839,9 +1026,12 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned if (compressor != compressors.end()) { if ((Mode & ReadWrite) == ReadWrite) - _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); + return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); - _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + if (ExecCompressor(*compressor, NULL /*d->compressor_pid*/, FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + d->pipe = true; + d->compressor = *compressor; } else { @@ -875,6 +1065,7 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A { Close(); d = new FileFdPrivate; + d->openmode = Mode; Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; if (OpenInternDescriptor(Mode, Compress) == false) @@ -890,6 +1081,7 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) { if (Compress == None) return true; +#ifdef APT_USE_ZLIB else if (Compress == Gzip) { if ((Mode & ReadWrite) == ReadWrite) @@ -902,8 +1094,29 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) return false; Flags |= Compressed; } +#endif else - return false; + { + std::string name; + switch (Compress) + { + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + default: return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + } + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end() || + ExecCompressor(*compressor, NULL /*&(d->compressor_pid)*/, + FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", name.c_str(), FileName.c_str()); + d->pipe = true; + } return true; } /*}}}*/ @@ -926,12 +1139,14 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) errno = 0; if (Actual != 0) *Actual = 0; - + *((char *)To) = '\0'; do { +#ifdef APT_USE_ZLIB if (d->gz != NULL) Res = gzread(d->gz,To,Size); else +#endif Res = read(iFd,To,Size); if (Res < 0 && errno == EINTR) continue; @@ -968,8 +1183,11 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) files because of the naive implementation! */ char* FileFd::ReadLine(char *To, unsigned long long const Size) { + *To = '\0'; +#ifdef APT_USE_ZLIB if (d->gz != NULL) return gzgets(d->gz, To, Size); +#endif unsigned long long read = 0; if (Read(To, Size, &read) == false) @@ -993,9 +1211,11 @@ bool FileFd::Write(const void *From,unsigned long long Size) errno = 0; do { +#ifdef APT_USE_ZLIB if (d->gz != NULL) Res = gzwrite(d->gz,From,Size); else +#endif Res = write(iFd,From,Size); if (Res < 0 && errno == EINTR) continue; @@ -1022,10 +1242,21 @@ bool FileFd::Write(const void *From,unsigned long long Size) /* */ bool FileFd::Seek(unsigned long long To) { + if (d->pipe == true) + { + // FIXME: What about OpenDescriptor() stuff here? + close(iFd); + bool result = ExecCompressor(d->compressor, NULL, FileName, iFd, (d->openmode & ReadOnly) != ReadOnly); + if (result == true && To != 0) + result &= Skip(To); + return result; + } int res; +#ifdef USE_ZLIB if (d->gz) res = gzseek(d->gz,To,SEEK_SET); else +#endif res = lseek(iFd,To,SEEK_SET); if (res != (signed)To) { @@ -1042,9 +1273,11 @@ bool FileFd::Seek(unsigned long long To) bool FileFd::Skip(unsigned long long Over) { int res; +#ifdef USE_ZLIB if (d->gz != NULL) res = gzseek(d->gz,Over,SEEK_CUR); else +#endif res = lseek(iFd,Over,SEEK_CUR); if (res < 0) { @@ -1080,9 +1313,11 @@ bool FileFd::Truncate(unsigned long long To) unsigned long long FileFd::Tell() { off_t Res; +#ifdef USE_ZLIB if (d->gz != NULL) Res = gztell(d->gz); else +#endif Res = lseek(iFd,0,SEEK_CUR); if (Res == (off_t)-1) _error->Errno("lseek","Failed to determine the current file position"); @@ -1095,9 +1330,19 @@ unsigned long long FileFd::Tell() unsigned long long FileFd::FileSize() { struct stat Buf; - - if (fstat(iFd,&Buf) != 0) + if (d->pipe == false && fstat(iFd,&Buf) != 0) return _error->Errno("fstat","Unable to determine the file size"); + + // for compressor pipes st_size is undefined and at 'best' zero + if (d->pipe == true || S_ISFIFO(Buf.st_mode)) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + d->pipe = true; + if (stat(FileName.c_str(), &Buf) != 0) + return _error->Errno("stat","Unable to determine the file size"); + } + return Buf.st_size; } /*}}}*/ @@ -1108,10 +1353,25 @@ unsigned long long FileFd::Size() { unsigned long long size = FileSize(); + // for compressor pipes st_size is undefined and at 'best' zero, + // so we 'read' the content and 'seek' back - see there + if (d->pipe == true) + { + // FIXME: If we have read first and then FileSize() the report is wrong + size = 0; + char ignore[1000]; + unsigned long long read = 0; + do { + Read(ignore, sizeof(ignore), &read); + size += read; + } while(read != 0); + Seek(0); + } +#ifdef USE_ZLIB // only check gzsize if we are actually a gzip file, just checking for // "gz" is not sufficient as uncompressed files could be opened with // gzopen in "direct" mode as well - if (d->gz && !gzdirect(d->gz) && size > 0) + else if (d->gz && !gzdirect(d->gz) && size > 0) { /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do * this ourselves; the original (uncompressed) file size is the last 32 @@ -1135,6 +1395,7 @@ unsigned long long FileFd::Size() return _error->Errno("lseek","Unable to seek in gzipped file"); return size; } +#endif return size; } @@ -1145,11 +1406,25 @@ unsigned long long FileFd::Size() time_t FileFd::ModificationTime() { struct stat Buf; - if (fstat(iFd,&Buf) != 0) + if (d->pipe == false && fstat(iFd,&Buf) != 0) { _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); return 0; } + + // for compressor pipes st_size is undefined and at 'best' zero + if (d->pipe == true || S_ISFIFO(Buf.st_mode)) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + d->pipe = true; + if (stat(FileName.c_str(), &Buf) != 0) + { + _error->Errno("fstat","Unable to determine the modification time of file %s", FileName.c_str()); + return 0; + } + } + return Buf.st_mtime; } /*}}}*/ @@ -1164,12 +1439,14 @@ bool FileFd::Close() bool Res = true; if ((Flags & AutoClose) == AutoClose) { +#ifdef USE_ZLIB if (d != NULL && d->gz != NULL) { int const e = gzclose(d->gz); // gzdopen() on empty files always fails with "buffer error" here, ignore that if (e != 0 && e != Z_BUF_ERROR) Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); } else +#endif if (iFd > 0 && close(iFd) != 0) Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str()); } @@ -1191,6 +1468,8 @@ bool FileFd::Close() if (d != NULL) { +// if (d->compressor_pid != -1) +// ExecWait(d->compressor_pid, "FileFdCompressor", true); delete d; d = NULL; } @@ -1210,4 +1489,5 @@ bool FileFd::Sync() return true; } /*}}}*/ -gzFile FileFd::gzFd() {return d->gz;}; + +gzFile FileFd::gzFd() { return (gzFile) d->gz; } diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 209ca91e7..f96dc72dc 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -22,6 +22,7 @@ #define PKGLIB_FILEUTL_H #include +#include #include #include @@ -110,6 +111,7 @@ class FileFd inline int Fd() {return iFd;}; inline void Fd(int fd) {iFd = fd;}; __deprecated gzFile gzFd(); + inline bool IsOpen() {return iFd >= 0;}; inline bool Failed() {return (Flags & Fail) == Fail;}; inline void EraseOnFailure() {Flags |= DelOnFail;}; @@ -170,6 +172,14 @@ bool WaitFd(int Fd,bool write = false,unsigned long timeout = 0); pid_t ExecFork(); bool ExecWait(pid_t Pid,const char *Name,bool Reap = false); +bool ExecCompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, int const FileFd, int &OutFd, bool const Comp = true); +inline bool ExecDecompressor(APT::Configuration::Compressor const &Prog, + pid_t *Pid, int const FileFd, int &OutFd) +{ + return ExecCompressor(Prog, Pid, FileFd, OutFd, true); +} + // File string manipulators std::string flNotDir(std::string File); std::string flNotFile(std::string File); diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 1fb84b0af..a67ab3698 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -66,7 +66,7 @@ MMap::~MMap() bool MMap::Map(FileFd &Fd) { iSize = Fd.Size(); - + // Set the permissions. int Prot = PROT_READ; int Map = MAP_SHARED; @@ -97,6 +97,13 @@ bool MMap::Map(FileFd &Fd) { // The filesystem doesn't support this particular kind of mmap. // So we allocate a buffer and read the whole file into it. + if ((Flags & ReadOnly) == ReadOnly) + { + // for readonly, we don't need sync, so make it simple + Base = new unsigned char[iSize]; + return Fd.Read(Base, iSize); + } + // FIXME: Writing to compressed fd's ? int const dupped_fd = dup(Fd.Fd()); if (dupped_fd == -1) return _error->Errno("mmap", _("Couldn't duplicate file descriptor %i"), Fd.Fd()); diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 28568d5e3..bdb50f6bf 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -773,6 +773,7 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, // file. to provide Component pinning we use the section name now FileI->Component = WriteUniqString(component); + // FIXME: Code depends on the fact that Release files aren't compressed FILE* release = fdopen(dup(File.Fd()), "r"); if (release == NULL) return false; diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index f6457aa39..3747e3570 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -38,65 +38,6 @@ using namespace std; -// DecompressFile - wrapper for decompressing compressed files /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool DecompressFile(string Filename, int *fd, off_t *FileSize) -{ - struct stat Buf; - *fd = -1; - - std::vector const compressor = APT::Configuration::getCompressors(); - std::vector::const_iterator UnCompress; - std::string file = std::string(Filename).append(UnCompress->Extension); - for (UnCompress = compressor.begin(); UnCompress != compressor.end(); ++UnCompress) - { - if (stat(file.c_str(), &Buf) == 0) - break; - } - - if (UnCompress == compressor.end()) - return _error->Errno("decompressor", "Unable to parse file"); - - *FileSize = Buf.st_size; - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - *fd = Pipe[1]; - - // The child.. - pid_t Pid = ExecFork(); - if (Pid == 0) - { - dup2(Pipe[1],STDOUT_FILENO); - SetCloseExec(STDOUT_FILENO, false); - - std::vector Args; - Args.push_back(UnCompress->Binary.c_str()); - for (std::vector::const_iterator a = UnCompress->UncompressArgs.begin(); - a != UnCompress->UncompressArgs.end(); ++a) - Args.push_back(a->c_str()); - Args.push_back("--stdout"); - Args.push_back(file.c_str()); - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - } - - // Wait for decompress to finish - if (ExecWait(Pid, UnCompress->Binary.c_str(), false) == false) - return false; - - return true; -} - /*}}}*/ // IndexCopy::CopyPackages - Copy the package files from the CD /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -142,24 +83,10 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List, for (vector::iterator I = List.begin(); I != List.end(); ++I) { string OrigPath = string(*I,CDROM.length()); - off_t FileSize = 0; // Open the package file - FileFd Pkg; - if (RealFileExists(*I + GetFileName()) == true) - { - Pkg.Open(*I + GetFileName(),FileFd::ReadOnly); - FileSize = Pkg.Size(); - } - else - { - int fd; - if (!DecompressFile(string(*I + GetFileName()), &fd, &FileSize)) - return _error->Errno("decompress","Decompress failed for %s", - string(*I + GetFileName()).c_str()); - Pkg.Fd(dup(fd)); - Pkg.Seek(0); - } + FileFd Pkg(*I + GetFileName(), FileFd::ReadOnly, FileFd::Extension); + off_t const FileSize = Pkg.Size(); pkgTagFile Parser(&Pkg); if (_error->PendingError() == true) @@ -868,23 +795,11 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ for (vector::iterator I = List.begin(); I != List.end(); ++I) { string OrigPath = string(*I,CDROM.length()); - off_t FileSize = 0; - + // Open the package file - FileFd Pkg; - if (RealFileExists(*I) == true) - { - Pkg.Open(*I,FileFd::ReadOnly); - FileSize = Pkg.Size(); - } - else - { - int fd; - if (!DecompressFile(*I, &fd, &FileSize)) - return _error->Errno("decompress","Decompress failed for %s", (*I).c_str()); - Pkg.Fd(dup(fd)); - Pkg.Seek(0); - } + FileFd Pkg(*I, FileFd::ReadOnly, FileFd::Extension); + off_t const FileSize = Pkg.Size(); + pkgTagFile Parser(&Pkg); if (_error->PendingError() == true) return false; diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index bf0f858d9..37a713efd 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -16,6 +16,7 @@ // Include Files /*{{{*/ #include +#include #include #include #include @@ -261,73 +262,6 @@ bool MultiCompress::Finalize(unsigned long long &OutSize) return true; } /*}}}*/ -// MultiCompress::OpenCompress - Open the compressor /*{{{*/ -// --------------------------------------------------------------------- -/* This opens the compressor, either in compress mode or decompress - mode. FileFd is always the compressor input/output file, - OutFd is the created pipe, Input for Compress, Output for Decompress. */ -bool MultiCompress::OpenCompress(APT::Configuration::Compressor const &Prog, - pid_t &Pid,int const &FileFd,int &OutFd,bool const &Comp) -{ - Pid = -1; - - // No compression - if (Prog.Binary.empty() == true) - { - OutFd = dup(FileFd); - return true; - } - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - if (Comp == true) - OutFd = Pipe[1]; - else - OutFd = Pipe[0]; - - // The child.. - Pid = ExecFork(); - if (Pid == 0) - { - if (Comp == true) - { - dup2(FileFd,STDOUT_FILENO); - dup2(Pipe[0],STDIN_FILENO); - } - else - { - dup2(FileFd,STDIN_FILENO); - dup2(Pipe[1],STDOUT_FILENO); - } - - SetCloseExec(STDOUT_FILENO,false); - SetCloseExec(STDIN_FILENO,false); - - std::vector Args; - Args.push_back(Prog.Binary.c_str()); - std::vector const * const addArgs = - (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); - for (std::vector::const_iterator a = addArgs->begin(); - a != addArgs->end(); ++a) - Args.push_back(a->c_str()); - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - }; - if (Comp == true) - close(Pipe[0]); - else - close(Pipe[1]); - return true; -} - /*}}}*/ // MultiCompress::OpenOld - Open an old file /*{{{*/ // --------------------------------------------------------------------- /* This opens one of the original output files, possibly decompressing it. */ @@ -344,7 +278,7 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) return false; // Decompress the file so we can read it - if (OpenCompress(Best->CompressProg,Proc,F.Fd(),Fd,false) == false) + if (ExecCompressor(Best->CompressProg,&Proc,F.Fd(),Fd,false) == false) return false; return true; @@ -374,8 +308,8 @@ bool MultiCompress::Child(int const &FD) // Start the compression children. for (Files *I = Outputs; I != 0; I = I->Next) { - if (OpenCompress(I->CompressProg,I->CompressProc,I->TmpFile.Fd(), - I->Fd,true) == false) + if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(), + I->Fd,true) == false) return false; } diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index 5496b06d0..ac8bd0746 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -41,8 +41,6 @@ class MultiCompress pid_t Outputter; mode_t Permissions; - bool OpenCompress(APT::Configuration::Compressor const &Prog, - pid_t &Pid,int const &FileFd, int &OutFd,bool const &Comp); bool Child(int const &Fd); bool Start(); bool Die(); -- cgit v1.2.3 From a4f6bdc8bd91c7282ae9ac60c44844c6f0058a65 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 00:54:37 +0100 Subject: revert 2184.1.2: do not pollute namespace in headers The breakage is just to big for now, so guard the change with #ifndef APT_8_CLEANER_HEADERS and be nice to library users --- apt-inst/contrib/extracttar.h | 4 ++++ apt-pkg/acquire.h | 5 +++++ apt-pkg/algorithms.h | 4 ++++ apt-pkg/cdrom.h | 4 ++++ apt-pkg/contrib/cdromutl.h | 4 ++++ apt-pkg/contrib/configuration.h | 4 ++++ apt-pkg/contrib/fileutl.h | 4 ++++ apt-pkg/contrib/hashes.h | 6 ++++++ apt-pkg/contrib/hashsum_template.h | 5 +++++ apt-pkg/contrib/md5.h | 5 +++++ apt-pkg/contrib/mmap.h | 4 ++++ apt-pkg/contrib/progress.h | 4 ++++ apt-pkg/contrib/sha1.h | 5 +++++ apt-pkg/contrib/strutl.h | 6 ++++++ apt-pkg/deb/dpkgpm.h | 5 +++++ apt-pkg/indexcopy.h | 5 +++++ apt-pkg/indexfile.h | 4 ++++ apt-pkg/metaindex.h | 4 ++++ apt-pkg/packagemanager.h | 4 ++++ apt-pkg/pkgcache.h | 5 ++++- apt-pkg/policy.h | 4 ++++ apt-pkg/srcrecords.h | 5 +++++ apt-pkg/vendor.h | 4 ++++ apt-pkg/vendorlist.h | 5 +++++ apt-pkg/version.h | 4 ++++ apt-pkg/versionmatch.h | 4 ++++ 26 files changed, 116 insertions(+), 1 deletion(-) diff --git a/apt-inst/contrib/extracttar.h b/apt-inst/contrib/extracttar.h index 8754e8dcc..e104417c1 100644 --- a/apt-inst/contrib/extracttar.h +++ b/apt-inst/contrib/extracttar.h @@ -18,6 +18,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +using std::min; +#endif + class pkgDirStream; class ExtractTar diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 93772403d..3d5d7a4b7 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -75,6 +75,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::vector; +using std::string; +#endif + class pkgAcquireStatus; /** \brief The core download scheduler. {{{ diff --git a/apt-pkg/algorithms.h b/apt-pkg/algorithms.h index 948fe1103..fdb64fc59 100644 --- a/apt-pkg/algorithms.h +++ b/apt-pkg/algorithms.h @@ -36,6 +36,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +using std::ostream; +#endif + class pkgAcquireStatus; class pkgSimulate : public pkgPackageManager /*{{{*/ diff --git a/apt-pkg/cdrom.h b/apt-pkg/cdrom.h index 319254fd0..4fcf5abcd 100644 --- a/apt-pkg/cdrom.h +++ b/apt-pkg/cdrom.h @@ -4,6 +4,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using namespace std; +#endif + class Configuration; class OpProgress; diff --git a/apt-pkg/contrib/cdromutl.h b/apt-pkg/contrib/cdromutl.h index 2c6afac0f..e94045b5c 100644 --- a/apt-pkg/contrib/cdromutl.h +++ b/apt-pkg/contrib/cdromutl.h @@ -12,6 +12,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + // mount cdrom, DeviceName (e.g. /dev/sr0) is optional bool MountCdrom(std::string Path, std::string DeviceName=""); bool UnmountCdrom(std::string Path); diff --git a/apt-pkg/contrib/configuration.h b/apt-pkg/contrib/configuration.h index f6f2a3c1d..4c2e75041 100644 --- a/apt-pkg/contrib/configuration.h +++ b/apt-pkg/contrib/configuration.h @@ -34,6 +34,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class Configuration { public: diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index f96dc72dc..8a986b82b 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -29,6 +29,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + /* Define this for python-apt */ #define APT_HAS_GZIP 1 diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index 81851dede..b206eccb8 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -22,6 +22,12 @@ #include #include + +#ifndef APT_8_CLEANER_HEADERS +using std::min; +using std::vector; +#endif + // helper class that contains hash function name // and hash class HashString diff --git a/apt-pkg/contrib/hashsum_template.h b/apt-pkg/contrib/hashsum_template.h index 27d192b82..6301ac9d0 100644 --- a/apt-pkg/contrib/hashsum_template.h +++ b/apt-pkg/contrib/hashsum_template.h @@ -15,6 +15,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::min; +#endif + template class HashSumValue { diff --git a/apt-pkg/contrib/md5.h b/apt-pkg/contrib/md5.h index a207da4e4..25631b166 100644 --- a/apt-pkg/contrib/md5.h +++ b/apt-pkg/contrib/md5.h @@ -31,6 +31,11 @@ #include "hashsum_template.h" +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::min; +#endif + typedef HashSumValue<128> MD5SumValue; class MD5Summation : public SummationImplementation diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index 2ed4a95f8..602de94f8 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -28,6 +28,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class FileFd; /* This should be a 32 bit type, larger tyes use too much ram and smaller diff --git a/apt-pkg/contrib/progress.h b/apt-pkg/contrib/progress.h index 7635719bc..3a6943aee 100644 --- a/apt-pkg/contrib/progress.h +++ b/apt-pkg/contrib/progress.h @@ -25,6 +25,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class Configuration; class OpProgress { diff --git a/apt-pkg/contrib/sha1.h b/apt-pkg/contrib/sha1.h index b4b139a22..a8d55eb13 100644 --- a/apt-pkg/contrib/sha1.h +++ b/apt-pkg/contrib/sha1.h @@ -20,6 +20,11 @@ #include "hashsum_template.h" +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::min; +#endif + typedef HashSumValue<160> SHA1SumValue; class SHA1Summation : public SummationImplementation diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 93f4bef4f..337139d5d 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -27,6 +27,12 @@ #include "macros.h" +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::vector; +using std::ostream; +#endif + bool UTF8ToCodeset(const char *codeset, const std::string &orig, std::string *dest); char *_strstrip(char *String); char *_strtabexpand(char *String,size_t Len); diff --git a/apt-pkg/deb/dpkgpm.h b/apt-pkg/deb/dpkgpm.h index 6b62360b7..aab39f633 100644 --- a/apt-pkg/deb/dpkgpm.h +++ b/apt-pkg/deb/dpkgpm.h @@ -15,6 +15,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::vector; +using std::map; +#endif + class pkgDPkgPMPrivate; class pkgDPkgPM : public pkgPackageManager diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h index 21294ae7e..e3de1afd9 100644 --- a/apt-pkg/indexcopy.h +++ b/apt-pkg/indexcopy.h @@ -14,6 +14,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::vector; +#endif + class pkgTagSection; class FileFd; class indexRecords; diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index 5e162a846..1d34dc773 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -29,6 +29,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class pkgAcquire; class pkgCacheGenerator; class OpProgress; diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 9cc79a7a6..0f95257e0 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -6,6 +6,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class pkgAcquire; class pkgCacheGenerator; class OpProgress; diff --git a/apt-pkg/packagemanager.h b/apt-pkg/packagemanager.h index d4989a6e0..1d807795d 100644 --- a/apt-pkg/packagemanager.h +++ b/apt-pkg/packagemanager.h @@ -29,6 +29,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class pkgAcquire; class pkgDepCache; class pkgSourceList; diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index fd1a02149..1a7013551 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -74,11 +74,14 @@ #ifndef PKGLIB_PKGCACHE_H #define PKGLIB_PKGCACHE_H - #include #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class pkgVersioningSystem; class pkgCache /*{{{*/ { diff --git a/apt-pkg/policy.h b/apt-pkg/policy.h index 3c8246e3b..5172a3c3b 100644 --- a/apt-pkg/policy.h +++ b/apt-pkg/policy.h @@ -38,6 +38,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::vector; +#endif + class pkgPolicy : public pkgDepCache::Policy { protected: diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h index a55bc74fa..06f0dce6c 100644 --- a/apt-pkg/srcrecords.h +++ b/apt-pkg/srcrecords.h @@ -17,6 +17,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::vector; +#endif + class pkgSourceList; class pkgIndexFile; class pkgSrcRecords diff --git a/apt-pkg/vendor.h b/apt-pkg/vendor.h index 9b157378c..6484adf9b 100644 --- a/apt-pkg/vendor.h +++ b/apt-pkg/vendor.h @@ -6,6 +6,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + // A class representing a particular software provider. class __deprecated Vendor { diff --git a/apt-pkg/vendorlist.h b/apt-pkg/vendorlist.h index 733d23a32..4e050477f 100644 --- a/apt-pkg/vendorlist.h +++ b/apt-pkg/vendorlist.h @@ -17,6 +17,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +using std::vector; +#endif + class Vendor; class Configuration; diff --git a/apt-pkg/version.h b/apt-pkg/version.h index 92dbc2576..e0e0e6c14 100644 --- a/apt-pkg/version.h +++ b/apt-pkg/version.h @@ -23,6 +23,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class pkgVersioningSystem { public: diff --git a/apt-pkg/versionmatch.h b/apt-pkg/versionmatch.h index da103fc5b..433396fc9 100644 --- a/apt-pkg/versionmatch.h +++ b/apt-pkg/versionmatch.h @@ -39,6 +39,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +using std::string; +#endif + class pkgVersionMatch { // Version Matching -- cgit v1.2.3 From b9dadc24b9477b466bc8058c765d76c65ecc7125 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 01:22:38 +0100 Subject: revert 2184.1.3: forward declaration instead of headers The breakage is just to big for now, so guard the change with #ifndef APT_8_CLEANER_HEADERS and be nice to library users --- apt-inst/contrib/arfile.h | 3 +++ apt-inst/contrib/extracttar.h | 2 ++ apt-inst/database.h | 4 ++++ apt-inst/deb/debfile.h | 4 ++++ apt-pkg/acquire-item.h | 8 ++++++++ apt-pkg/acquire-method.h | 5 +++++ apt-pkg/algorithms.h | 1 + apt-pkg/cachefile.h | 6 ++++++ apt-pkg/cacheset.h | 4 ++++ apt-pkg/cdrom.h | 1 + apt-pkg/contrib/cmndline.h | 4 ++++ apt-pkg/contrib/mmap.h | 1 + apt-pkg/contrib/netrc.h | 4 ++++ apt-pkg/deb/deblistparser.h | 4 ++++ apt-pkg/deb/debmetaindex.h | 4 ++++ apt-pkg/deb/debrecords.h | 4 ++++ apt-pkg/depcache.h | 5 +++++ apt-pkg/edsp.h | 5 +++++ apt-pkg/edsp/edspindexfile.h | 4 ++++ apt-pkg/edsp/edsplistparser.h | 6 ++++++ apt-pkg/indexrecords.h | 4 ++++ apt-pkg/init.h | 5 +++++ apt-pkg/metaindex.h | 3 +++ apt-pkg/packagemanager.h | 1 + apt-pkg/pkgsystem.h | 4 ++++ apt-pkg/sourcelist.h | 26 ++++++++++++++------------ apt-pkg/tagfile.h | 4 ++++ apt-pkg/vendorlist.h | 2 ++ 28 files changed, 116 insertions(+), 12 deletions(-) diff --git a/apt-inst/contrib/arfile.h b/apt-inst/contrib/arfile.h index 2be1323d1..0f62a34a0 100644 --- a/apt-inst/contrib/arfile.h +++ b/apt-inst/contrib/arfile.h @@ -17,6 +17,9 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif class FileFd; diff --git a/apt-inst/contrib/extracttar.h b/apt-inst/contrib/extracttar.h index e104417c1..4b29df314 100644 --- a/apt-inst/contrib/extracttar.h +++ b/apt-inst/contrib/extracttar.h @@ -19,6 +19,8 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include +#include using std::min; #endif diff --git a/apt-inst/database.h b/apt-inst/database.h index ccfee3797..64e149f98 100644 --- a/apt-inst/database.h +++ b/apt-inst/database.h @@ -25,6 +25,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class pkgFLCache; class OpProgress; diff --git a/apt-inst/deb/debfile.h b/apt-inst/deb/debfile.h index 2c4734f9e..5e1ea1d2f 100644 --- a/apt-inst/deb/debfile.h +++ b/apt-inst/deb/debfile.h @@ -29,6 +29,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class FileFd; class pkgDataBase; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 27b8e887b..51d539450 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -25,6 +25,14 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#include +#include +#include +#endif + /** \addtogroup acquire * @{ * diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index c3f042ee0..2dd9ad685 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -25,6 +25,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#endif + class Hashes; class pkgAcqMethod { diff --git a/apt-pkg/algorithms.h b/apt-pkg/algorithms.h index fdb64fc59..185d11e96 100644 --- a/apt-pkg/algorithms.h +++ b/apt-pkg/algorithms.h @@ -37,6 +37,7 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include using std::ostream; #endif diff --git a/apt-pkg/cachefile.h b/apt-pkg/cachefile.h index b56e42855..802b12b61 100644 --- a/apt-pkg/cachefile.h +++ b/apt-pkg/cachefile.h @@ -20,6 +20,12 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#include +#endif + class pkgPolicy; class pkgSourceList; class OpProgress; diff --git a/apt-pkg/cacheset.h b/apt-pkg/cacheset.h index d1e396e0f..91d7eec1c 100644 --- a/apt-pkg/cacheset.h +++ b/apt-pkg/cacheset.h @@ -20,6 +20,10 @@ #include #include + +#ifndef APT_8_CLEANER_HEADERS +#include +#endif /*}}}*/ class pkgCacheFile; diff --git a/apt-pkg/cdrom.h b/apt-pkg/cdrom.h index 4fcf5abcd..cedfccff7 100644 --- a/apt-pkg/cdrom.h +++ b/apt-pkg/cdrom.h @@ -5,6 +5,7 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include using namespace std; #endif diff --git a/apt-pkg/contrib/cmndline.h b/apt-pkg/contrib/cmndline.h index b201d9855..9f505fd41 100644 --- a/apt-pkg/contrib/cmndline.h +++ b/apt-pkg/contrib/cmndline.h @@ -44,6 +44,10 @@ #ifndef PKGLIB_CMNDLINE_H #define PKGLIB_CMNDLINE_H +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class Configuration; class CommandLine diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index 602de94f8..6bd4a2d86 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -29,6 +29,7 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include using std::string; #endif diff --git a/apt-pkg/contrib/netrc.h b/apt-pkg/contrib/netrc.h index 7b94eba88..5931d4a42 100644 --- a/apt-pkg/contrib/netrc.h +++ b/apt-pkg/contrib/netrc.h @@ -16,6 +16,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + #define DOT_CHAR "." #define DIR_CHAR "/" diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 9519d9711..386d291a2 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -14,6 +14,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class debListParser : public pkgCacheGenerator::ListParser { public: diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 0cba2d8a8..b9ecab97c 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -8,6 +8,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class debReleaseIndex : public metaIndex { public: diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h index 9c7ea6b48..b5e3bbdba 100644 --- a/apt-pkg/deb/debrecords.h +++ b/apt-pkg/deb/debrecords.h @@ -18,6 +18,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class debRecordParser : public pkgRecords::Parser { /** \brief dpointer placeholder (for later in case we need it) */ diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h index f6e6c0afc..7358048ed 100644 --- a/apt-pkg/depcache.h +++ b/apt-pkg/depcache.h @@ -46,6 +46,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#endif + class OpProgress; class pkgDepCache : protected pkgCache::Namespace diff --git a/apt-pkg/edsp.h b/apt-pkg/edsp.h index 07bbbdd03..12b06d143 100644 --- a/apt-pkg/edsp.h +++ b/apt-pkg/edsp.h @@ -15,6 +15,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#endif + class pkgDepCache; class OpProgress; diff --git a/apt-pkg/edsp/edspindexfile.h b/apt-pkg/edsp/edspindexfile.h index 9670c4837..de10f2d2f 100644 --- a/apt-pkg/edsp/edspindexfile.h +++ b/apt-pkg/edsp/edspindexfile.h @@ -10,6 +10,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class edspIndex : public debStatusIndex { /** \brief dpointer placeholder (for later in case we need it) */ diff --git a/apt-pkg/edsp/edsplistparser.h b/apt-pkg/edsp/edsplistparser.h index 5d82716c7..a7bf9de95 100644 --- a/apt-pkg/edsp/edsplistparser.h +++ b/apt-pkg/edsp/edsplistparser.h @@ -13,6 +13,12 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#include +#endif + class FileFd; class edspListParser : public debListParser diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h index fa60a0847..a98b939bc 100644 --- a/apt-pkg/indexrecords.h +++ b/apt-pkg/indexrecords.h @@ -13,6 +13,10 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class indexRecords { bool parseSumData(const char *&Start, const char *End, std::string &Name, diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 0c1c7ae5a..b6f3df753 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -13,6 +13,11 @@ #ifndef PKGLIB_INIT_H #define PKGLIB_INIT_H +#ifndef APT_8_CLEANER_HEADERS +#include +#include +#endif + class pkgSystem; class Configuration; diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 0f95257e0..5783735ff 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -7,6 +7,9 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include +#include +#include using std::string; #endif diff --git a/apt-pkg/packagemanager.h b/apt-pkg/packagemanager.h index 1d807795d..1a6a9f01c 100644 --- a/apt-pkg/packagemanager.h +++ b/apt-pkg/packagemanager.h @@ -30,6 +30,7 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include using std::string; #endif diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h index 211fd0d56..75f7b9fcc 100644 --- a/apt-pkg/pkgsystem.h +++ b/apt-pkg/pkgsystem.h @@ -41,6 +41,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class pkgDepCache; class pkgPackageManager; class pkgVersioningSystem; diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 4509e54b9..03e29ec34 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -32,9 +32,11 @@ #include #include +#ifndef APT_8_CLEANER_HEADERS +#include using std::string; using std::vector; - +#endif class pkgAcquire; class pkgIndexFile; @@ -58,31 +60,31 @@ class pkgSourceList const char *Label; bool FixupURI(string &URI) const; - virtual bool ParseLine(vector &List, + virtual bool ParseLine(std::vector &List, const char *Buffer, - unsigned long const &CurLine,string const &File) const; - virtual bool CreateItem(vector &List,string const &URI, - string const &Dist,string const &Section, - std::map const &Options) const = 0; + unsigned long const &CurLine,std::string const &File) const; + virtual bool CreateItem(vector &List,std::string const &URI, + std::string const &Dist,std::string const &Section, + std::map const &Options) const = 0; Type(); virtual ~Type() {}; }; - typedef vector::const_iterator const_iterator; + typedef std::vector::const_iterator const_iterator; protected: - vector SrcList; + std::vector SrcList; public: bool ReadMainList(); - bool Read(string File); + bool Read(std::string File); // CNC:2003-03-03 void Reset(); - bool ReadAppend(string File); - bool ReadSourceDir(string Dir); + bool ReadAppend(std::string File); + bool ReadSourceDir(std::string Dir); // List accessors inline const_iterator begin() const {return SrcList.begin();}; @@ -98,7 +100,7 @@ class pkgSourceList time_t GetLastModifiedTime(); pkgSourceList(); - pkgSourceList(string File); + pkgSourceList(std::string File); ~pkgSourceList(); }; diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index a5bf5ac90..fd24471c1 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -24,6 +24,10 @@ #include +#ifndef APT_8_CLEANER_HEADERS +#include +#endif + class FileFd; class pkgTagSection diff --git a/apt-pkg/vendorlist.h b/apt-pkg/vendorlist.h index 4e050477f..a86ccde7c 100644 --- a/apt-pkg/vendorlist.h +++ b/apt-pkg/vendorlist.h @@ -18,6 +18,8 @@ #include #ifndef APT_8_CLEANER_HEADERS +#include +#include using std::string; using std::vector; #endif -- cgit v1.2.3 From 73437844e2f22a17203dac0ba72317769ec54398 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 01:43:28 +0100 Subject: =?UTF-8?q?note=20to=20myself:=20In=20case=20you=20rename=20someth?= =?UTF-8?q?ing,=20make=20sure=20that=20you=20have=20renamed=20it=20everywh?= =?UTF-8?q?ere=20as=20otherwise=20stuff=20"magically"=20starts=20to=20fail?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes commit 2209 as the mixture of #define names generates a lovely compilable but non-functional mixture of gzip usage… --- apt-pkg/contrib/fileutl.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 727d3ddb5..25ac5275c 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1252,7 +1252,7 @@ bool FileFd::Seek(unsigned long long To) return result; } int res; -#ifdef USE_ZLIB +#ifdef APT_USE_ZLIB if (d->gz) res = gzseek(d->gz,To,SEEK_SET); else @@ -1273,7 +1273,7 @@ bool FileFd::Seek(unsigned long long To) bool FileFd::Skip(unsigned long long Over) { int res; -#ifdef USE_ZLIB +#ifdef APT_USE_ZLIB if (d->gz != NULL) res = gzseek(d->gz,Over,SEEK_CUR); else @@ -1313,7 +1313,7 @@ bool FileFd::Truncate(unsigned long long To) unsigned long long FileFd::Tell() { off_t Res; -#ifdef USE_ZLIB +#ifdef APT_USE_ZLIB if (d->gz != NULL) Res = gztell(d->gz); else @@ -1367,7 +1367,7 @@ unsigned long long FileFd::Size() } while(read != 0); Seek(0); } -#ifdef USE_ZLIB +#ifdef APT_USE_ZLIB // only check gzsize if we are actually a gzip file, just checking for // "gz" is not sufficient as uncompressed files could be opened with // gzopen in "direct" mode as well @@ -1439,7 +1439,7 @@ bool FileFd::Close() bool Res = true; if ((Flags & AutoClose) == AutoClose) { -#ifdef USE_ZLIB +#ifdef APT_USE_ZLIB if (d != NULL && d->gz != NULL) { int const e = gzclose(d->gz); // gzdopen() on empty files always fails with "buffer error" here, ignore that -- cgit v1.2.3 From aee1aac6f75906ec73dacffc55e7026002201f98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 23:48:14 +0100 Subject: allow Open() and OpenDescriptor() to be called with a Compressor --- apt-pkg/contrib/fileutl.cc | 187 +++++++++++++++++++++++---------------------- apt-pkg/contrib/fileutl.h | 4 +- 2 files changed, 98 insertions(+), 93 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 25ac5275c..c2b684089 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -48,7 +48,7 @@ // so while the current implementation satisfies the testcases it is not a real option // to disable it for now #define APT_USE_ZLIB 1 -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB #include #endif @@ -63,7 +63,7 @@ using namespace std; class FileFdPrivate { public: -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB gzFile gz; #else void* gz; @@ -914,95 +914,77 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned { if (Mode == ReadOnlyGzip) return Open(FileName, ReadOnly, Gzip, Perms); - Close(); - d = new FileFdPrivate; - d->openmode = Mode; - Flags = AutoClose; if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) return _error->Error("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); - if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) - return _error->Error("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); - - int fileflags = 0; -#define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE - if_FLAGGED_SET(ReadWrite, O_RDWR); - else if_FLAGGED_SET(ReadOnly, O_RDONLY); - else if_FLAGGED_SET(WriteOnly, O_WRONLY); - else return _error->Error("No openmode provided in FileFd::Open for %s", FileName.c_str()); - - if_FLAGGED_SET(Create, O_CREAT); - if_FLAGGED_SET(Exclusive, O_EXCL); - else if_FLAGGED_SET(Atomic, O_EXCL); - if_FLAGGED_SET(Empty, O_TRUNC); -#undef if_FLAGGED_SET // FIXME: Denote inbuilt compressors somehow - as we don't need to have the binaries for them std::vector const compressors = APT::Configuration::getCompressors(); std::vector::const_iterator compressor = compressors.begin(); if (Compress == Auto) { - Compress = None; for (; compressor != compressors.end(); ++compressor) { std::string file = std::string(FileName).append(compressor->Extension); if (FileExists(file) == false) continue; FileName = file; - if (compressor->Binary == ".") - Compress = None; - else - Compress = Extension; break; } } else if (Compress == Extension) { - Compress = None; std::string ext = flExtension(FileName); - if (ext != FileName) - { + if (ext == FileName) + ext.clear(); + else ext = "." + ext; - for (; compressor != compressors.end(); ++compressor) - if (ext == compressor->Extension) + for (; compressor != compressors.end(); ++compressor) + if (ext == compressor->Extension) + break; + // no matching extension - assume uncompressed (imagine files like 'example.org_Packages') + if (compressor == compressors.end()) + for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor) + if (compressor->Name == ".") break; - } } - else if (Compress != None) + else { std::string name; switch (Compress) { + case None: name = "."; break; case Gzip: name = "gzip"; break; case Bzip2: name = "bzip2"; break; case Lzma: name = "lzma"; break; case Xz: name = "xz"; break; - default: return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + case Auto: + case Extension: + // Unreachable + return _error->Error("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); } for (; compressor != compressors.end(); ++compressor) if (compressor->Name == name) break; - if (compressor == compressors.end() && name != "gzip") + if (compressor == compressors.end()) return _error->Error("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); } - // if we have them, use inbuilt compressors instead of forking - if (compressor != compressors.end()) - { -#ifdef APT_USE_ZLIB - if (compressor->Name == "gzip") - { - Compress = Gzip; - compressor = compressors.end(); - } - else -#endif - if (compressor->Name == ".") - { - Compress = None; - compressor = compressors.end(); - } - } + if (compressor == compressors.end()) + return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + return Open(FileName, Mode, *compressor, Perms); +} +bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms) +{ + Close(); + d = new FileFdPrivate; + d->openmode = Mode; + Flags = AutoClose; + + if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) + return _error->Error("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); + if ((Mode & ReadWrite) == 0) + return _error->Error("No openmode provided in FileFd::Open for %s", FileName.c_str()); if ((Mode & Atomic) == Atomic) { @@ -1023,18 +1005,35 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned unlink(FileName.c_str()); } - if (compressor != compressors.end()) + // if we have them, use inbuilt compressors instead of forking + if (compressor.Name != "." +#if APT_USE_ZLIB + && compressor.Name != "gzip" +#endif + ) { if ((Mode & ReadWrite) == ReadWrite) - return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); + return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor.Name.c_str(), FileName.c_str()); - if (ExecCompressor(*compressor, NULL /*d->compressor_pid*/, FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) - return _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + if (ExecCompressor(compressor, NULL /*d->compressor_pid*/, FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), FileName.c_str()); d->pipe = true; - d->compressor = *compressor; + d->compressor = compressor; } else { + int fileflags = 0; + #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE + if_FLAGGED_SET(ReadWrite, O_RDWR); + else if_FLAGGED_SET(ReadOnly, O_RDONLY); + else if_FLAGGED_SET(WriteOnly, O_WRONLY); + + if_FLAGGED_SET(Create, O_CREAT); + if_FLAGGED_SET(Exclusive, O_EXCL); + else if_FLAGGED_SET(Atomic, O_EXCL); + if_FLAGGED_SET(Empty, O_TRUNC); + #undef if_FLAGGED_SET + if (TemporaryFileName.empty() == false) iFd = open(TemporaryFileName.c_str(), fileflags, Perms); else @@ -1042,7 +1041,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned if (iFd != -1) { - if (OpenInternDescriptor(Mode, Compress) == false) + if (OpenInternDescriptor(Mode, compressor) == false) { close (iFd); iFd = -1; @@ -1062,13 +1061,37 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned // --------------------------------------------------------------------- /* */ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose) +{ + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + std::string name; + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Auto: + case Extension: + return _error->Error("Opening Fd %d in Auto or Extension compression mode is not supported", Fd); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + return _error->Error("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + + return OpenDescriptor(Fd, Mode, *compressor, AutoClose); +} +bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) { Close(); d = new FileFdPrivate; d->openmode = Mode; Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; - if (OpenInternDescriptor(Mode, Compress) == false) + if (OpenInternDescriptor(Mode, compressor) == false) { if (AutoClose) close (iFd); @@ -1077,12 +1100,12 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A this->FileName = ""; return true; } -bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) +bool FileFd::OpenInternDescriptor(OpenMode Mode, APT::Configuration::Compressor const &compressor) { - if (Compress == None) + if (compressor.Name == ".") return true; -#ifdef APT_USE_ZLIB - else if (Compress == Gzip) +#if APT_USE_ZLIB + else if (compressor.Name == "gzip") { if ((Mode & ReadWrite) == ReadWrite) d->gz = gzdopen(iFd, "r+"); @@ -1096,27 +1119,7 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) } #endif else - { - std::string name; - switch (Compress) - { - case Gzip: name = "gzip"; break; - case Bzip2: name = "bzip2"; break; - case Lzma: name = "lzma"; break; - case Xz: name = "xz"; break; - default: return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); - } - std::vector const compressors = APT::Configuration::getCompressors(); - std::vector::const_iterator compressor = compressors.begin(); - for (; compressor != compressors.end(); ++compressor) - if (compressor->Name == name) - break; - if (compressor == compressors.end() || - ExecCompressor(*compressor, NULL /*&(d->compressor_pid)*/, - FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) - return _error->Error("Forking external compressor %s is not implemented for %s", name.c_str(), FileName.c_str()); - d->pipe = true; - } + return _error->Error("Can't find a match for specified compressor %s for file %s", compressor.Name.c_str(), FileName.c_str()); return true; } /*}}}*/ @@ -1142,7 +1145,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) *((char *)To) = '\0'; do { -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d->gz != NULL) Res = gzread(d->gz,To,Size); else @@ -1184,7 +1187,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) char* FileFd::ReadLine(char *To, unsigned long long const Size) { *To = '\0'; -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d->gz != NULL) return gzgets(d->gz, To, Size); #endif @@ -1211,7 +1214,7 @@ bool FileFd::Write(const void *From,unsigned long long Size) errno = 0; do { -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d->gz != NULL) Res = gzwrite(d->gz,From,Size); else @@ -1252,7 +1255,7 @@ bool FileFd::Seek(unsigned long long To) return result; } int res; -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d->gz) res = gzseek(d->gz,To,SEEK_SET); else @@ -1273,7 +1276,7 @@ bool FileFd::Seek(unsigned long long To) bool FileFd::Skip(unsigned long long Over) { int res; -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d->gz != NULL) res = gzseek(d->gz,Over,SEEK_CUR); else @@ -1313,7 +1316,7 @@ bool FileFd::Truncate(unsigned long long To) unsigned long long FileFd::Tell() { off_t Res; -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d->gz != NULL) Res = gztell(d->gz); else @@ -1367,7 +1370,7 @@ unsigned long long FileFd::Size() } while(read != 0); Seek(0); } -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB // only check gzsize if we are actually a gzip file, just checking for // "gz" is not sufficient as uncompressed files could be opened with // gzopen in "direct" mode as well @@ -1439,7 +1442,7 @@ bool FileFd::Close() bool Res = true; if ((Flags & AutoClose) == AutoClose) { -#ifdef APT_USE_ZLIB +#if APT_USE_ZLIB if (d != NULL && d->gz != NULL) { int const e = gzclose(d->gz); // gzdopen() on empty files always fails with "buffer error" here, ignore that diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 8a986b82b..51277290e 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -101,10 +101,12 @@ class FileFd } bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long const Perms = 0666); + bool Open(std::string FileName,OpenMode Mode,APT::Configuration::Compressor const &compressor,unsigned long const Perms = 0666); inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long const Perms = 0666) { return Open(FileName, Mode, None, Perms); }; bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); + bool OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compressor const &compressor, bool AutoClose=false); inline bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false) { return OpenDescriptor(Fd, Mode, None, AutoClose); }; @@ -145,7 +147,7 @@ class FileFd private: FileFdPrivate* d; - bool OpenInternDescriptor(OpenMode Mode, CompressMode Compress); + bool OpenInternDescriptor(OpenMode Mode, APT::Configuration::Compressor const &compressor); }; bool RunScripts(const char *Cnf); -- cgit v1.2.3 From 52b47296f61ec3ca1075bbfb44982f5caa541e7c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 14 Dec 2011 22:11:43 +0100 Subject: use FileFd instead of forking the compression childs by hand --- apt-pkg/contrib/fileutl.cc | 90 +++++++++++++++++++++++++++++++++++---------- apt-pkg/contrib/fileutl.h | 20 +++++----- ftparchive/multicompress.cc | 30 ++------------- ftparchive/multicompress.h | 1 - 4 files changed, 83 insertions(+), 58 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index c2b684089..60396fc3d 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -50,6 +50,8 @@ #define APT_USE_ZLIB 1 #if APT_USE_ZLIB #include +#else +#warning "Usage of zlib is DISABLED!" #endif #ifdef WORDS_BIGENDIAN @@ -71,7 +73,7 @@ class FileFdPrivate { pid_t compressor_pid; bool pipe; APT::Configuration::Compressor compressor; - FileFd::OpenMode openmode; + unsigned int openmode; FileFdPrivate() : gz(NULL), compressor_pid(-1), pipe(false) {}; }; @@ -858,8 +860,13 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog, for (int J = 0; J != 2; J++) SetCloseExec(Pipe[J],true); + int FileFd = -1; if (Comp == true) + { OutFd = Pipe[1]; + // FIXME: we should handle openmode and permission from Open() here + FileFd = open(FileName.c_str(), O_WRONLY, 0666); + } else OutFd = Pipe[0]; @@ -872,13 +879,14 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog, if (Comp == true) { dup2(Pipe[0],STDIN_FILENO); + dup2(FileFd,STDOUT_FILENO); SetCloseExec(STDIN_FILENO,false); } else { dup2(Pipe[1],STDOUT_FILENO); - SetCloseExec(STDOUT_FILENO,false); } + SetCloseExec(STDOUT_FILENO,false); std::vector Args; Args.push_back(Prog.Binary.c_str()); @@ -887,8 +895,11 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog, for (std::vector::const_iterator a = addArgs->begin(); a != addArgs->end(); ++a) Args.push_back(a->c_str()); - Args.push_back("--stdout"); - Args.push_back(FileName.c_str()); + if (Comp == false) + { + Args.push_back("--stdout"); + Args.push_back(FileName.c_str()); + } Args.push_back(NULL); execvp(Args[0],(char **)&Args[0]); @@ -896,7 +907,10 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog, _exit(100); } if (Comp == true) + { close(Pipe[0]); + close(FileFd); + } else close(Pipe[1]); @@ -910,7 +924,7 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog, // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ -bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long const Perms) +bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const Perms) { if (Mode == ReadOnlyGzip) return Open(FileName, ReadOnly, Gzip, Perms); @@ -934,11 +948,20 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned } else if (Compress == Extension) { - std::string ext = flExtension(FileName); - if (ext == FileName) - ext.clear(); - else - ext = "." + ext; + std::string::size_type const found = FileName.find_last_of('.'); + std::string ext; + if (found != std::string::npos) + { + ext = FileName.substr(found); + if (ext == ".new" || ext == ".bak") + { + std::string::size_type const found2 = FileName.find_last_of('.', found - 1); + if (found2 != std::string::npos) + ext = FileName.substr(found2, found - found2); + else + ext.clear(); + } + } for (; compressor != compressors.end(); ++compressor) if (ext == compressor->Extension) break; @@ -960,8 +983,8 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned case Xz: name = "xz"; break; case Auto: case Extension: - // Unreachable - return _error->Error("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); + // Unreachable + return _error->Error("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); } for (; compressor != compressors.end(); ++compressor) if (compressor->Name == name) @@ -974,7 +997,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); return Open(FileName, Mode, *compressor, Perms); } -bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms) +bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms) { Close(); d = new FileFdPrivate; @@ -1015,8 +1038,35 @@ bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor c if ((Mode & ReadWrite) == ReadWrite) return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor.Name.c_str(), FileName.c_str()); - if (ExecCompressor(compressor, NULL /*d->compressor_pid*/, FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) - return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), FileName.c_str()); + if ((Mode & (WriteOnly | Create)) == (WriteOnly | Create)) + { + if (TemporaryFileName.empty() == false) + { + if (RealFileExists(TemporaryFileName) == false) + { + iFd = open(TemporaryFileName.c_str(), O_WRONLY | O_CREAT, Perms); + close(iFd); + iFd = -1; + } + } + else if (RealFileExists(FileName) == false) + { + iFd = open(FileName.c_str(), O_WRONLY | O_CREAT, Perms); + close(iFd); + iFd = -1; + } + } + + if (TemporaryFileName.empty() == false) + { + if (ExecCompressor(compressor, &(d->compressor_pid), TemporaryFileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), TemporaryFileName.c_str()); + } + else + { + if (ExecCompressor(compressor, &(d->compressor_pid), FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) + return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), FileName.c_str()); + } d->pipe = true; d->compressor = compressor; } @@ -1060,7 +1110,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor c // FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose) +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose) { std::vector const compressors = APT::Configuration::getCompressors(); std::vector::const_iterator compressor = compressors.begin(); @@ -1084,7 +1134,7 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A return OpenDescriptor(Fd, Mode, *compressor, AutoClose); } -bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) { Close(); d = new FileFdPrivate; @@ -1100,7 +1150,7 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compresso this->FileName = ""; return true; } -bool FileFd::OpenInternDescriptor(OpenMode Mode, APT::Configuration::Compressor const &compressor) +bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) { if (compressor.Name == ".") return true; @@ -1471,8 +1521,8 @@ bool FileFd::Close() if (d != NULL) { -// if (d->compressor_pid != -1) -// ExecWait(d->compressor_pid, "FileFdCompressor", true); + if (d->compressor_pid != -1) + ExecWait(d->compressor_pid, "FileFdCompressor", true); delete d; d = NULL; } diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 51277290e..f14f97b69 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -100,14 +100,14 @@ class FileFd return T; } - bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long const Perms = 0666); - bool Open(std::string FileName,OpenMode Mode,APT::Configuration::Compressor const &compressor,unsigned long const Perms = 0666); - inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long const Perms = 0666) { + bool Open(std::string FileName,unsigned int const Mode,CompressMode Compress,unsigned long const Perms = 0666); + bool Open(std::string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor,unsigned long const Perms = 0666); + inline bool Open(std::string const &FileName,unsigned int const Mode, unsigned long const Perms = 0666) { return Open(FileName, Mode, None, Perms); }; - bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); - bool OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compressor const &compressor, bool AutoClose=false); - inline bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false) { + bool OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose=false); + bool OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose=false); + inline bool OpenDescriptor(int Fd, unsigned int const Mode, bool AutoClose=false) { return OpenDescriptor(Fd, Mode, None, AutoClose); }; bool Close(); @@ -126,16 +126,16 @@ class FileFd inline bool IsCompressed() {return (Flags & Compressed) == Compressed;}; inline std::string &Name() {return FileName;}; - FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) + FileFd(std::string FileName,unsigned int const Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) { Open(FileName,Mode, None, Perms); }; - FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) + FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) { Open(FileName,Mode, Compress, Perms); }; FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {}; - FileFd(int const Fd, OpenMode Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL) + FileFd(int const Fd, unsigned int const Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL) { OpenDescriptor(Fd, Mode, Compress); }; @@ -147,7 +147,7 @@ class FileFd private: FileFdPrivate* d; - bool OpenInternDescriptor(OpenMode Mode, APT::Configuration::Compressor const &compressor); + bool OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor); }; bool RunScripts(const char *Cnf); diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 37a713efd..2a930ca6b 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -91,7 +91,7 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress, /* Open all the temp files now so we can report any errors. File is made unreable to prevent people from touching it during creating. */ for (Files *I = Outputs; I != 0; I = I->Next) - I->TmpFile.Open(I->Output + ".new",FileFd::WriteEmpty,0600); + I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600); if (_error->PendingError() == true) return; @@ -183,11 +183,6 @@ bool MultiCompress::Start() _exit(0); }; - /* Tidy up the temp files, we open them in the constructor so as to - get proper error reporting. Close them now. */ - for (Files *I = Outputs; I != 0; I = I->Next) - I->TmpFile.Close(); - close(Pipe[0]); Input = fdopen(Pipe[1],"w"); if (Input == 0) @@ -305,14 +300,6 @@ bool MultiCompress::CloseOld(int Fd,pid_t Proc) is new then the temp files are renamed, otherwise they are erased. */ bool MultiCompress::Child(int const &FD) { - // Start the compression children. - for (Files *I = Outputs; I != 0; I = I->Next) - { - if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(), - I->Fd,true) == false) - return false; - } - /* Okay, now we just feed data from FD to all the other FDs. Also stash a hash of the data to use later. */ SetNonBlock(FD,false); @@ -332,25 +319,14 @@ bool MultiCompress::Child(int const &FD) FileSize += Res; for (Files *I = Outputs; I != 0; I = I->Next) { - if (write(I->Fd,Buffer,Res) != Res) + if (I->TmpFile.Write(Buffer, Res) == false) { _error->Errno("write",_("IO to subprocess/file failed")); break; } } } - - // Close all the writers - for (Files *I = Outputs; I != 0; I = I->Next) - close(I->Fd); - - // Wait for the compressors to exit - for (Files *I = Outputs; I != 0; I = I->Next) - { - if (I->CompressProc != -1) - ExecWait(I->CompressProc, I->CompressProg.Binary.c_str(), false); - } - + if (_error->PendingError() == true) return false; diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index ac8bd0746..2dc7095d7 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -34,7 +34,6 @@ class MultiCompress FileFd TmpFile; pid_t CompressProc; time_t OldMTime; - int Fd; }; Files *Outputs; -- cgit v1.2.3 From 12d1f5b3e295c589371bf7de27b7918310d08480 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 14 Dec 2011 22:35:03 +0100 Subject: remove the second usage instance of ExecCompressor in ftparchive by again using the FileFd directly --- ftparchive/multicompress.cc | 42 +++++++++--------------------------------- ftparchive/multicompress.h | 3 +-- ftparchive/writer.cc | 23 ++++++++--------------- 3 files changed, 18 insertions(+), 50 deletions(-) diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 2a930ca6b..1fea589e2 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -260,7 +260,7 @@ bool MultiCompress::Finalize(unsigned long long &OutSize) // MultiCompress::OpenOld - Open an old file /*{{{*/ // --------------------------------------------------------------------- /* This opens one of the original output files, possibly decompressing it. */ -bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) +bool MultiCompress::OpenOld(FileFd &Fd) { Files *Best = Outputs; for (Files *I = Outputs; I != 0; I = I->Next) @@ -268,29 +268,9 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) Best = I; // Open the file - FileFd F(Best->Output,FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - - // Decompress the file so we can read it - if (ExecCompressor(Best->CompressProg,&Proc,F.Fd(),Fd,false) == false) - return false; - - return true; + return Fd.Open(Best->Output, FileFd::ReadOnly, FileFd::Extension); } /*}}}*/ -// MultiCompress::CloseOld - Close the old file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool MultiCompress::CloseOld(int Fd,pid_t Proc) -{ - close(Fd); - if (Proc != -1) - if (ExecWait(Proc,_("decompressor"),false) == false) - return false; - return true; -} - /*}}}*/ // MultiCompress::Child - The writer child /*{{{*/ // --------------------------------------------------------------------- /* The child process forks a bunch of compression children and takes @@ -345,31 +325,27 @@ bool MultiCompress::Child(int const &FD) // Check the MD5 of the lowest cost entity. while (Missing == false) { - int CompFd = -1; - pid_t Proc = -1; - if (OpenOld(CompFd,Proc) == false) + FileFd CompFd; + if (OpenOld(CompFd) == false) { _error->Discard(); break; } - + // Compute the hash MD5Summation OldMD5; unsigned long long NewFileSize = 0; while (1) { - int Res = read(CompFd,Buffer,sizeof(Buffer)); + unsigned long long Res = 0; + if (CompFd.Read(Buffer,sizeof(Buffer), &Res) == false) + return _error->Errno("read",_("Failed to read while computing MD5")); if (Res == 0) break; - if (Res < 0) - return _error->Errno("read",_("Failed to read while computing MD5")); NewFileSize += Res; OldMD5.Add(Buffer,Res); } - - // Tidy the compressor - if (CloseOld(CompFd,Proc) == false) - return false; + CompFd.Close(); // Check the hash if (OldMD5.Result() == MD5.Result() && diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index 2dc7095d7..388fad22e 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -51,8 +51,7 @@ class MultiCompress unsigned long UpdateMTime; bool Finalize(unsigned long long &OutSize); - bool OpenOld(int &Fd,pid_t &Proc); - bool CloseOld(int Fd,pid_t Proc); + bool OpenOld(FileFd &Fd); static bool GetStat(std::string const &Output,std::string const &Compress,struct stat &St); MultiCompress(std::string const &Output,std::string const &Compress, diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 27e53faf8..02777713c 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -889,22 +889,16 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres MultiCompress Pkgs(PkgFile,PkgCompress,0,false); if (_error->PendingError() == true) return false; - + // Open the package file - int CompFd = -1; - pid_t Proc = -1; - if (Pkgs.OpenOld(CompFd,Proc) == false) + FileFd Fd; + if (Pkgs.OpenOld(Fd) == false) return false; - - // No auto-close FD - FileFd Fd(CompFd,false); + pkgTagFile Tags(&Fd); if (_error->PendingError() == true) - { - Pkgs.CloseOld(CompFd,Proc); return false; - } - + // Parse. pkgTagSection Section; while (Tags.Step(Section) == true) @@ -926,11 +920,10 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres _error->DumpErrors(); } } - + // Tidy the compressor - if (Pkgs.CloseOld(CompFd,Proc) == false) - return false; - + Fd.Close(); + return true; } -- cgit v1.2.3 From 73688d27f60b2da3889a06362ee567101e3b331e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 15 Dec 2011 09:13:21 +0100 Subject: =?UTF-8?q?atleast=20libapt=20should=20announce=20to=20itself=20th?= =?UTF-8?q?at=20it=20is=20clean=E2=80=A6=20(and=20be=20it=20if=20it=20trie?= =?UTF-8?q?s=20to=20announce=20that=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apt-pkg/deb/debindexfile.cc | 2 ++ apt-pkg/edsp/edspindexfile.cc | 4 ++-- apt-pkg/pkgcachegen.cc | 16 +++++++++------- apt-pkg/sourcelist.h | 4 ++-- apt-pkg/srcrecords.cc | 8 ++++---- buildlib/config.h.in | 2 ++ cmdline/apt-extracttemplates.h | 10 ++++++---- 7 files changed, 27 insertions(+), 19 deletions(-) diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 2635d52c8..84791a70a 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -26,6 +26,8 @@ #include /*}}}*/ +using std::string; + // SourcesIndex::debSourcesIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc index 5d824f9cb..482581979 100644 --- a/apt-pkg/edsp/edspindexfile.cc +++ b/apt-pkg/edsp/edspindexfile.cc @@ -24,7 +24,7 @@ // edspIndex::edspIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -edspIndex::edspIndex(string File) : debStatusIndex(File) +edspIndex::edspIndex(std::string File) : debStatusIndex(File) { } /*}}}*/ @@ -44,7 +44,7 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const if (Prog != NULL) Prog->SubProgress(0,File); - if (Gen.SelectFile(File,string(),*this) == false) + if (Gen.SelectFile(File,std::string(),*this) == false) return _error->Error("Problem with SelectFile %s",File.c_str()); // Store the IMS information diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 9f999c41b..ec072fddd 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -35,12 +35,14 @@ #include /*}}}*/ -typedef vector::iterator FileIterator; +typedef std::vector::iterator FileIterator; template std::vector pkgCacheGenerator::Dynamic::toReMap; bool IsDuplicateDescription(pkgCache::DescIterator Desc, MD5SumValue const &CurMd5, std::string const &CurLang); +using std::string; + // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- /* We set the dirty flag and make sure that is written to the disk */ @@ -1221,14 +1223,14 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress MMap **OutMap,bool AllowMem) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); - - vector Files; - for (vector::const_iterator i = List.begin(); + + std::vector Files; + for (std::vector::const_iterator i = List.begin(); i != List.end(); ++i) { - vector *Indexes = (*i)->GetIndexFiles(); - for (vector::const_iterator j = Indexes->begin(); + std::vector *Indexes = (*i)->GetIndexFiles(); + for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) Files.push_back (*j); @@ -1418,7 +1420,7 @@ __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutM { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); } bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) { - vector Files; + std::vector Files; unsigned long EndOfSource = Files.size(); if (_system->AddStatusFiles(Files) == false) return false; diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 03e29ec34..02e27101a 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -59,11 +59,11 @@ class pkgSourceList const char *Name; const char *Label; - bool FixupURI(string &URI) const; + bool FixupURI(std::string &URI) const; virtual bool ParseLine(std::vector &List, const char *Buffer, unsigned long const &CurLine,std::string const &File) const; - virtual bool CreateItem(vector &List,std::string const &URI, + virtual bool CreateItem(std::vector &List,std::string const &URI, std::string const &Dist,std::string const &Section, std::map const &Options) const = 0; Type(); diff --git a/apt-pkg/srcrecords.cc b/apt-pkg/srcrecords.cc index f6d2d5158..48b643eac 100644 --- a/apt-pkg/srcrecords.cc +++ b/apt-pkg/srcrecords.cc @@ -29,8 +29,8 @@ pkgSrcRecords::pkgSrcRecords(pkgSourceList &List) : Files(0), Current(0) { for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); ++I) { - vector *Indexes = (*I)->GetIndexFiles(); - for (vector::const_iterator J = Indexes->begin(); + std::vector *Indexes = (*I)->GetIndexFiles(); + for (std::vector::const_iterator J = Indexes->begin(); J != Indexes->end(); ++J) { Parser* P = (*J)->CreateSrcParser(); @@ -58,7 +58,7 @@ pkgSrcRecords::pkgSrcRecords(pkgSourceList &List) : Files(0), Current(0) pkgSrcRecords::~pkgSrcRecords() { // Blow away all the parser objects - for(vector::iterator I = Files.begin(); I != Files.end(); ++I) + for(std::vector::iterator I = Files.begin(); I != Files.end(); ++I) delete *I; } /*}}}*/ @@ -68,7 +68,7 @@ pkgSrcRecords::~pkgSrcRecords() bool pkgSrcRecords::Restart() { Current = Files.begin(); - for (vector::iterator I = Files.begin(); + for (std::vector::iterator I = Files.begin(); I != Files.end(); ++I) (*I)->Restart(); diff --git a/buildlib/config.h.in b/buildlib/config.h.in index 256911231..4798fe3f5 100644 --- a/buildlib/config.h.in +++ b/buildlib/config.h.in @@ -44,3 +44,5 @@ /* The package name string */ #undef PACKAGE + +#define APT_8_CLEANER_HEADERS diff --git a/cmdline/apt-extracttemplates.h b/cmdline/apt-extracttemplates.h index d0f90d3f4..6d07a09c2 100644 --- a/cmdline/apt-extracttemplates.h +++ b/cmdline/apt-extracttemplates.h @@ -14,6 +14,8 @@ #include #include +#include + class DebFile : public pkgDirStream { FileFd File; @@ -31,11 +33,11 @@ public: bool Go(); bool ParseInfo(); - static string GetInstalledVer(const string &package); + static std::string GetInstalledVer(const std::string &package); - string Package; - string Version; - string DepVer, PreDepVer; + std::string Package; + std::string Version; + std::string DepVer, PreDepVer; unsigned int DepOp, PreDepOp; char *Config; -- cgit v1.2.3 From 561f860a385087ea4d863f01cb5e0e050a5e360f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 15 Dec 2011 23:38:38 +0100 Subject: refactor compressor calling so that we don't (need to) export ExecCompressor anymore and therefore are also able to drop quiet a bit of duplicated code --- apt-pkg/contrib/fileutl.cc | 376 +++++++++++++++------------------------------ apt-pkg/contrib/fileutl.h | 8 - 2 files changed, 120 insertions(+), 264 deletions(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 60396fc3d..44486905f 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -51,7 +51,7 @@ #if APT_USE_ZLIB #include #else -#warning "Usage of zlib is DISABLED!" +#pragma message "Usage of zlib is DISABLED!" #endif #ifdef WORDS_BIGENDIAN @@ -70,11 +70,12 @@ class FileFdPrivate { #else void* gz; #endif + int compressed_fd; pid_t compressor_pid; bool pipe; APT::Configuration::Compressor compressor; unsigned int openmode; - FileFdPrivate() : gz(NULL), compressor_pid(-1), pipe(false) {}; + FileFdPrivate() : gz(NULL), compressed_fd(-1), compressor_pid(-1), pipe(false) {}; }; // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ @@ -740,187 +741,6 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) } /*}}}*/ -// ExecCompressor - Open a de/compressor pipe /*{{{*/ -// --------------------------------------------------------------------- -/* This opens the compressor, either in compress mode or decompress - mode. FileFd is always the compressor input/output file, - OutFd is the created pipe, Input for Compress, Output for Decompress. */ -bool ExecCompressor(APT::Configuration::Compressor const &Prog, - pid_t *Pid, int const FileFd, int &OutFd, bool const Comp) -{ - if (Pid != NULL) - *Pid = -1; - - // No compression - if (Prog.Binary.empty() == true) - { - OutFd = dup(FileFd); - return true; - } - - // Handle 'decompression' of empty files - if (Comp == false) - { - struct stat Buf; - fstat(FileFd, &Buf); - if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) - { - OutFd = FileFd; - return true; - } - } - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - if (Comp == true) - OutFd = Pipe[1]; - else - OutFd = Pipe[0]; - - // The child.. - pid_t child = ExecFork(); - if (Pid != NULL) - *Pid = child; - if (child == 0) - { - if (Comp == true) - { - dup2(FileFd,STDOUT_FILENO); - dup2(Pipe[0],STDIN_FILENO); - } - else - { - dup2(FileFd,STDIN_FILENO); - dup2(Pipe[1],STDOUT_FILENO); - } - - SetCloseExec(STDOUT_FILENO,false); - SetCloseExec(STDIN_FILENO,false); - - std::vector Args; - Args.push_back(Prog.Binary.c_str()); - std::vector const * const addArgs = - (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); - for (std::vector::const_iterator a = addArgs->begin(); - a != addArgs->end(); ++a) - Args.push_back(a->c_str()); - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - } - if (Comp == true) - close(Pipe[0]); - else - close(Pipe[1]); - - if (Pid == NULL) - ExecWait(child, Prog.Binary.c_str(), true); - - return true; -} -bool ExecCompressor(APT::Configuration::Compressor const &Prog, - pid_t *Pid, std::string const &FileName, int &OutFd, bool const Comp) -{ - if (Pid != NULL) - *Pid = -1; - - // No compression - if (Prog.Binary.empty() == true) - { - if (Comp == true) - OutFd = open(FileName.c_str(), O_WRONLY, 0666); - else - OutFd = open(FileName.c_str(), O_RDONLY); - return true; - } - - // Handle 'decompression' of empty files - if (Comp == false) - { - struct stat Buf; - stat(FileName.c_str(), &Buf); - if (Buf.st_size == 0) - { - OutFd = open(FileName.c_str(), O_RDONLY); - return true; - } - } - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - int FileFd = -1; - if (Comp == true) - { - OutFd = Pipe[1]; - // FIXME: we should handle openmode and permission from Open() here - FileFd = open(FileName.c_str(), O_WRONLY, 0666); - } - else - OutFd = Pipe[0]; - - // The child.. - pid_t child = ExecFork(); - if (Pid != NULL) - *Pid = child; - if (child == 0) - { - if (Comp == true) - { - dup2(Pipe[0],STDIN_FILENO); - dup2(FileFd,STDOUT_FILENO); - SetCloseExec(STDIN_FILENO,false); - } - else - { - dup2(Pipe[1],STDOUT_FILENO); - } - SetCloseExec(STDOUT_FILENO,false); - - std::vector Args; - Args.push_back(Prog.Binary.c_str()); - std::vector const * const addArgs = - (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); - for (std::vector::const_iterator a = addArgs->begin(); - a != addArgs->end(); ++a) - Args.push_back(a->c_str()); - if (Comp == false) - { - Args.push_back("--stdout"); - Args.push_back(FileName.c_str()); - } - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - } - if (Comp == true) - { - close(Pipe[0]); - close(FileFd); - } - else - close(Pipe[1]); - - if (Pid == NULL) - ExecWait(child, Prog.Binary.c_str(), false); - - return true; -} - /*}}}*/ - // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ @@ -1028,80 +848,33 @@ bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Co unlink(FileName.c_str()); } - // if we have them, use inbuilt compressors instead of forking - if (compressor.Name != "." -#if APT_USE_ZLIB - && compressor.Name != "gzip" -#endif - ) - { - if ((Mode & ReadWrite) == ReadWrite) - return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor.Name.c_str(), FileName.c_str()); + int fileflags = 0; + #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE + if_FLAGGED_SET(ReadWrite, O_RDWR); + else if_FLAGGED_SET(ReadOnly, O_RDONLY); + else if_FLAGGED_SET(WriteOnly, O_WRONLY); - if ((Mode & (WriteOnly | Create)) == (WriteOnly | Create)) - { - if (TemporaryFileName.empty() == false) - { - if (RealFileExists(TemporaryFileName) == false) - { - iFd = open(TemporaryFileName.c_str(), O_WRONLY | O_CREAT, Perms); - close(iFd); - iFd = -1; - } - } - else if (RealFileExists(FileName) == false) - { - iFd = open(FileName.c_str(), O_WRONLY | O_CREAT, Perms); - close(iFd); - iFd = -1; - } - } + if_FLAGGED_SET(Create, O_CREAT); + if_FLAGGED_SET(Empty, O_TRUNC); + if_FLAGGED_SET(Exclusive, O_EXCL); + else if_FLAGGED_SET(Atomic, O_EXCL); + #undef if_FLAGGED_SET - if (TemporaryFileName.empty() == false) - { - if (ExecCompressor(compressor, &(d->compressor_pid), TemporaryFileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) - return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), TemporaryFileName.c_str()); - } - else - { - if (ExecCompressor(compressor, &(d->compressor_pid), FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false) - return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), FileName.c_str()); - } - d->pipe = true; - d->compressor = compressor; - } + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), fileflags, Perms); else - { - int fileflags = 0; - #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE - if_FLAGGED_SET(ReadWrite, O_RDWR); - else if_FLAGGED_SET(ReadOnly, O_RDONLY); - else if_FLAGGED_SET(WriteOnly, O_WRONLY); - - if_FLAGGED_SET(Create, O_CREAT); - if_FLAGGED_SET(Exclusive, O_EXCL); - else if_FLAGGED_SET(Atomic, O_EXCL); - if_FLAGGED_SET(Empty, O_TRUNC); - #undef if_FLAGGED_SET - - if (TemporaryFileName.empty() == false) - iFd = open(TemporaryFileName.c_str(), fileflags, Perms); - else - iFd = open(FileName.c_str(), fileflags, Perms); + iFd = open(FileName.c_str(), fileflags, Perms); + if (iFd == -1 || OpenInternDescriptor(Mode, compressor) == false) + { if (iFd != -1) { - if (OpenInternDescriptor(Mode, compressor) == false) - { - close (iFd); - iFd = -1; - } + close (iFd); + iFd = -1; } + return _error->Errno("open",_("Could not open file %s"), FileName.c_str()); } - if (iFd == -1) - return _error->Errno("open",_("Could not open file %s"),FileName.c_str()); - this->FileName = FileName; SetCloseExec(iFd,true); return true; @@ -1152,7 +925,8 @@ bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration: } bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) { - if (compressor.Name == ".") + d->compressor = compressor; + if (compressor.Name == "." || compressor.Binary.empty() == true) return true; #if APT_USE_ZLIB else if (compressor.Name == "gzip") @@ -1166,10 +940,90 @@ bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::C if (d->gz == NULL) return false; Flags |= Compressed; + return true; } #endif + + if ((Mode & ReadWrite) == ReadWrite) + return _error->Error("ReadWrite mode is not supported for file %s", FileName.c_str()); + + bool const Comp = (Mode & WriteOnly) == WriteOnly; + // Handle 'decompression' of empty files + if (Comp == false) + { + struct stat Buf; + fstat(iFd, &Buf); + if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) + return true; + + // We don't need the file open - instead let the compressor open it + // as he properly knows better how to efficiently read from 'his' file + if (FileName.empty() == false) + close(iFd); + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return _error->Errno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + d->compressed_fd = iFd; + d->pipe = true; + + if (Comp == true) + iFd = Pipe[1]; + else + iFd = Pipe[0]; + + // The child.. + d->compressor_pid = ExecFork(); + if (d->compressor_pid == 0) + { + if (Comp == true) + { + dup2(d->compressed_fd,STDOUT_FILENO); + dup2(Pipe[0],STDIN_FILENO); + } + else + { + if (FileName.empty() == true) + dup2(d->compressed_fd,STDIN_FILENO); + dup2(Pipe[1],STDOUT_FILENO); + } + + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + + std::vector Args; + Args.push_back(compressor.Binary.c_str()); + std::vector const * const addArgs = + (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs); + for (std::vector::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + if (Comp == false && FileName.empty() == false) + { + Args.push_back("--stdout"); + if (TemporaryFileName.empty() == false) + Args.push_back(TemporaryFileName.c_str()); + else + Args.push_back(FileName.c_str()); + } + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); else - return _error->Error("Can't find a match for specified compressor %s for file %s", compressor.Name.c_str(), FileName.c_str()); + close(Pipe[1]); + if (Comp == true || FileName.empty() == true) + close(d->compressed_fd); + return true; } /*}}}*/ @@ -1297,12 +1151,22 @@ bool FileFd::Seek(unsigned long long To) { if (d->pipe == true) { - // FIXME: What about OpenDescriptor() stuff here? + if ((d->openmode & ReadOnly) != ReadOnly) + return _error->Error("Reopen is only implemented for read-only files!"); close(iFd); - bool result = ExecCompressor(d->compressor, NULL, FileName, iFd, (d->openmode & ReadOnly) != ReadOnly); - if (result == true && To != 0) - result &= Skip(To); - return result; + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), O_RDONLY); + else if (FileName.empty() == false) + iFd = open(FileName.c_str(), O_RDONLY); + else + return _error->Error("Reopen is not implemented for OpenDescriptor()-FileFd!"); + + if (OpenInternDescriptor(d->openmode, d->compressor) == false) + return _error->Error("Seek on file %s because it couldn't be reopened", FileName.c_str()); + + if (To != 0) + return Skip(To); + return true; } int res; #if APT_USE_ZLIB @@ -1521,7 +1385,7 @@ bool FileFd::Close() if (d != NULL) { - if (d->compressor_pid != -1) + if (d->compressor_pid > 0) ExecWait(d->compressor_pid, "FileFdCompressor", true); delete d; d = NULL; diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index f14f97b69..147535df1 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -178,14 +178,6 @@ bool WaitFd(int Fd,bool write = false,unsigned long timeout = 0); pid_t ExecFork(); bool ExecWait(pid_t Pid,const char *Name,bool Reap = false); -bool ExecCompressor(APT::Configuration::Compressor const &Prog, - pid_t *Pid, int const FileFd, int &OutFd, bool const Comp = true); -inline bool ExecDecompressor(APT::Configuration::Compressor const &Prog, - pid_t *Pid, int const FileFd, int &OutFd) -{ - return ExecCompressor(Prog, Pid, FileFd, OutFd, true); -} - // File string manipulators std::string flNotDir(std::string File); std::string flNotFile(std::string File); -- cgit v1.2.3 From 6fd947bd48449652edf783cfb1362391e63f9be1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 16 Dec 2011 00:04:52 +0100 Subject: try seeking on fds opened with OpenDescriptor before giving up --- apt-pkg/contrib/fileutl.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 44486905f..a98c2cb85 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1154,12 +1154,19 @@ bool FileFd::Seek(unsigned long long To) if ((d->openmode & ReadOnly) != ReadOnly) return _error->Error("Reopen is only implemented for read-only files!"); close(iFd); + iFd = 0; if (TemporaryFileName.empty() == false) iFd = open(TemporaryFileName.c_str(), O_RDONLY); else if (FileName.empty() == false) iFd = open(FileName.c_str(), O_RDONLY); else - return _error->Error("Reopen is not implemented for OpenDescriptor()-FileFd!"); + { + if (d->compressed_fd > 0) + if (lseek(d->compressed_fd, 0, SEEK_SET) != 0) + iFd = d->compressed_fd; + if (iFd <= 0) + return _error->Error("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!"); + } if (OpenInternDescriptor(d->openmode, d->compressor) == false) return _error->Error("Seek on file %s because it couldn't be reopened", FileName.c_str()); -- cgit v1.2.3