From 699b209e5122f8fcd85fc4666c9b7020286ab0d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 00:17:30 +0100 Subject: Allow the FileFd to use an external Compressor to uncompress a given file internally so that it is exported and can be used like a "normal" uncompressed file with FileFd This allows us to hide th zlib usage in the implementation and use gzip instead if we don't have zlib builtin (the same for other compressors). The code includes quiet a few FIXME's so while all tests are working it shouldn't be used just yet outside of libapt as it might break. --- ftparchive/multicompress.cc | 74 +++------------------------------------------ ftparchive/multicompress.h | 2 -- 2 files changed, 4 insertions(+), 72 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index bf0f858d9..37a713efd 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -16,6 +16,7 @@ // Include Files /*{{{*/ #include +#include #include #include #include @@ -261,73 +262,6 @@ bool MultiCompress::Finalize(unsigned long long &OutSize) return true; } /*}}}*/ -// MultiCompress::OpenCompress - Open the compressor /*{{{*/ -// --------------------------------------------------------------------- -/* This opens the compressor, either in compress mode or decompress - mode. FileFd is always the compressor input/output file, - OutFd is the created pipe, Input for Compress, Output for Decompress. */ -bool MultiCompress::OpenCompress(APT::Configuration::Compressor const &Prog, - pid_t &Pid,int const &FileFd,int &OutFd,bool const &Comp) -{ - Pid = -1; - - // No compression - if (Prog.Binary.empty() == true) - { - OutFd = dup(FileFd); - return true; - } - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - if (Comp == true) - OutFd = Pipe[1]; - else - OutFd = Pipe[0]; - - // The child.. - Pid = ExecFork(); - if (Pid == 0) - { - if (Comp == true) - { - dup2(FileFd,STDOUT_FILENO); - dup2(Pipe[0],STDIN_FILENO); - } - else - { - dup2(FileFd,STDIN_FILENO); - dup2(Pipe[1],STDOUT_FILENO); - } - - SetCloseExec(STDOUT_FILENO,false); - SetCloseExec(STDIN_FILENO,false); - - std::vector Args; - Args.push_back(Prog.Binary.c_str()); - std::vector const * const addArgs = - (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); - for (std::vector::const_iterator a = addArgs->begin(); - a != addArgs->end(); ++a) - Args.push_back(a->c_str()); - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - }; - if (Comp == true) - close(Pipe[0]); - else - close(Pipe[1]); - return true; -} - /*}}}*/ // MultiCompress::OpenOld - Open an old file /*{{{*/ // --------------------------------------------------------------------- /* This opens one of the original output files, possibly decompressing it. */ @@ -344,7 +278,7 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) return false; // Decompress the file so we can read it - if (OpenCompress(Best->CompressProg,Proc,F.Fd(),Fd,false) == false) + if (ExecCompressor(Best->CompressProg,&Proc,F.Fd(),Fd,false) == false) return false; return true; @@ -374,8 +308,8 @@ bool MultiCompress::Child(int const &FD) // Start the compression children. for (Files *I = Outputs; I != 0; I = I->Next) { - if (OpenCompress(I->CompressProg,I->CompressProc,I->TmpFile.Fd(), - I->Fd,true) == false) + if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(), + I->Fd,true) == false) return false; } diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index 5496b06d0..ac8bd0746 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -41,8 +41,6 @@ class MultiCompress pid_t Outputter; mode_t Permissions; - bool OpenCompress(APT::Configuration::Compressor const &Prog, - pid_t &Pid,int const &FileFd, int &OutFd,bool const &Comp); bool Child(int const &Fd); bool Start(); bool Die(); -- cgit v1.2.3 From 52b47296f61ec3ca1075bbfb44982f5caa541e7c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 14 Dec 2011 22:11:43 +0100 Subject: use FileFd instead of forking the compression childs by hand --- ftparchive/multicompress.cc | 30 +++--------------------------- ftparchive/multicompress.h | 1 - 2 files changed, 3 insertions(+), 28 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 37a713efd..2a930ca6b 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -91,7 +91,7 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress, /* Open all the temp files now so we can report any errors. File is made unreable to prevent people from touching it during creating. */ for (Files *I = Outputs; I != 0; I = I->Next) - I->TmpFile.Open(I->Output + ".new",FileFd::WriteEmpty,0600); + I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600); if (_error->PendingError() == true) return; @@ -183,11 +183,6 @@ bool MultiCompress::Start() _exit(0); }; - /* Tidy up the temp files, we open them in the constructor so as to - get proper error reporting. Close them now. */ - for (Files *I = Outputs; I != 0; I = I->Next) - I->TmpFile.Close(); - close(Pipe[0]); Input = fdopen(Pipe[1],"w"); if (Input == 0) @@ -305,14 +300,6 @@ bool MultiCompress::CloseOld(int Fd,pid_t Proc) is new then the temp files are renamed, otherwise they are erased. */ bool MultiCompress::Child(int const &FD) { - // Start the compression children. - for (Files *I = Outputs; I != 0; I = I->Next) - { - if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(), - I->Fd,true) == false) - return false; - } - /* Okay, now we just feed data from FD to all the other FDs. Also stash a hash of the data to use later. */ SetNonBlock(FD,false); @@ -332,25 +319,14 @@ bool MultiCompress::Child(int const &FD) FileSize += Res; for (Files *I = Outputs; I != 0; I = I->Next) { - if (write(I->Fd,Buffer,Res) != Res) + if (I->TmpFile.Write(Buffer, Res) == false) { _error->Errno("write",_("IO to subprocess/file failed")); break; } } } - - // Close all the writers - for (Files *I = Outputs; I != 0; I = I->Next) - close(I->Fd); - - // Wait for the compressors to exit - for (Files *I = Outputs; I != 0; I = I->Next) - { - if (I->CompressProc != -1) - ExecWait(I->CompressProc, I->CompressProg.Binary.c_str(), false); - } - + if (_error->PendingError() == true) return false; diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index ac8bd0746..2dc7095d7 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -34,7 +34,6 @@ class MultiCompress FileFd TmpFile; pid_t CompressProc; time_t OldMTime; - int Fd; }; Files *Outputs; -- cgit v1.2.3 From 12d1f5b3e295c589371bf7de27b7918310d08480 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 14 Dec 2011 22:35:03 +0100 Subject: remove the second usage instance of ExecCompressor in ftparchive by again using the FileFd directly --- ftparchive/multicompress.cc | 42 +++++++++--------------------------------- ftparchive/multicompress.h | 3 +-- ftparchive/writer.cc | 23 ++++++++--------------- 3 files changed, 18 insertions(+), 50 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 2a930ca6b..1fea589e2 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -260,7 +260,7 @@ bool MultiCompress::Finalize(unsigned long long &OutSize) // MultiCompress::OpenOld - Open an old file /*{{{*/ // --------------------------------------------------------------------- /* This opens one of the original output files, possibly decompressing it. */ -bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) +bool MultiCompress::OpenOld(FileFd &Fd) { Files *Best = Outputs; for (Files *I = Outputs; I != 0; I = I->Next) @@ -268,29 +268,9 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) Best = I; // Open the file - FileFd F(Best->Output,FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - - // Decompress the file so we can read it - if (ExecCompressor(Best->CompressProg,&Proc,F.Fd(),Fd,false) == false) - return false; - - return true; + return Fd.Open(Best->Output, FileFd::ReadOnly, FileFd::Extension); } /*}}}*/ -// MultiCompress::CloseOld - Close the old file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool MultiCompress::CloseOld(int Fd,pid_t Proc) -{ - close(Fd); - if (Proc != -1) - if (ExecWait(Proc,_("decompressor"),false) == false) - return false; - return true; -} - /*}}}*/ // MultiCompress::Child - The writer child /*{{{*/ // --------------------------------------------------------------------- /* The child process forks a bunch of compression children and takes @@ -345,31 +325,27 @@ bool MultiCompress::Child(int const &FD) // Check the MD5 of the lowest cost entity. while (Missing == false) { - int CompFd = -1; - pid_t Proc = -1; - if (OpenOld(CompFd,Proc) == false) + FileFd CompFd; + if (OpenOld(CompFd) == false) { _error->Discard(); break; } - + // Compute the hash MD5Summation OldMD5; unsigned long long NewFileSize = 0; while (1) { - int Res = read(CompFd,Buffer,sizeof(Buffer)); + unsigned long long Res = 0; + if (CompFd.Read(Buffer,sizeof(Buffer), &Res) == false) + return _error->Errno("read",_("Failed to read while computing MD5")); if (Res == 0) break; - if (Res < 0) - return _error->Errno("read",_("Failed to read while computing MD5")); NewFileSize += Res; OldMD5.Add(Buffer,Res); } - - // Tidy the compressor - if (CloseOld(CompFd,Proc) == false) - return false; + CompFd.Close(); // Check the hash if (OldMD5.Result() == MD5.Result() && diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index 2dc7095d7..388fad22e 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -51,8 +51,7 @@ class MultiCompress unsigned long UpdateMTime; bool Finalize(unsigned long long &OutSize); - bool OpenOld(int &Fd,pid_t &Proc); - bool CloseOld(int Fd,pid_t Proc); + bool OpenOld(FileFd &Fd); static bool GetStat(std::string const &Output,std::string const &Compress,struct stat &St); MultiCompress(std::string const &Output,std::string const &Compress, diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 27e53faf8..02777713c 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -889,22 +889,16 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres MultiCompress Pkgs(PkgFile,PkgCompress,0,false); if (_error->PendingError() == true) return false; - + // Open the package file - int CompFd = -1; - pid_t Proc = -1; - if (Pkgs.OpenOld(CompFd,Proc) == false) + FileFd Fd; + if (Pkgs.OpenOld(Fd) == false) return false; - - // No auto-close FD - FileFd Fd(CompFd,false); + pkgTagFile Tags(&Fd); if (_error->PendingError() == true) - { - Pkgs.CloseOld(CompFd,Proc); return false; - } - + // Parse. pkgTagSection Section; while (Tags.Step(Section) == true) @@ -926,11 +920,10 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres _error->DumpErrors(); } } - + // Tidy the compressor - if (Pkgs.CloseOld(CompFd,Proc) == false) - return false; - + Fd.Close(); + return true; } -- cgit v1.2.3