From ea54214002c09eeb4dd498d97a564471ec9993c5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Sep 2011 10:09:00 +0200 Subject: reorder includes: add if needed and include it at first --- ftparchive/multicompress.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'ftparchive/multicompress.cc') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index f82879015..04560f4ab 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -14,18 +14,20 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ -#include "multicompress.h" - -#include +#include + #include #include #include - + #include #include #include #include -#include +#include + +#include "multicompress.h" +#include /*}}}*/ using namespace std; -- cgit v1.2.3 From 650faab01603caac04494d54cf6b10a65c00ea13 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Sep 2011 17:46:48 +0200 Subject: Support large files in the complete toolset. Indexes of this size are pretty unlikely for now, but we need it for deb packages which could become bigger than 4GB now (LP: #815895) --- ftparchive/multicompress.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'ftparchive/multicompress.cc') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 04560f4ab..a80d6a34d 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -215,7 +215,7 @@ bool MultiCompress::Die() // MultiCompress::Finalize - Finish up writing /*{{{*/ // --------------------------------------------------------------------- /* This is only necessary for statistics reporting. */ -bool MultiCompress::Finalize(unsigned long &OutSize) +bool MultiCompress::Finalize(unsigned long long &OutSize) { OutSize = 0; if (Input == 0 || Die() == false) @@ -383,7 +383,7 @@ bool MultiCompress::Child(int const &FD) stash a hash of the data to use later. */ SetNonBlock(FD,false); unsigned char Buffer[32*1024]; - unsigned long FileSize = 0; + unsigned long long FileSize = 0; MD5Summation MD5; while (1) { @@ -445,7 +445,7 @@ bool MultiCompress::Child(int const &FD) // Compute the hash MD5Summation OldMD5; - unsigned long NewFileSize = 0; + unsigned long long NewFileSize = 0; while (1) { int Res = read(CompFd,Buffer,sizeof(Buffer)); -- cgit v1.2.3 From 699b209e5122f8fcd85fc4666c9b7020286ab0d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 13 Dec 2011 00:17:30 +0100 Subject: Allow the FileFd to use an external Compressor to uncompress a given file internally so that it is exported and can be used like a "normal" uncompressed file with FileFd This allows us to hide th zlib usage in the implementation and use gzip instead if we don't have zlib builtin (the same for other compressors). The code includes quiet a few FIXME's so while all tests are working it shouldn't be used just yet outside of libapt as it might break. --- ftparchive/multicompress.cc | 74 +++------------------------------------------ 1 file changed, 4 insertions(+), 70 deletions(-) (limited to 'ftparchive/multicompress.cc') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index bf0f858d9..37a713efd 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -16,6 +16,7 @@ // Include Files /*{{{*/ #include +#include #include #include #include @@ -261,73 +262,6 @@ bool MultiCompress::Finalize(unsigned long long &OutSize) return true; } /*}}}*/ -// MultiCompress::OpenCompress - Open the compressor /*{{{*/ -// --------------------------------------------------------------------- -/* This opens the compressor, either in compress mode or decompress - mode. FileFd is always the compressor input/output file, - OutFd is the created pipe, Input for Compress, Output for Decompress. */ -bool MultiCompress::OpenCompress(APT::Configuration::Compressor const &Prog, - pid_t &Pid,int const &FileFd,int &OutFd,bool const &Comp) -{ - Pid = -1; - - // No compression - if (Prog.Binary.empty() == true) - { - OutFd = dup(FileFd); - return true; - } - - // Create a data pipe - int Pipe[2] = {-1,-1}; - if (pipe(Pipe) != 0) - return _error->Errno("pipe",_("Failed to create subprocess IPC")); - for (int J = 0; J != 2; J++) - SetCloseExec(Pipe[J],true); - - if (Comp == true) - OutFd = Pipe[1]; - else - OutFd = Pipe[0]; - - // The child.. - Pid = ExecFork(); - if (Pid == 0) - { - if (Comp == true) - { - dup2(FileFd,STDOUT_FILENO); - dup2(Pipe[0],STDIN_FILENO); - } - else - { - dup2(FileFd,STDIN_FILENO); - dup2(Pipe[1],STDOUT_FILENO); - } - - SetCloseExec(STDOUT_FILENO,false); - SetCloseExec(STDIN_FILENO,false); - - std::vector Args; - Args.push_back(Prog.Binary.c_str()); - std::vector const * const addArgs = - (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs); - for (std::vector::const_iterator a = addArgs->begin(); - a != addArgs->end(); ++a) - Args.push_back(a->c_str()); - Args.push_back(NULL); - - execvp(Args[0],(char **)&Args[0]); - cerr << _("Failed to exec compressor ") << Args[0] << endl; - _exit(100); - }; - if (Comp == true) - close(Pipe[0]); - else - close(Pipe[1]); - return true; -} - /*}}}*/ // MultiCompress::OpenOld - Open an old file /*{{{*/ // --------------------------------------------------------------------- /* This opens one of the original output files, possibly decompressing it. */ @@ -344,7 +278,7 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) return false; // Decompress the file so we can read it - if (OpenCompress(Best->CompressProg,Proc,F.Fd(),Fd,false) == false) + if (ExecCompressor(Best->CompressProg,&Proc,F.Fd(),Fd,false) == false) return false; return true; @@ -374,8 +308,8 @@ bool MultiCompress::Child(int const &FD) // Start the compression children. for (Files *I = Outputs; I != 0; I = I->Next) { - if (OpenCompress(I->CompressProg,I->CompressProc,I->TmpFile.Fd(), - I->Fd,true) == false) + if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(), + I->Fd,true) == false) return false; } -- cgit v1.2.3 From 52b47296f61ec3ca1075bbfb44982f5caa541e7c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 14 Dec 2011 22:11:43 +0100 Subject: use FileFd instead of forking the compression childs by hand --- ftparchive/multicompress.cc | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) (limited to 'ftparchive/multicompress.cc') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 37a713efd..2a930ca6b 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -91,7 +91,7 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress, /* Open all the temp files now so we can report any errors. File is made unreable to prevent people from touching it during creating. */ for (Files *I = Outputs; I != 0; I = I->Next) - I->TmpFile.Open(I->Output + ".new",FileFd::WriteEmpty,0600); + I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600); if (_error->PendingError() == true) return; @@ -183,11 +183,6 @@ bool MultiCompress::Start() _exit(0); }; - /* Tidy up the temp files, we open them in the constructor so as to - get proper error reporting. Close them now. */ - for (Files *I = Outputs; I != 0; I = I->Next) - I->TmpFile.Close(); - close(Pipe[0]); Input = fdopen(Pipe[1],"w"); if (Input == 0) @@ -305,14 +300,6 @@ bool MultiCompress::CloseOld(int Fd,pid_t Proc) is new then the temp files are renamed, otherwise they are erased. */ bool MultiCompress::Child(int const &FD) { - // Start the compression children. - for (Files *I = Outputs; I != 0; I = I->Next) - { - if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(), - I->Fd,true) == false) - return false; - } - /* Okay, now we just feed data from FD to all the other FDs. Also stash a hash of the data to use later. */ SetNonBlock(FD,false); @@ -332,25 +319,14 @@ bool MultiCompress::Child(int const &FD) FileSize += Res; for (Files *I = Outputs; I != 0; I = I->Next) { - if (write(I->Fd,Buffer,Res) != Res) + if (I->TmpFile.Write(Buffer, Res) == false) { _error->Errno("write",_("IO to subprocess/file failed")); break; } } } - - // Close all the writers - for (Files *I = Outputs; I != 0; I = I->Next) - close(I->Fd); - - // Wait for the compressors to exit - for (Files *I = Outputs; I != 0; I = I->Next) - { - if (I->CompressProc != -1) - ExecWait(I->CompressProc, I->CompressProg.Binary.c_str(), false); - } - + if (_error->PendingError() == true) return false; -- cgit v1.2.3 From 12d1f5b3e295c589371bf7de27b7918310d08480 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 14 Dec 2011 22:35:03 +0100 Subject: remove the second usage instance of ExecCompressor in ftparchive by again using the FileFd directly --- ftparchive/multicompress.cc | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) (limited to 'ftparchive/multicompress.cc') diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 2a930ca6b..1fea589e2 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -260,7 +260,7 @@ bool MultiCompress::Finalize(unsigned long long &OutSize) // MultiCompress::OpenOld - Open an old file /*{{{*/ // --------------------------------------------------------------------- /* This opens one of the original output files, possibly decompressing it. */ -bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) +bool MultiCompress::OpenOld(FileFd &Fd) { Files *Best = Outputs; for (Files *I = Outputs; I != 0; I = I->Next) @@ -268,29 +268,9 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc) Best = I; // Open the file - FileFd F(Best->Output,FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - - // Decompress the file so we can read it - if (ExecCompressor(Best->CompressProg,&Proc,F.Fd(),Fd,false) == false) - return false; - - return true; + return Fd.Open(Best->Output, FileFd::ReadOnly, FileFd::Extension); } /*}}}*/ -// MultiCompress::CloseOld - Close the old file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool MultiCompress::CloseOld(int Fd,pid_t Proc) -{ - close(Fd); - if (Proc != -1) - if (ExecWait(Proc,_("decompressor"),false) == false) - return false; - return true; -} - /*}}}*/ // MultiCompress::Child - The writer child /*{{{*/ // --------------------------------------------------------------------- /* The child process forks a bunch of compression children and takes @@ -345,31 +325,27 @@ bool MultiCompress::Child(int const &FD) // Check the MD5 of the lowest cost entity. while (Missing == false) { - int CompFd = -1; - pid_t Proc = -1; - if (OpenOld(CompFd,Proc) == false) + FileFd CompFd; + if (OpenOld(CompFd) == false) { _error->Discard(); break; } - + // Compute the hash MD5Summation OldMD5; unsigned long long NewFileSize = 0; while (1) { - int Res = read(CompFd,Buffer,sizeof(Buffer)); + unsigned long long Res = 0; + if (CompFd.Read(Buffer,sizeof(Buffer), &Res) == false) + return _error->Errno("read",_("Failed to read while computing MD5")); if (Res == 0) break; - if (Res < 0) - return _error->Errno("read",_("Failed to read while computing MD5")); NewFileSize += Res; OldMD5.Add(Buffer,Res); } - - // Tidy the compressor - if (CloseOld(CompFd,Proc) == false) - return false; + CompFd.Close(); // Check the hash if (OldMD5.Result() == MD5.Result() && -- cgit v1.2.3