From 1abbc47c045770476f5f9a57c58989d13290d51b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 17 Dec 2011 17:31:47 +0100 Subject: keep track of where we are in a filedescriptor so we can use it as Tell() information if we are working on a pipe which can't seek --- apt-pkg/contrib/fileutl.cc | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index a98c2cb85..bb836e93b 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -75,7 +75,9 @@ class FileFdPrivate { bool pipe; APT::Configuration::Compressor compressor; unsigned int openmode; - FileFdPrivate() : gz(NULL), compressed_fd(-1), compressor_pid(-1), pipe(false) {}; + unsigned long long seekpos; + FileFdPrivate() : gz(NULL), compressed_fd(-1), compressor_pid(-1), pipe(false), + openmode(0), seekpos(0) {}; }; // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ @@ -1065,6 +1067,7 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) To = (char *)To + Res; Size -= Res; + d->seekpos += Res; if (Actual != 0) *Actual += Res; } @@ -1134,6 +1137,7 @@ bool FileFd::Write(const void *From,unsigned long long Size) From = (char *)From + Res; Size -= Res; + d->seekpos += Res; } while (Res > 0 && Size > 0); @@ -1151,6 +1155,13 @@ bool FileFd::Seek(unsigned long long To) { if (d->pipe == true) { + // Our poor man seeking in pipes is costly, so try to avoid it + unsigned long long seekpos = Tell(); + if (seekpos == To) + return true; + else if (seekpos < To) + return Skip(To - seekpos); + if ((d->openmode & ReadOnly) != ReadOnly) return _error->Error("Reopen is only implemented for read-only files!"); close(iFd); @@ -1173,6 +1184,8 @@ bool FileFd::Seek(unsigned long long To) if (To != 0) return Skip(To); + + d->seekpos = To; return true; } int res; @@ -1187,7 +1200,8 @@ bool FileFd::Seek(unsigned long long To) Flags |= Fail; return _error->Error("Unable to seek to %llu", To); } - + + d->seekpos = To; return true; } /*}}}*/ @@ -1208,7 +1222,8 @@ bool FileFd::Skip(unsigned long long Over) Flags |= Fail; return _error->Error("Unable to seek ahead %llu",Over); } - + d->seekpos = res; + return true; } /*}}}*/ @@ -1236,6 +1251,13 @@ bool FileFd::Truncate(unsigned long long To) /* */ unsigned long long FileFd::Tell() { + // In theory, we could just return seekpos here always instead of + // seeking around, but not all users of FileFd use always Seek() and co + // so d->seekpos isn't always true and we can just use it as a hint if + // we have nothing else, but not always as an authority… + if (d->pipe == true) + return d->seekpos; + off_t Res; #if APT_USE_ZLIB if (d->gz != NULL) @@ -1245,6 +1267,7 @@ unsigned long long FileFd::Tell() Res = lseek(iFd,0,SEEK_CUR); if (Res == (off_t)-1) _error->Errno("lseek","Failed to determine the current file position"); + d->seekpos = Res; return Res; } /*}}}*/ @@ -1281,15 +1304,14 @@ unsigned long long FileFd::Size() // so we 'read' the content and 'seek' back - see there if (d->pipe == true) { - // FIXME: If we have read first and then FileSize() the report is wrong - size = 0; + unsigned long long const oldSeek = Tell(); char ignore[1000]; unsigned long long read = 0; do { Read(ignore, sizeof(ignore), &read); - size += read; } while(read != 0); - Seek(0); + size = Tell(); + Seek(oldSeek); } #if APT_USE_ZLIB // only check gzsize if we are actually a gzip file, just checking for @@ -1301,7 +1323,6 @@ unsigned long long FileFd::Size() * this ourselves; the original (uncompressed) file size is the last 32 * bits of the file */ // FIXME: Size for gz-files is limited by 32bit… no largefile support - off_t orig_pos = lseek(iFd, 0, SEEK_CUR); if (lseek(iFd, -4, SEEK_END) < 0) return _error->Errno("lseek","Unable to seek to end of gzipped file"); size = 0L; @@ -1315,7 +1336,7 @@ unsigned long long FileFd::Size() size = tmp_size; #endif - if (lseek(iFd, orig_pos, SEEK_SET) < 0) + if (lseek(iFd, d->seekpos, SEEK_SET) < 0) return _error->Errno("lseek","Unable to seek in gzipped file"); return size; } -- cgit v1.2.3 From 109eb1511d0cdfa4af3196105cada30bcbb77bc8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 17 Dec 2011 23:53:31 +0100 Subject: try to avoid direct usage of .Fd() if possible and do read()s and co on the FileFd instead --- apt-pkg/acquire-item.cc | 4 ++-- apt-pkg/contrib/hashes.cc | 40 +++++++++++++++++++++++++++++++++----- apt-pkg/contrib/hashes.h | 5 +++++ apt-pkg/contrib/hashsum.cc | 22 +++++++++++++++++++++ apt-pkg/contrib/hashsum_template.h | 3 +++ apt-pkg/deb/debindexfile.cc | 3 --- 6 files changed, 67 insertions(+), 10 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 453fce109..f231c42b4 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -438,7 +438,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); + SHA1.AddFD(fd); string const local_sha1 = SHA1.Result(); if(local_sha1 == ServerSha1) @@ -669,7 +669,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ FileFd fd(FinalFile, FileFd::ReadOnly); SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); + SHA1.AddFD(fd); string local_sha1 = string(SHA1.Result()); if(Debug) std::clog << "QueueNextDiff: " diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 05001f042..e1a431823 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -61,25 +61,25 @@ bool HashString::VerifyFile(std::string filename) const /*{{{*/ if(Type == "MD5Sum") { MD5Summation MD5; - MD5.AddFD(Fd.Fd(), Fd.Size()); + MD5.AddFD(Fd); fileHash = (std::string)MD5.Result(); } else if (Type == "SHA1") { SHA1Summation SHA1; - SHA1.AddFD(Fd.Fd(), Fd.Size()); + SHA1.AddFD(Fd); fileHash = (std::string)SHA1.Result(); } else if (Type == "SHA256") { SHA256Summation SHA256; - SHA256.AddFD(Fd.Fd(), Fd.Size()); + SHA256.AddFD(Fd); fileHash = (std::string)SHA256.Result(); } else if (Type == "SHA512") { SHA512Summation SHA512; - SHA512.AddFD(Fd.Fd(), Fd.Size()); + SHA512.AddFD(Fd); fileHash = (std::string)SHA512.Result(); } Fd.Close(); @@ -134,6 +134,36 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5, SHA512.Add(Buf,Res); } return true; +} +bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, bool const addMD5, + bool const addSHA1, bool const addSHA256, bool const addSHA512) +{ + unsigned char Buf[64*64]; + bool const ToEOF = (Size == 0); + while (Size != 0 || ToEOF) + { + unsigned long long n = sizeof(Buf); + if (!ToEOF) n = std::min(Size, n); + unsigned long long a = 0; + if (Fd.Read(Buf, n, &a) == false) // error + return false; + if (ToEOF == false) + { + if (a != n) // short read + return false; + } + else if (a == 0) // EOF + break; + Size -= a; + if (addMD5 == true) + MD5.Add(Buf, a); + if (addSHA1 == true) + SHA1.Add(Buf, a); + if (addSHA256 == true) + SHA256.Add(Buf, a); + if (addSHA512 == true) + SHA512.Add(Buf, a); + } + return true; } /*}}}*/ - diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index b206eccb8..0c0b6c6a7 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -74,6 +75,10 @@ class Hashes { return AddFD(Fd, Size, true, true, true, true); }; bool AddFD(int const Fd, unsigned long long Size, bool const addMD5, bool const addSHA1, bool const addSHA256, bool const addSHA512); + inline bool AddFD(FileFd &Fd,unsigned long long Size = 0) + { return AddFD(Fd, Size, true, true, true, true); }; + bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5, + bool const addSHA1, bool const addSHA256, bool const addSHA512); inline bool Add(const unsigned char *Beg,const unsigned char *End) {return Add(Beg,End-Beg);}; }; diff --git a/apt-pkg/contrib/hashsum.cc b/apt-pkg/contrib/hashsum.cc index ff3b112bb..289e43aa4 100644 --- a/apt-pkg/contrib/hashsum.cc +++ b/apt-pkg/contrib/hashsum.cc @@ -24,5 +24,27 @@ bool SummationImplementation::AddFD(int const Fd, unsigned long long Size) { Add(Buf,Res); } return true; +} +bool SummationImplementation::AddFD(FileFd &Fd, unsigned long long Size) { + unsigned char Buf[64 * 64]; + bool ToEOF = (Size == 0); + while (Size != 0 || ToEOF) + { + unsigned long long n = sizeof(Buf); + if (!ToEOF) n = std::min(Size, n); + unsigned long long a = 0; + if (Fd.Read(Buf, n, &a) == false) // error + return false; + if (ToEOF == false) + { + if (a != n) // short read + return false; + } + else if (a == 0) // EOF + break; + Size -= a; + Add(Buf, a); + } + return true; } /*}}}*/ diff --git a/apt-pkg/contrib/hashsum_template.h b/apt-pkg/contrib/hashsum_template.h index 6301ac9d0..51e3b0862 100644 --- a/apt-pkg/contrib/hashsum_template.h +++ b/apt-pkg/contrib/hashsum_template.h @@ -10,6 +10,8 @@ #ifndef APTPKG_HASHSUM_TEMPLATE_H #define APTPKG_HASHSUM_TEMPLATE_H +#include + #include #include #include @@ -108,6 +110,7 @@ class SummationImplementation { return Add((const unsigned char *)Beg, End - Beg); }; bool AddFD(int Fd, unsigned long long Size = 0); + bool AddFD(FileFd &Fd, unsigned long long Size = 0); }; #endif diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 84791a70a..5dc2a2ac2 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -600,9 +600,6 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); - struct stat St; - if (fstat(Pkg.Fd(),&St) != 0) - return _error->Errno("fstat","Failed to stat"); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); CFile->Archive = Gen.WriteUniqString("now"); -- cgit v1.2.3 From 40468850491c4f5bc7060763a6f03cdc570d514e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Dec 2011 01:21:20 +0100 Subject: usage of Skipping in pipes can't work, so we ignore-read instead Also, read only one char in each step of ReadLine instead of back-"seeking" --- apt-pkg/contrib/fileutl.cc | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index bb836e93b..b350973af 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1100,15 +1100,19 @@ char* FileFd::ReadLine(char *To, unsigned long long const Size) #endif unsigned long long read = 0; - if (Read(To, Size, &read) == false) + while ((Size - 1) != read) + { + unsigned long long done = 0; + if (Read(To + read, 1, &done) == false) + return NULL; + if (done == 0) + break; + if (To[read++] == '\n') + break; + } + if (read == 0) return NULL; - char* c = To; - for (; *c != '\n' && *c != '\0' && read != 0; --read, ++c) - ; // find the end of the line - if (*c != '\0') - *c = '\0'; - if (read != 0) - Seek(Tell() - read); + To[read] = '\0'; return To; } /*}}}*/ @@ -1210,6 +1214,20 @@ bool FileFd::Seek(unsigned long long To) /* */ bool FileFd::Skip(unsigned long long Over) { + if (d->pipe == true) + { + d->seekpos += Over; + char buffer[1024]; + while (Over != 0) + { + unsigned long long toread = std::min((unsigned long long) sizeof(buffer), Over); + if (Read(buffer, toread) == false) + return _error->Error("Unable to seek ahead %llu",Over); + Over -= toread; + } + return true; + } + int res; #if APT_USE_ZLIB if (d->gz != NULL) -- cgit v1.2.3