From 92fcbfc16396d9a2fbde0edb0902d4ebe7ff0090 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 30 Jun 2009 21:37:30 +0200 Subject: add the various foldmarkers in apt-pkg & cmdline (no code change) --- apt-pkg/contrib/hashes.cc | 11 ++++------- apt-pkg/contrib/sha256.cc | 32 +++++++++++++------------------- 2 files changed, 17 insertions(+), 26 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index fcc2f887c..52b9bfbe6 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -34,7 +34,7 @@ HashString::HashString(string Type, string Hash) : Type(Type), Hash(Hash) { } -HashString::HashString(string StringedHash) +HashString::HashString(string StringedHash) /*{{{*/ { // legacy: md5sum without "MD5Sum:" prefix if (StringedHash.find(":") == string::npos && StringedHash.size() == 32) @@ -50,9 +50,8 @@ HashString::HashString(string StringedHash) if(_config->FindB("Debug::Hashes",false) == true) std::clog << "HashString(string): " << Type << " : " << Hash << std::endl; } - - -bool HashString::VerifyFile(string filename) const + /*}}}*/ +bool HashString::VerifyFile(string filename) const /*{{{*/ { FileFd fd; MD5Summation MD5; @@ -83,7 +82,7 @@ bool HashString::VerifyFile(string filename) const return (fileHash == Hash); } - + /*}}}*/ const char** HashString::SupportedHashes() { return _SupportedHashes; @@ -94,13 +93,11 @@ bool HashString::empty() const return (Type.empty() || Hash.empty()); } - string HashString::toStr() const { return Type+string(":")+Hash; } - // Hashes::AddFD - Add the contents of the FD /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/contrib/sha256.cc b/apt-pkg/contrib/sha256.cc index ecda3d8e8..e380c13ae 100644 --- a/apt-pkg/contrib/sha256.cc +++ b/apt-pkg/contrib/sha256.cc @@ -1,5 +1,5 @@ /* - * Cryptographic API. + * Cryptographic API. {{{ * * SHA-256, as specified in * http://csrc.nist.gov/cryptval/shs/sha256-384-512.pdf @@ -17,7 +17,7 @@ * Software Foundation; either version 2 of the License, or (at your option) * any later version. * - */ + */ /*}}}*/ #ifdef __GNUG__ #pragma implementation "apt-pkg/sha256.h" @@ -65,20 +65,20 @@ static inline u32 Maj(u32 x, u32 y, u32 z) #define H6 0x1f83d9ab #define H7 0x5be0cd19 -static inline void LOAD_OP(int I, u32 *W, const u8 *input) +static inline void LOAD_OP(int I, u32 *W, const u8 *input) /*{{{*/ { W[I] = ( ((u32) input[I * 4 + 0] << 24) | ((u32) input[I * 4 + 1] << 16) | ((u32) input[I * 4 + 2] << 8) | ((u32) input[I * 4 + 3])); } - + /*}}}*/ static inline void BLEND_OP(int I, u32 *W) { W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; } -static void sha256_transform(u32 *state, const u8 *input) +static void sha256_transform(u32 *state, const u8 *input) /*{{{*/ { u32 a, b, c, d, e, f, g, h, t1, t2; u32 W[64]; @@ -240,8 +240,8 @@ static void sha256_transform(u32 *state, const u8 *input) a = b = c = d = e = f = g = h = t1 = t2 = 0; memset(W, 0, 64 * sizeof(u32)); } - -SHA256Summation::SHA256Summation() + /*}}}*/ +SHA256Summation::SHA256Summation() /*{{{*/ { Sum.state[0] = H0; Sum.state[1] = H1; @@ -255,8 +255,8 @@ SHA256Summation::SHA256Summation() memset(Sum.buf, 0, sizeof(Sum.buf)); Done = false; } - -bool SHA256Summation::Add(const u8 *data, unsigned long len) + /*}}}*/ +bool SHA256Summation::Add(const u8 *data, unsigned long len) /*{{{*/ { struct sha256_ctx *sctx = ∑ unsigned int i, index, part_len; @@ -291,8 +291,8 @@ bool SHA256Summation::Add(const u8 *data, unsigned long len) return true; } - -SHA256SumValue SHA256Summation::Result() + /*}}}*/ +SHA256SumValue SHA256Summation::Result() /*{{{*/ { struct sha256_ctx *sctx = ∑ if (!Done) { @@ -340,7 +340,7 @@ SHA256SumValue SHA256Summation::Result() return res; } - + /*}}}*/ // SHA256SumValue::SHA256SumValue - Constructs the sum from a string /*{{{*/ // --------------------------------------------------------------------- /* The string form of a SHA256 is a 64 character hex number */ @@ -349,7 +349,6 @@ SHA256SumValue::SHA256SumValue(string Str) memset(Sum,0,sizeof(Sum)); Set(Str); } - /*}}}*/ // SHA256SumValue::SHA256SumValue - Default constructor /*{{{*/ // --------------------------------------------------------------------- @@ -358,7 +357,6 @@ SHA256SumValue::SHA256SumValue() { memset(Sum,0,sizeof(Sum)); } - /*}}}*/ // SHA256SumValue::Set - Set the sum from a string /*{{{*/ // --------------------------------------------------------------------- @@ -391,9 +389,7 @@ string SHA256SumValue::Value() const return string(Result); } - - - + /*}}}*/ // SHA256SumValue::operator == - Comparator /*{{{*/ // --------------------------------------------------------------------- /* Call memcmp on the buffer */ @@ -402,8 +398,6 @@ bool SHA256SumValue::operator == (const SHA256SumValue & rhs) const return memcmp(Sum,rhs.Sum,sizeof(Sum)) == 0; } /*}}}*/ - - // SHA256Summation::AddFD - Add content of file into the checksum /*{{{*/ // --------------------------------------------------------------------- /* */ -- cgit v1.2.3 From f1c6a8ca0511c623a16bb804ed2c5b6c26c83d78 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 8 Jul 2009 01:03:15 +0200 Subject: [contrib/mmap] implements a theoretical dynamic growing mmap based on Michael Vogts patch in #195018 this commit implements the use of mmap (as preferred) instead of a static char array. In theory this made it possible to grow the mmap as needed, but as it is currently impossible to move the mmap around in the memory the grow is likely to fail but it improve the memory usage a bit, so it is not totally useless for now - and maybe we can adjust the pointers in the future... --- apt-pkg/contrib/mmap.cc | 84 ++++++++++++++++++++++++++++++++++++------------- apt-pkg/contrib/mmap.h | 2 ++ 2 files changed, 65 insertions(+), 21 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 04a45811b..073bc9eb3 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -13,11 +13,6 @@ libc6 generates warnings -- which should be errors, g++ isn't properly strict. - The configure test notes that some OS's have broken private mmap's - so on those OS's we can't use mmap. This means we have to use - configure to test mmap and can't rely on the POSIX - _POSIX_MAPPED_FILES test. - ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ @@ -166,13 +161,23 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace) /*}}}*/ // DynamicMMap::DynamicMMap - Constructor for a non-file backed map /*{{{*/ // --------------------------------------------------------------------- -/* This is just a fancy malloc really.. */ +/* We try here to use mmap to reserve some space - this is much more + cooler than the fallback solution to simply allocate a char array + and could come in handy later than we are able to grow such an mmap */ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long WorkSpace) : MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace) { if (_error->PendingError() == true) return; - + +#ifdef _POSIX_MAPPED_FILES + // use anonymous mmap() to get the memory + Base = (unsigned char*) mmap(0, WorkSpace, PROT_READ|PROT_WRITE, + MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + if(Base != MAP_FAILED) + return; +#endif + // fallback to a static allocated space Base = new unsigned char[WorkSpace]; memset(Base,0,WorkSpace); iSize = 0; @@ -185,7 +190,11 @@ DynamicMMap::~DynamicMMap() { if (Fd == 0) { +#ifdef _POSIX_MAPPED_FILES + munmap(Base, WorkSpace); +#else delete [] (unsigned char *)Base; +#endif return; } @@ -207,14 +216,16 @@ unsigned long DynamicMMap::RawAllocate(unsigned long Size,unsigned long Aln) iSize = Result + Size; - // Just in case error check - if (Result + Size > WorkSpace) + // try to grow the buffer + while(Result + Size > WorkSpace) { - _error->Error(_("Dynamic MMap ran out of room. Please increase the size " - "of APT::Cache-Limit. Current value: %lu. (man 5 apt.conf)"), WorkSpace); - return 0; + if(!Grow()) + { + _error->Error(_("Dynamic MMap ran out of room. Please increase the size " + "of APT::Cache-Limit. Current value: %lu. (man 5 apt.conf)"), WorkSpace); + return 0; + } } - return Result; } /*}}}*/ @@ -234,7 +245,6 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) if (I->ItemSize == ItemSize) break; } - // No pool is allocated, use an unallocated one if (I == Pools + PoolCount) { @@ -270,14 +280,17 @@ unsigned long DynamicMMap::WriteString(const char *String, unsigned long Len) { unsigned long Result = iSize; - // Just in case error check - if (Result + Len > WorkSpace) + // try to grow the buffer + while(Result + Len > WorkSpace) { - _error->Error(_("Dynamic MMap ran out of room. Please increase the size " - "of APT::Cache-Limit. Current value: %lu. (man 5 apt.conf)"), WorkSpace); - return 0; - } - + if(!Grow()) + { + _error->Error(_("Dynamic MMap ran out of room. Please increase the size " + "of APT::Cache-Limit. Current value: %lu. (man 5 apt.conf)"), WorkSpace); + return 0; + } + } + if (Len == (unsigned long)-1) Len = strlen(String); iSize += Len + 1; @@ -286,3 +299,32 @@ unsigned long DynamicMMap::WriteString(const char *String, return Result; } /*}}}*/ +// DynamicMMap::Grow - Grow the mmap /*{{{*/ +// --------------------------------------------------------------------- +/* This method will try to grow the mmap we currently use. This doesn't + work most of the time because we can't move the mmap around in the + memory for now as this would require to adjust quite a lot of pointers + but why we should not at least try to grow it before we give up? */ +bool DynamicMMap::Grow() +{ +#ifdef _POSIX_MAPPED_FILES + unsigned long newSize = WorkSpace + 1024*1024; + + if(Fd != 0) + { + Fd->Seek(newSize - 1); + char C = 0; + Fd->Write(&C,sizeof(C)); + } + + Base = mremap(Base, WorkSpace, newSize, 0); + if(Base == MAP_FAILED) + return false; + + WorkSpace = newSize; + return true; +#else + return false; +#endif +} + /*}}}*/ diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index 19cf7582d..bde62217d 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -84,6 +84,8 @@ class DynamicMMap : public MMap unsigned long WorkSpace; Pool *Pools; unsigned int PoolCount; + + bool Grow(); public: -- cgit v1.2.3 From 13eb93fcb6a03afd8fc05fa2b4c60046473e8507 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 8 Jul 2009 01:11:16 +0200 Subject: add a segfault handler to MMap to show the Cache-Limit message, which can be deactivated with MMap::SegfaultHandler=false (Closes: 535218) --- apt-pkg/contrib/mmap.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 073bc9eb3..5f56178f4 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -19,6 +19,7 @@ #define _BSD_SOURCE #include #include +#include #include @@ -26,6 +27,8 @@ #include #include #include +#include +#include #include /*}}}*/ @@ -136,6 +139,20 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) } /*}}}*/ +// DynamicMMapSegfaultHandler /*{{{*/ +// --------------------------------------------------------------------- +/* In theory, the mmap should never segfault because we check the available + size of our mmap before we use it, but there are a few reports out there + which state that the mmap segfaults without further notice. So this handler + will take care of all these segfaults which should never happen... */ +void DynamicMMapSegfaultHandler(int) +{ + _error->Error(_("Dynamic MMap segfaults, most likely because it ran out of room. " + "Please increase the size of APT::Cache-Limit. (man 5 apt.conf)")); + _error->DumpErrors(); + exit(EXIT_FAILURE); +} + /*}}}*/ // DynamicMMap::DynamicMMap - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -170,6 +187,13 @@ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long WorkSpace) : if (_error->PendingError() == true) return; + if (_config->FindB("MMap::SegfaultHandler",true) == true) + { + struct sigaction sa; + sa.sa_handler = DynamicMMapSegfaultHandler; + sigaction(SIGSEGV, &sa, NULL); + } + #ifdef _POSIX_MAPPED_FILES // use anonymous mmap() to get the memory Base = (unsigned char*) mmap(0, WorkSpace, PROT_READ|PROT_WRITE, -- cgit v1.2.3 From 9d800312c90b7d3b52f2f4c3cea3ebfc67bf000b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jul 2009 23:40:35 +0200 Subject: [contrib/error.cc] place a colon between errno and error text in output of GlobalError::Errno and WarningE (as it described in the comments of these methods) --- apt-pkg/contrib/error.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/error.cc b/apt-pkg/contrib/error.cc index db8c53c36..927b7e05c 100644 --- a/apt-pkg/contrib/error.cc +++ b/apt-pkg/contrib/error.cc @@ -84,17 +84,17 @@ bool GlobalError::Errno(const char *Function,const char *Description,...) char S[400]; vsnprintf(S,sizeof(S),Description,args); snprintf(S + strlen(S),sizeof(S) - strlen(S), - " - %s (%i %s)",Function,errno,strerror(errno)); + " - %s (%i: %s)",Function,errno,strerror(errno)); // Put it on the list Item *Itm = new Item; Itm->Text = S; Itm->Error = true; Insert(Itm); - + PendingFlag = true; - return false; + return false; } /*}}}*/ // GlobalError::WarningE - Get part of the warn string from errno /*{{{*/ @@ -112,15 +112,16 @@ bool GlobalError::WarningE(const char *Function,const char *Description,...) // sprintf the description char S[400]; vsnprintf(S,sizeof(S),Description,args); - snprintf(S + strlen(S),sizeof(S) - strlen(S)," - %s (%i %s)",Function,errno,strerror(errno)); + snprintf(S + strlen(S),sizeof(S) - strlen(S), + " - %s (%i: %s)",Function,errno,strerror(errno)); // Put it on the list Item *Itm = new Item; Itm->Text = S; Itm->Error = false; Insert(Itm); - - return false; + + return false; } /*}}}*/ // GlobalError::Error - Add an error to the list /*{{{*/ -- cgit v1.2.3 From c5f44afc2446d738e30ea4c6021d4b60915546b1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 21 Jul 2009 16:54:28 +0200 Subject: eliminate (hopefully all) segfaults in pkgcachegen.cc and mmap.cc which can arise if cache doesn't fit into the mmap (Closes: #535218) This removes also the previously introduced SegfaultSignalHandler: The handler works, but is ugly by design... --- apt-pkg/contrib/mmap.cc | 68 +++++++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 45 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 5f56178f4..ba4482131 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -19,7 +19,6 @@ #define _BSD_SOURCE #include #include -#include #include @@ -28,7 +27,6 @@ #include #include #include -#include #include /*}}}*/ @@ -139,24 +137,11 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) } /*}}}*/ -// DynamicMMapSegfaultHandler /*{{{*/ -// --------------------------------------------------------------------- -/* In theory, the mmap should never segfault because we check the available - size of our mmap before we use it, but there are a few reports out there - which state that the mmap segfaults without further notice. So this handler - will take care of all these segfaults which should never happen... */ -void DynamicMMapSegfaultHandler(int) -{ - _error->Error(_("Dynamic MMap segfaults, most likely because it ran out of room. " - "Please increase the size of APT::Cache-Limit. (man 5 apt.conf)")); - _error->DumpErrors(); - exit(EXIT_FAILURE); -} /*}}}*/ // DynamicMMap::DynamicMMap - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace) : +DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace) : MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(WorkSpace) { if (_error->PendingError() == true) @@ -187,13 +172,6 @@ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long WorkSpace) : if (_error->PendingError() == true) return; - if (_config->FindB("MMap::SegfaultHandler",true) == true) - { - struct sigaction sa; - sa.sa_handler = DynamicMMapSegfaultHandler; - sigaction(SIGSEGV, &sa, NULL); - } - #ifdef _POSIX_MAPPED_FILES // use anonymous mmap() to get the memory Base = (unsigned char*) mmap(0, WorkSpace, PROT_READ|PROT_WRITE, @@ -237,9 +215,9 @@ unsigned long DynamicMMap::RawAllocate(unsigned long Size,unsigned long Aln) unsigned long Result = iSize; if (Aln != 0) Result += Aln - (iSize%Aln); - + iSize = Result + Size; - + // try to grow the buffer while(Result + Size > WorkSpace) { @@ -258,7 +236,7 @@ unsigned long DynamicMMap::RawAllocate(unsigned long Size,unsigned long Aln) /* This allocates an Item of size ItemSize so that it is aligned to its size in the file. */ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) -{ +{ // Look for a matching pool entry Pool *I; Pool *Empty = 0; @@ -283,17 +261,24 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) I->ItemSize = ItemSize; I->Count = 0; } - + + unsigned long Result = 0; // Out of space, allocate some more if (I->Count == 0) { - I->Count = 20*1024/ItemSize; - I->Start = RawAllocate(I->Count*ItemSize,ItemSize); - } + const unsigned long size = 20*1024; + I->Count = size/ItemSize; + Result = RawAllocate(size,ItemSize); + // Does the allocation failed ? + if (Result == 0 && _error->PendingError()) + return 0; + I->Start = Result; + } + else + Result = I->Start; I->Count--; - unsigned long Result = I->Start; - I->Start += ItemSize; + I->Start += ItemSize; return Result/ItemSize; } /*}}}*/ @@ -303,21 +288,14 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) unsigned long DynamicMMap::WriteString(const char *String, unsigned long Len) { - unsigned long Result = iSize; - // try to grow the buffer - while(Result + Len > WorkSpace) - { - if(!Grow()) - { - _error->Error(_("Dynamic MMap ran out of room. Please increase the size " - "of APT::Cache-Limit. Current value: %lu. (man 5 apt.conf)"), WorkSpace); - return 0; - } - } - if (Len == (unsigned long)-1) Len = strlen(String); - iSize += Len + 1; + + unsigned long Result = RawAllocate(Len+1,0); + + if (Result == 0 && _error->PendingError()) + return 0; + memcpy((char *)Base + Result,String,Len); ((char *)Base)[Result + Len] = 0; return Result; -- cgit v1.2.3