From d6c4a9764d052c9755ab934c97c7a84c48ebd618 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 9 Nov 2009 10:17:19 +0100 Subject: extent the mmap to be able to handle currently not implemented (but planed) growable mmaps --- apt-pkg/contrib/mmap.cc | 127 ++++++++++++++++++++++++++++++++---------------- apt-pkg/contrib/mmap.h | 10 ++-- 2 files changed, 92 insertions(+), 45 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 4d5fcf71e..f440f9489 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -140,8 +140,10 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) // DynamicMMap::DynamicMMap - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace) : - MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(WorkSpace) +DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Workspace, + unsigned long const &Grow, unsigned long const &Limit) : + MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(Workspace), + GrowFactor(Grow), Limit(Limit) { if (_error->PendingError() == true) return; @@ -165,32 +167,48 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace) /* We try here to use mmap to reserve some space - this is much more cooler than the fallback solution to simply allocate a char array and could come in handy later than we are able to grow such an mmap */ -DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long WorkSpace) : - MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace) +DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace, + unsigned long const &Grow, unsigned long const &Limit) : + MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace), + GrowFactor(Grow), Limit(Limit) { - if (_error->PendingError() == true) - return; + if (_error->PendingError() == true) + return; + + // disable Moveable if we don't grow + if (Grow == 0) + Flags &= ~Moveable; + +#ifndef __linux__ + // kfreebsd doesn't have mremap, so we use the fallback + if ((Flags & Moveable) == Moveable) + Flags |= Fallback; +#endif #ifdef _POSIX_MAPPED_FILES - // Set the permissions. - int Prot = PROT_READ; - int Map = MAP_PRIVATE | MAP_ANONYMOUS; - if ((Flags & ReadOnly) != ReadOnly) - Prot |= PROT_WRITE; - if ((Flags & Public) == Public) - Map = MAP_SHARED | MAP_ANONYMOUS; + if ((Flags & Fallback) != Fallback) { + // Set the permissions. + int Prot = PROT_READ; + int Map = MAP_PRIVATE | MAP_ANONYMOUS; + if ((Flags & ReadOnly) != ReadOnly) + Prot |= PROT_WRITE; + if ((Flags & Public) == Public) + Map = MAP_SHARED | MAP_ANONYMOUS; - // use anonymous mmap() to get the memory - Base = (unsigned char*) mmap(0, WorkSpace, Prot, Map, -1, 0); + // use anonymous mmap() to get the memory + Base = (unsigned char*) mmap(0, WorkSpace, Prot, Map, -1, 0); - if(Base == MAP_FAILED) - _error->Errno("DynamicMMap",_("Couldn't make mmap of %lu bytes"),WorkSpace); -#else - // fallback to a static allocated space - Base = new unsigned char[WorkSpace]; - memset(Base,0,WorkSpace); + if(Base == MAP_FAILED) + _error->Errno("DynamicMMap",_("Couldn't make mmap of %lu bytes"),WorkSpace); + + iSize = 0; + return; + } #endif - iSize = 0; + // fallback to a static allocated space + Base = new unsigned char[WorkSpace]; + memset(Base,0,WorkSpace); + iSize = 0; } /*}}}*/ // DynamicMMap::~DynamicMMap - Destructor /*{{{*/ @@ -311,30 +329,55 @@ unsigned long DynamicMMap::WriteString(const char *String, /*}}}*/ // DynamicMMap::Grow - Grow the mmap /*{{{*/ // --------------------------------------------------------------------- -/* This method will try to grow the mmap we currently use. This doesn't - work most of the time because we can't move the mmap around in the - memory for now as this would require to adjust quite a lot of pointers - but why we should not at least try to grow it before we give up? */ -bool DynamicMMap::Grow() -{ -#if defined(_POSIX_MAPPED_FILES) && defined(__linux__) - unsigned long newSize = WorkSpace + 1024*1024; +/* This method is a wrapper around different methods to (try to) grow + a mmap (or our char[]-fallback). Encounterable environments: + 1. Moveable + !Fallback + linux -> mremap with MREMAP_MAYMOVE + 2. Moveable + !Fallback + !linux -> not possible (forbidden by constructor) + 3. Moveable + Fallback -> realloc + 4. !Moveable + !Fallback + linux -> mremap alone - which will fail in 99,9% + 5. !Moveable + !Fallback + !linux -> not possible (forbidden by constructor) + 6. !Moveable + Fallback -> not possible + [ While Moveable and Fallback stands for the equally named flags and + "linux" indicates a linux kernel instead of a freebsd kernel. ] + So what you can see here is, that a MMAP which want to be growable need + to be moveable to have a real chance but that this method will at least try + the nearly impossible 4 to grow it before it finally give up: Never say never. */ +bool DynamicMMap::Grow() { + if (Limit != 0 && WorkSpace >= Limit) + return _error->Error(_("The size of a MMap has already reached the defined limit of %lu bytes," + "abort the try to grow the MMap."), Limit); - if(Fd != 0) - { - Fd->Seek(newSize - 1); - char C = 0; - Fd->Write(&C,sizeof(C)); - } + unsigned long const newSize = WorkSpace + 1024*1024; - Base = mremap(Base, WorkSpace, newSize, 0); - if(Base == MAP_FAILED) - return false; + if(Fd != 0) { + Fd->Seek(newSize - 1); + char C = 0; + Fd->Write(&C,sizeof(C)); + } + if ((Flags & Fallback) != Fallback) { +#if defined(_POSIX_MAPPED_FILES) && defined(__linux__) + #ifdef MREMAP_MAYMOVE + if ((Flags & Moveable) == Moveable) + Base = mremap(Base, WorkSpace, newSize, MREMAP_MAYMOVE); + else + #endif + Base = mremap(Base, WorkSpace, newSize, 0); - WorkSpace = newSize; - return true; + if(Base == MAP_FAILED) + return false; #else - return false; + return false; #endif + } else { + if ((Flags & Moveable) != Moveable) + return false; + + Base = realloc(Base, newSize); + if (Base == NULL) + return false; + } + + WorkSpace = newSize; + return true; } /*}}}*/ diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index bde62217d..cd2b15ba2 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -50,7 +50,7 @@ class MMap public: enum OpenFlags {NoImmMap = (1<<0),Public = (1<<1),ReadOnly = (1<<2), - UnMapped = (1<<3)}; + UnMapped = (1<<3), Moveable = (1<<4), Fallback = (1 << 5)}; // Simple accessors inline operator void *() {return Base;}; @@ -82,6 +82,8 @@ class DynamicMMap : public MMap FileFd *Fd; unsigned long WorkSpace; + unsigned long const GrowFactor; + unsigned long const Limit; Pool *Pools; unsigned int PoolCount; @@ -96,8 +98,10 @@ class DynamicMMap : public MMap inline unsigned long WriteString(const string &S) {return WriteString(S.c_str(),S.length());}; void UsePools(Pool &P,unsigned int Count) {Pools = &P; PoolCount = Count;}; - DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace = 2*1024*1024); - DynamicMMap(unsigned long Flags,unsigned long WorkSpace = 2*1024*1024); + DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &WorkSpace = 2*1024*1024, + unsigned long const &Grow = 1024*1024, unsigned long const &Limit = 0); + DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace = 2*1024*1024, + unsigned long const &Grow = 1024*1024, unsigned long const &Limit = 0); virtual ~DynamicMMap(); }; -- cgit v1.2.3 From ce857f32cf3c73ee67147ea0eafadb5a1c5da952 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 25 Nov 2009 23:24:43 +0100 Subject: another round of method hardening with const& in Configuration --- apt-pkg/contrib/configuration.cc | 24 ++++++++++++------------ apt-pkg/contrib/configuration.h | 37 +++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 30 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index 4e8586e83..ff49ce857 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -85,7 +85,7 @@ Configuration::~Configuration() /* This will lookup a single item by name below another item. It is a helper function for the main lookup function */ Configuration::Item *Configuration::Lookup(Item *Head,const char *S, - unsigned long Len,bool Create) + unsigned long const &Len,bool const &Create) { int Res = 1; Item *I = Head->Child; @@ -118,7 +118,7 @@ Configuration::Item *Configuration::Lookup(Item *Head,const char *S, // --------------------------------------------------------------------- /* This performs a fully scoped lookup of a given name, possibly creating new items */ -Configuration::Item *Configuration::Lookup(const char *Name,bool Create) +Configuration::Item *Configuration::Lookup(const char *Name,bool const &Create) { if (Name == 0) return Root->Child; @@ -245,7 +245,7 @@ vector Configuration::FindVector(const char *Name) const // Configuration::FindI - Find an integer value /*{{{*/ // --------------------------------------------------------------------- /* */ -int Configuration::FindI(const char *Name,int Default) const +int Configuration::FindI(const char *Name,int const &Default) const { const Item *Itm = Lookup(Name); if (Itm == 0 || Itm->Value.empty() == true) @@ -262,7 +262,7 @@ int Configuration::FindI(const char *Name,int Default) const // Configuration::FindB - Find a boolean type /*{{{*/ // --------------------------------------------------------------------- /* */ -bool Configuration::FindB(const char *Name,bool Default) const +bool Configuration::FindB(const char *Name,bool const &Default) const { const Item *Itm = Lookup(Name); if (Itm == 0 || Itm->Value.empty() == true) @@ -338,7 +338,7 @@ void Configuration::Set(const char *Name,const string &Value) // Configuration::Set - Set an integer value /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Set(const char *Name,int Value) +void Configuration::Set(const char *Name,int const &Value) { Item *Itm = Lookup(Name,true); if (Itm == 0) @@ -351,7 +351,7 @@ void Configuration::Set(const char *Name,int Value) // Configuration::Clear - Clear an single value from a list /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Clear(const string Name, int Value) +void Configuration::Clear(string const &Name, int const &Value) { char S[300]; snprintf(S,sizeof(S),"%i",Value); @@ -361,7 +361,7 @@ void Configuration::Clear(const string Name, int Value) // Configuration::Clear - Clear an single value from a list /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Clear(const string Name, string Value) +void Configuration::Clear(string const &Name, string const &Value) { Item *Top = Lookup(Name.c_str(),false); if (Top == 0 || Top->Child == 0) @@ -392,7 +392,7 @@ void Configuration::Clear(const string Name, string Value) // Configuration::Clear - Clear an entire tree /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Clear(string Name) +void Configuration::Clear(string const &Name) { Item *Top = Lookup(Name.c_str(),false); if (Top == 0) @@ -507,8 +507,8 @@ string Configuration::Item::FullTag(const Item *Stop) const sections like 'zone "foo.org" { .. };' This causes each section to be added in with a tag like "zone::foo.org" instead of being split tag/value. AsSectional enables Sectional parsing.*/ -bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, - unsigned Depth) +bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectional, + unsigned const &Depth) { // Open the stream for reading ifstream F(FName.c_str(),ios::in); @@ -835,8 +835,8 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, // ReadConfigDir - Read a directory of config files /*{{{*/ // --------------------------------------------------------------------- /* */ -bool ReadConfigDir(Configuration &Conf,const string &Dir,bool AsSectional, - unsigned Depth) +bool ReadConfigDir(Configuration &Conf,const string &Dir,bool const &AsSectional, + unsigned const &Depth) { DIR *D = opendir(Dir.c_str()); if (D == 0) diff --git a/apt-pkg/contrib/configuration.h b/apt-pkg/contrib/configuration.h index e2da83f5b..2494c1d7c 100644 --- a/apt-pkg/contrib/configuration.h +++ b/apt-pkg/contrib/configuration.h @@ -58,8 +58,8 @@ class Configuration Item *Root; bool ToFree; - Item *Lookup(Item *Head,const char *S,unsigned long Len,bool Create); - Item *Lookup(const char *Name,bool Create); + Item *Lookup(Item *Head,const char *S,unsigned long const &Len,bool const &Create); + Item *Lookup(const char *Name,const bool &Create); inline const Item *Lookup(const char *Name) const { return ((Configuration *)this)->Lookup(Name,false); @@ -68,32 +68,33 @@ class Configuration public: string Find(const char *Name,const char *Default = 0) const; - string Find(const string Name,const char *Default = 0) const {return Find(Name.c_str(),Default);}; + string Find(string const &Name,const char *Default = 0) const {return Find(Name.c_str(),Default);}; + string Find(string const &Name, string const &Default) const {return Find(Name.c_str(),Default.c_str());}; string FindFile(const char *Name,const char *Default = 0) const; string FindDir(const char *Name,const char *Default = 0) const; - std::vector FindVector(const string &Name) const; + std::vector FindVector(string const &Name) const; std::vector FindVector(const char *Name) const; - int FindI(const char *Name,int Default = 0) const; - int FindI(const string Name,int Default = 0) const {return FindI(Name.c_str(),Default);}; - bool FindB(const char *Name,bool Default = false) const; - bool FindB(const string Name,bool Default = false) const {return FindB(Name.c_str(),Default);}; + int FindI(const char *Name,int const &Default = 0) const; + int FindI(string const &Name,int const &Default = 0) const {return FindI(Name.c_str(),Default);}; + bool FindB(const char *Name,bool const &Default = false) const; + bool FindB(string const &Name,bool const &Default = false) const {return FindB(Name.c_str(),Default);}; string FindAny(const char *Name,const char *Default = 0) const; - inline void Set(const string Name,string Value) {Set(Name.c_str(),Value);}; + inline void Set(const string &Name,const string &Value) {Set(Name.c_str(),Value);}; void CndSet(const char *Name,const string &Value); void Set(const char *Name,const string &Value); - void Set(const char *Name,int Value); + void Set(const char *Name,const int &Value); - inline bool Exists(const string Name) const {return Exists(Name.c_str());}; + inline bool Exists(const string &Name) const {return Exists(Name.c_str());}; bool Exists(const char *Name) const; bool ExistsAny(const char *Name) const; // clear a whole tree - void Clear(const string Name); + void Clear(const string &Name); // remove a certain value from a list (e.g. the list of "APT::Keep-Fds") - void Clear(const string List, string Value); - void Clear(const string List, int Value); + void Clear(string const &List, string const &Value); + void Clear(string const &List, int const &Value); inline const Item *Tree(const char *Name) const {return Lookup(Name);}; @@ -108,11 +109,11 @@ class Configuration extern Configuration *_config; bool ReadConfigFile(Configuration &Conf,const string &FName, - bool AsSectional = false, - unsigned Depth = 0); + bool const &AsSectional = false, + unsigned const &Depth = 0); bool ReadConfigDir(Configuration &Conf,const string &Dir, - bool AsSectional = false, - unsigned Depth = 0); + bool const &AsSectional = false, + unsigned const &Depth = 0); #endif -- cgit v1.2.3 From 52643bec17df4e36a9bd27183886e2c0c7a8ebd8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 3 Jan 2010 19:37:34 +0100 Subject: Add a GetListOfFilesInDir() helper method which replaces the old code copies used to load the various parts-files --- apt-pkg/contrib/configuration.cc | 39 ++------------------------------- apt-pkg/contrib/fileutl.cc | 47 ++++++++++++++++++++++++++++++++++++++++ apt-pkg/contrib/fileutl.h | 3 +++ 3 files changed, 52 insertions(+), 37 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index ff49ce857..89aa854a3 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -22,14 +22,8 @@ #include #include -#include #include #include - -#include -#include -#include -#include using namespace std; /*}}}*/ @@ -837,37 +831,8 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio /* */ bool ReadConfigDir(Configuration &Conf,const string &Dir,bool const &AsSectional, unsigned const &Depth) -{ - DIR *D = opendir(Dir.c_str()); - if (D == 0) - return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); - - vector List; - - for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) - { - if (Ent->d_name[0] == '.') - continue; - - // Skip bad file names ala run-parts - const char *C = Ent->d_name; - for (; *C != 0; C++) - if (isalpha(*C) == 0 && isdigit(*C) == 0 && *C != '_' && *C != '-') - break; - if (*C != 0) - continue; - - // Make sure it is a file and not something else - string File = flCombine(Dir,Ent->d_name); - struct stat St; - if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0) - continue; - - List.push_back(File); - } - closedir(D); - - sort(List.begin(),List.end()); +{ + vector const List = GetListOfFilesInDir(Dir, "", true); // Read the files for (vector::const_iterator I = List.begin(); I != List.end(); I++) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 4240d9f49..3adab8fe7 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -34,9 +34,11 @@ #include #include #include +#include #include #include #include +#include /*}}}*/ using namespace std; @@ -195,6 +197,51 @@ bool FileExists(string File) return true; } /*}}}*/ +// GetListOfFilesInDir - returns a vector of files in the given dir /*{{{*/ +// --------------------------------------------------------------------- +/* If an extension is given only files with this extension are included + in the returned vector, otherwise every "normal" file is included. */ +std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, + bool const &SortList) { + std::vector List; + DIR *D = opendir(Dir.c_str()); + if (D == 0) { + _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + return List; + } + + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) { + if (Ent->d_name[0] == '.') + continue; + + if (Ext.empty() == false && flExtension(Ent->d_name) != Ext) + continue; + + // Skip bad file names ala run-parts + const char *C = Ent->d_name; + for (; *C != 0; ++C) + if (isalpha(*C) == 0 && isdigit(*C) == 0 + && *C != '_' && *C != '-' && *C != '.') + break; + + if (*C != 0) + continue; + + // Make sure it is a file and not something else + string const File = flCombine(Dir,Ent->d_name); + struct stat St; + if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0) + continue; + + List.push_back(File); + } + closedir(D); + + if (SortList == true) + std::sort(List.begin(),List.end()); + return List; +} + /*}}}*/ // SafeGetCWD - This is a safer getcwd that returns a dynamic string /*{{{*/ // --------------------------------------------------------------------- /* We return / on failure. */ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 73b5ea3be..3cbf67fbb 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -23,6 +23,7 @@ #include +#include using std::string; @@ -81,6 +82,8 @@ bool RunScripts(const char *Cnf); bool CopyFile(FileFd &From,FileFd &To); int GetLock(string File,bool Errors = true); bool FileExists(string File); +std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, + bool const &SortList); string SafeGetCWD(); void SetCloseExec(int Fd,bool Close); void SetNonBlock(int Fd,bool Block); -- cgit v1.2.3 From e29a6bb14dcc004d174ad8502b76623139fbee06 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 16 Jan 2010 23:09:42 +0100 Subject: Fix the newly introduced method GetListOfFilesInDir to not accept every file if no extension is enforced (= restore old behaviour). (Closes: #565213) This commit includes also: * apt-pkg/policy.cc: - accept also partfiles with "pref" file extension as valid * apt-pkg/contrib/configuration.cc: - accept also partfiles with "conf" file extension as valid * doc/apt.conf.5.xml: - reorder description and split out syntax - add partfile name convention (Closes: #558348) * doc/apt_preferences.conf.5.xml: - describe partfile name convention also here And a lovely test application of course. --- apt-pkg/contrib/configuration.cc | 2 +- apt-pkg/contrib/fileutl.cc | 84 +++++++++++++++++++++++++++++++++++++--- apt-pkg/contrib/fileutl.h | 5 +++ 3 files changed, 85 insertions(+), 6 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index 80b089fac..7588b041c 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -832,7 +832,7 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio bool ReadConfigDir(Configuration &Conf,const string &Dir, bool const &AsSectional, unsigned const &Depth) { - vector const List = GetListOfFilesInDir(Dir, "", true); + vector const List = GetListOfFilesInDir(Dir, "conf", true, true); // Read the files for (vector::const_iterator I = List.begin(); I != List.end(); I++) diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index cce8a4512..da32983f1 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -202,8 +202,37 @@ bool FileExists(string File) /* If an extension is given only files with this extension are included in the returned vector, otherwise every "normal" file is included. */ std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, - bool const &SortList) + bool const &SortList) { + return GetListOfFilesInDir(Dir, Ext, SortList, false); +} +std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, + bool const &SortList, bool const &AllowNoExt) +{ + std::vector ext; + ext.reserve(2); + if (Ext.empty() == false) + ext.push_back(Ext); + if (AllowNoExt == true && ext.empty() == false) + ext.push_back(""); + return GetListOfFilesInDir(Dir, ext, SortList); +} +std::vector GetListOfFilesInDir(string const &Dir, std::vector const &Ext, + bool const &SortList) +{ + // Attention debuggers: need to be set with the environment config file! + bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false); + if (Debug == true) + { + std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl; + if (Ext.empty() == true) + std::clog << "\tNO extension" << std::endl; + else + for (std::vector::const_iterator e = Ext.begin(); + e != Ext.end(); ++e) + std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl; + } + std::vector List; DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -214,28 +243,73 @@ std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) { + // skip "hidden" files if (Ent->d_name[0] == '.') continue; - if (Ext.empty() == false && flExtension(Ent->d_name) != Ext) - continue; + // check for accepted extension: + // no extension given -> periods are bad as hell! + // extensions given -> "" extension allows no extension + if (Ext.empty() == false) + { + string d_ext = flExtension(Ent->d_name); + if (d_ext == Ent->d_name) // no extension + { + if (std::find(Ext.begin(), Ext.end(), "") == Ext.end()) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl; + continue; + } + } + else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end()) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl; + continue; + } + } - // Skip bad file names ala run-parts + // Skip bad filenames ala run-parts const char *C = Ent->d_name; for (; *C != 0; ++C) if (isalpha(*C) == 0 && isdigit(*C) == 0 - && *C != '_' && *C != '-' && *C != '.') + && *C != '_' && *C != '-') { + // no required extension -> dot is a bad character + if (*C == '.' && Ext.empty() == false) + continue; break; + } + // we don't reach the end of the name -> bad character included if (*C != 0) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad character »" + << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl; continue; + } + + // skip filenames which end with a period. These are never valid + if (*(C - 1) == '.') + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl; + continue; + } // Make sure it is a file and not something else string const File = flCombine(Dir,Ent->d_name); struct stat St; if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → stat says not a good file" << std::endl; continue; + } + if (Debug == true) + std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl; List.push_back(File); } closedir(D); diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 2807c29d9..85a94898c 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -82,8 +82,13 @@ bool RunScripts(const char *Cnf); bool CopyFile(FileFd &From,FileFd &To); int GetLock(string File,bool Errors = true); bool FileExists(string File); +// FIXME: next ABI-Break: merge the two method-headers std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, bool const &SortList); +std::vector GetListOfFilesInDir(string const &Dir, string const &Ext, + bool const &SortList, bool const &AllowNoExt); +std::vector GetListOfFilesInDir(string const &Dir, std::vector const &Ext, + bool const &SortList); string SafeGetCWD(); void SetCloseExec(int Fd,bool Close); void SetNonBlock(int Fd,bool Block); -- cgit v1.2.3 From d16aade9b781538ad5d6d79eda7b69ff075aad85 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 20 Jan 2010 14:18:12 +0100 Subject: * apt-pkg/contrib/strutl.cc: - fix malloc asseration fail with ja_JP.eucJP locale in apt-cache search. Thanks Kusanagi Kouichi! (Closes: #548884) --- apt-pkg/contrib/strutl.cc | 49 +++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index 4c05f2df8..2913fbf44 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -43,9 +43,10 @@ bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest) { iconv_t cd; const char *inbuf; - char *inptr, *outbuf, *outptr; - size_t insize, outsize; - + char *inptr, *outbuf; + size_t insize, bufsize; + dest->clear(); + cd = iconv_open(codeset, "UTF-8"); if (cd == (iconv_t)(-1)) { // Something went wrong @@ -55,33 +56,49 @@ bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest) else perror("iconv_open"); - // Clean the destination string - *dest = ""; - return false; } - insize = outsize = orig.size(); + insize = bufsize = orig.size(); inbuf = orig.data(); inptr = (char *)inbuf; - outbuf = new char[insize+1]; - outptr = outbuf; + outbuf = new char[bufsize]; + size_t lastError = -1; while (insize != 0) { + char *outptr = outbuf; + size_t outsize = bufsize; size_t const err = iconv(cd, &inptr, &insize, &outptr, &outsize); + dest->append(outbuf, outptr - outbuf); if (err == (size_t)(-1)) { - insize--; - outsize++; - inptr++; - *outptr = '?'; - outptr++; + switch (errno) + { + case EILSEQ: + insize--; + inptr++; + // replace a series of unknown multibytes with a single "?" + if (lastError != insize) { + lastError = insize - 1; + dest->append("?"); + } + break; + case EINVAL: + insize = 0; + break; + case E2BIG: + if (outptr == outbuf) + { + bufsize *= 2; + delete[] outbuf; + outbuf = new char[bufsize]; + } + break; + } } } - *outptr = '\0'; - *dest = outbuf; delete[] outbuf; iconv_close(cd); -- cgit v1.2.3 From 3d43e5390d83a95b08d09e8b811523f2d99a092c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 22 Jan 2010 08:27:20 +0100 Subject: add a few gcc helpers, including [un]likely() and __deprecated --- apt-pkg/contrib/system.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/system.h b/apt-pkg/contrib/system.h index 7ec3d7feb..b57093b93 100644 --- a/apt-pkg/contrib/system.h +++ b/apt-pkg/contrib/system.h @@ -55,4 +55,26 @@ #define CLRFLAG(v,f) ((v) &=~FLAG(f)) #define CHKFLAG(v,f) ((v) & FLAG(f) ? true : false) +// some nice optional GNUC features +#if __GNUC__ >= 3 + #define __must_check __attribute__ ((warn_unused_result)) + #define __deprecated __attribute__ ((deprecated)) + /* likely() and unlikely() can be used to mark boolean expressions + as (not) likely true which will help the compiler to optimise */ + #define likely(x) __builtin_expect (!!(x), 1) + #define unlikely(x) __builtin_expect (!!(x), 0) +#else + #define __must_check /* no warn_unused_result */ + #define __deprecated /* no deprecated */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif + +// cold functions are unlikely() to be called +#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4 + #define __cold __attribute__ ((__cold__)) +#else + #define __cold /* no cold marker */ +#endif + #endif -- cgit v1.2.3 From 2a75daa6be0647be3cc405f0ad9ed4f9f8638429 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 22 Jan 2010 08:38:09 +0100 Subject: mark the Error methods as __cold --- apt-pkg/contrib/error.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h index a3be6a575..86aa9eca3 100644 --- a/apt-pkg/contrib/error.h +++ b/apt-pkg/contrib/error.h @@ -53,6 +53,8 @@ #include +#include + using std::string; class GlobalError @@ -71,13 +73,13 @@ class GlobalError public: // Call to generate an error from a library call. - bool Errno(const char *Function,const char *Description,...) APT_MFORMAT2; - bool WarningE(const char *Function,const char *Description,...) APT_MFORMAT2; + bool Errno(const char *Function,const char *Description,...) APT_MFORMAT2 __cold; + bool WarningE(const char *Function,const char *Description,...) APT_MFORMAT2 __cold; /* A warning should be considered less severe than an error, and may be ignored by the client. */ - bool Error(const char *Description,...) APT_MFORMAT1; - bool Warning(const char *Description,...) APT_MFORMAT1; + bool Error(const char *Description,...) APT_MFORMAT1 __cold; + bool Warning(const char *Description,...) APT_MFORMAT1 __cold; // Simple accessors inline bool PendingError() {return PendingFlag;}; -- cgit v1.2.3 From 5c0d3668dd2b6852812502f33d64b1644c2b137a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 30 Jan 2010 22:19:57 +0100 Subject: * apt-pkg/contrib/macros.h: - move the header system.h with a new name to the public domain, to be able to use it in other headers (Closes: #567662) --- apt-pkg/contrib/error.h | 2 +- apt-pkg/contrib/hashes.cc | 4 +-- apt-pkg/contrib/macros.h | 79 ++++++++++++++++++++++++++++++++++++++++++++++ apt-pkg/contrib/md5.cc | 3 +- apt-pkg/contrib/sha1.cc | 2 +- apt-pkg/contrib/system.h | 80 ----------------------------------------------- 6 files changed, 84 insertions(+), 86 deletions(-) create mode 100644 apt-pkg/contrib/macros.h delete mode 100644 apt-pkg/contrib/system.h (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h index 86aa9eca3..31413b2dc 100644 --- a/apt-pkg/contrib/error.h +++ b/apt-pkg/contrib/error.h @@ -53,7 +53,7 @@ #include -#include +#include using std::string; diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index b43771ea7..985d89d90 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -14,9 +14,9 @@ #include #include #include - +#include + #include -#include #include #include /*}}}*/ diff --git a/apt-pkg/contrib/macros.h b/apt-pkg/contrib/macros.h new file mode 100644 index 000000000..e53eb32df --- /dev/null +++ b/apt-pkg/contrib/macros.h @@ -0,0 +1,79 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Macros Header - Various useful macro definitions + + This source is placed in the Public Domain, do with it what you will + It was originally written by Brian C. White. + + ##################################################################### */ + /*}}}*/ +// Private header +#ifndef MACROS_H +#define MACROS_H + +// MIN_VAL(SINT16) will return -0x8000 and MAX_VAL(SINT16) = 0x7FFF +#define MIN_VAL(t) (((t)(-1) > 0) ? (t)( 0) : (t)(((1L<<(sizeof(t)*8-1)) ))) +#define MAX_VAL(t) (((t)(-1) > 0) ? (t)(-1) : (t)(((1L<<(sizeof(t)*8-1))-1))) + +// Min/Max functions +#if !defined(MIN) +#if defined(__HIGHC__) +#define MIN(x,y) _min(x,y) +#define MAX(x,y) _max(x,y) +#endif + +// GNU C++ has a min/max operator +#if defined(__GNUG__) +#define MIN(A,B) ((A) ? (B)) +#endif + +/* Templates tend to mess up existing code that uses min/max because of the + strict matching requirements */ +#if !defined(MIN) +#define MIN(A,B) ((A) < (B)?(A):(B)) +#define MAX(A,B) ((A) > (B)?(A):(B)) +#endif +#endif + +/* Bound functions, bound will return the value b within the limits a-c + bounv will change b so that it is within the limits of a-c. */ +#define _bound(a,b,c) MIN(c,MAX(b,a)) +#define _boundv(a,b,c) b = _bound(a,b,c) +#define ABS(a) (((a) < (0)) ?-(a) : (a)) + +/* Usefull count macro, use on an array of things and it will return the + number of items in the array */ +#define _count(a) (sizeof(a)/sizeof(a[0])) + +// Flag Macros +#define FLAG(f) (1L << (f)) +#define SETFLAG(v,f) ((v) |= FLAG(f)) +#define CLRFLAG(v,f) ((v) &=~FLAG(f)) +#define CHKFLAG(v,f) ((v) & FLAG(f) ? true : false) + +// some nice optional GNUC features +#if __GNUC__ >= 3 + #define __must_check __attribute__ ((warn_unused_result)) + #define __deprecated __attribute__ ((deprecated)) + /* likely() and unlikely() can be used to mark boolean expressions + as (not) likely true which will help the compiler to optimise */ + #define likely(x) __builtin_expect (!!(x), 1) + #define unlikely(x) __builtin_expect (!!(x), 0) +#else + #define __must_check /* no warn_unused_result */ + #define __deprecated /* no deprecated */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif + +// cold functions are unlikely() to be called +#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4 + #define __cold __attribute__ ((__cold__)) +#else + #define __cold /* no cold marker */ +#endif + +#endif diff --git a/apt-pkg/contrib/md5.cc b/apt-pkg/contrib/md5.cc index 2bfd70f1b..c0fa8493d 100644 --- a/apt-pkg/contrib/md5.cc +++ b/apt-pkg/contrib/md5.cc @@ -37,14 +37,13 @@ // Include Files /*{{{*/ #include #include +#include #include #include #include // For htonl #include #include -#include - /*}}}*/ // byteSwap - Swap bytes in a buffer /*{{{*/ diff --git a/apt-pkg/contrib/sha1.cc b/apt-pkg/contrib/sha1.cc index b70f31dc6..eae52d52f 100644 --- a/apt-pkg/contrib/sha1.cc +++ b/apt-pkg/contrib/sha1.cc @@ -31,12 +31,12 @@ // Include Files /*{{{*/ #include #include +#include #include #include #include #include -#include /*}}}*/ // SHA1Transform - Alters an existing SHA-1 hash /*{{{*/ diff --git a/apt-pkg/contrib/system.h b/apt-pkg/contrib/system.h deleted file mode 100644 index b57093b93..000000000 --- a/apt-pkg/contrib/system.h +++ /dev/null @@ -1,80 +0,0 @@ -// -*- mode: cpp; mode: fold -*- -// Description /*{{{*/ -// $Id: system.h,v 1.3 1999/12/10 23:40:29 jgg Exp $ -/* ###################################################################### - - System Header - Usefull private definitions - - This source is placed in the Public Domain, do with it what you will - It was originally written by Brian C. White. - - ##################################################################### */ - /*}}}*/ -// Private header -#ifndef SYSTEM_H -#define SYSTEM_H - -// MIN_VAL(SINT16) will return -0x8000 and MAX_VAL(SINT16) = 0x7FFF -#define MIN_VAL(t) (((t)(-1) > 0) ? (t)( 0) : (t)(((1L<<(sizeof(t)*8-1)) ))) -#define MAX_VAL(t) (((t)(-1) > 0) ? (t)(-1) : (t)(((1L<<(sizeof(t)*8-1))-1))) - -// Min/Max functions -#if !defined(MIN) -#if defined(__HIGHC__) -#define MIN(x,y) _min(x,y) -#define MAX(x,y) _max(x,y) -#endif - -// GNU C++ has a min/max operator -#if defined(__GNUG__) -#define MIN(A,B) ((A) ? (B)) -#endif - -/* Templates tend to mess up existing code that uses min/max because of the - strict matching requirements */ -#if !defined(MIN) -#define MIN(A,B) ((A) < (B)?(A):(B)) -#define MAX(A,B) ((A) > (B)?(A):(B)) -#endif -#endif - -/* Bound functions, bound will return the value b within the limits a-c - bounv will change b so that it is within the limits of a-c. */ -#define _bound(a,b,c) MIN(c,MAX(b,a)) -#define _boundv(a,b,c) b = _bound(a,b,c) -#define ABS(a) (((a) < (0)) ?-(a) : (a)) - -/* Usefull count macro, use on an array of things and it will return the - number of items in the array */ -#define _count(a) (sizeof(a)/sizeof(a[0])) - -// Flag Macros -#define FLAG(f) (1L << (f)) -#define SETFLAG(v,f) ((v) |= FLAG(f)) -#define CLRFLAG(v,f) ((v) &=~FLAG(f)) -#define CHKFLAG(v,f) ((v) & FLAG(f) ? true : false) - -// some nice optional GNUC features -#if __GNUC__ >= 3 - #define __must_check __attribute__ ((warn_unused_result)) - #define __deprecated __attribute__ ((deprecated)) - /* likely() and unlikely() can be used to mark boolean expressions - as (not) likely true which will help the compiler to optimise */ - #define likely(x) __builtin_expect (!!(x), 1) - #define unlikely(x) __builtin_expect (!!(x), 0) -#else - #define __must_check /* no warn_unused_result */ - #define __deprecated /* no deprecated */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif - -// cold functions are unlikely() to be called -#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4 - #define __cold __attribute__ ((__cold__)) -#else - #define __cold /* no cold marker */ -#endif - -#endif -- cgit v1.2.3 From 8f3d83eeaec58e9347fe4c61dae18686782f94ca Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 30 Jan 2010 22:30:29 +0100 Subject: cleanup the error header a bit by moving the printf-macros out and remove the using std::string --- apt-pkg/contrib/error.h | 30 +++++++----------------------- apt-pkg/contrib/macros.h | 9 +++++++++ 2 files changed, 16 insertions(+), 23 deletions(-) (limited to 'apt-pkg/contrib') diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h index 31413b2dc..90747ff7e 100644 --- a/apt-pkg/contrib/error.h +++ b/apt-pkg/contrib/error.h @@ -40,28 +40,15 @@ #ifndef PKGLIB_ERROR_H #define PKGLIB_ERROR_H - - -#ifdef __GNUG__ -// Methods have a hidden this parameter that is visible to this attribute -#define APT_MFORMAT1 __attribute__ ((format (printf, 2, 3))) -#define APT_MFORMAT2 __attribute__ ((format (printf, 3, 4))) -#else -#define APT_MFORMAT1 -#define APT_MFORMAT2 -#endif - -#include - #include -using std::string; +#include class GlobalError { struct Item { - string Text; + std::string Text; bool Error; Item *Next; }; @@ -73,18 +60,18 @@ class GlobalError public: // Call to generate an error from a library call. - bool Errno(const char *Function,const char *Description,...) APT_MFORMAT2 __cold; - bool WarningE(const char *Function,const char *Description,...) APT_MFORMAT2 __cold; + bool Errno(const char *Function,const char *Description,...) __like_printf_2 __cold; + bool WarningE(const char *Function,const char *Description,...) __like_printf_2 __cold; /* A warning should be considered less severe than an error, and may be ignored by the client. */ - bool Error(const char *Description,...) APT_MFORMAT1 __cold; - bool Warning(const char *Description,...) APT_MFORMAT1 __cold; + bool Error(const char *Description,...) __like_printf_1 __cold; + bool Warning(const char *Description,...) __like_printf_1 __cold; // Simple accessors inline bool PendingError() {return PendingFlag;}; inline bool empty() {return List == 0;}; - bool PopMessage(string &Text); + bool PopMessage(std::string &Text); void Discard(); // Usefull routine to dump to cerr @@ -97,7 +84,4 @@ class GlobalError GlobalError *_GetErrorObj(); #define _error _GetErrorObj() -#undef APT_MFORMAT1 -#undef APT_MFORMAT2 - #endif diff --git a/apt-pkg/contrib/macros.h b/apt-pkg/contrib/macros.h index e53eb32df..9aeb77b81 100644 --- a/apt-pkg/contrib/macros.h +++ b/apt-pkg/contrib/macros.h @@ -76,4 +76,13 @@ #define __cold /* no cold marker */ #endif +#ifdef __GNUG__ +// Methods have a hidden this parameter that is visible to this attribute + #define __like_printf_1 __attribute__ ((format (printf, 2, 3))) + #define __like_printf_2 __attribute__ ((format (printf, 3, 4))) +#else + #define __like_printf_1 + #define __like_printf_2 +#endif + #endif -- cgit v1.2.3