diff options
55 files changed, 1836 insertions, 1510 deletions
diff --git a/CMake/Documentation.cmake b/CMake/Documentation.cmake index f3bbfdc6b..e5735873b 100644 --- a/CMake/Documentation.cmake +++ b/CMake/Documentation.cmake @@ -34,6 +34,8 @@ find_path(DOCBOOK_XSL manpages/docbook.xsl /usr/share/xml/docbook/xsl-stylesheets # Fedora /usr/share/sgml/docbook/xsl-stylesheets + #Brew + /usr/local/Cellar/docbook-xsl/1.79.1/docbook-xsl/ # Fink ${CMAKE_INSTALL_PREFIX}/share/xml/xsl/docbook-xsl # FreeBSD @@ -88,7 +90,7 @@ endfunction() # Process one document function(po4a_one stamp_out out full_document language deps) - path_join(full_path "${CMAKE_CURRENT_SOURCE_DIR}" "${full_document}") + path_join(full_path "${CMAKE_CURRENT_SOURCE_DIR}" "${full_document}") po4a_components(document _ section ext "${full_document}") # Calculate target file name @@ -266,7 +268,7 @@ function(add_docbook target) foreach(document ${DOC_DOCUMENTS}) foreach(lang ${DOC_LINGUAS}) - po4a_one(po4a_stamp po4a_out ${document} "${lang}" "${DOC_DEPENDS}") + po4a_one(po4a_stamp po4a_out ${document} "${lang}" "${DOC_DEPENDS}") xsltproc_one(STAMP_OUT xslt_stamp STAMP ${po4a_stamp} FULL_DOCUMENT ${po4a_out} diff --git a/CMake/Misc.cmake b/CMake/Misc.cmake index 6ad0b9479..72bd61f14 100644 --- a/CMake/Misc.cmake +++ b/CMake/Misc.cmake @@ -66,16 +66,16 @@ endfunction() # Generates a simple version script versioning everything with current SOVERSION function(add_version_script target) - get_target_property(soversion ${target} SOVERSION) - set(script "${CMAKE_CURRENT_BINARY_DIR}/${target}.versionscript") - string(REPLACE "-" "" name "${target}_${soversion}") - string(TOUPPER "${name}" name) - add_custom_command(OUTPUT "${script}" - COMMAND echo "${name} {global: *; };" > "${script}" - VERBATIM ) - add_custom_target(${target}-versionscript DEPENDS "${script}") - target_link_libraries(${target} PRIVATE -Wl,-version-script="${script}") - add_dependencies(${target} ${target}-versionscript) + #get_target_property(soversion ${target} SOVERSION) + #set(script "${CMAKE_CURRENT_BINARY_DIR}/${target}.versionscript") + #string(REPLACE "-" "" name "${target}_${soversion}") + #string(TOUPPER "${name}" name) + #add_custom_command(OUTPUT "${script}" + # COMMAND echo "${name} {global: *; };" > "${script}" + # VERBATIM ) + #add_custom_target(${target}-versionscript DEPENDS "${script}") + #target_link_libraries(${target} PRIVATE -Wl,-version-script="${script}") + #add_dependencies(${target} ${target}-versionscript) endfunction() function(path_join out path1 path2) diff --git a/CMakeLists.txt b/CMakeLists.txt index f43149530..55048698e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,12 +13,15 @@ enable_testing() option(WITH_DOC "Build documentation." ON) option(USE_NLS "Localisation support." ON) +INCLUDE_DIRECTORIES(/usr/src/skel/usr/include) +LINK_DIRECTORIES(/usr/src/skel/usr/lib) + set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake") # Add coverage target set(CMAKE_CXX_FLAGS_COVERAGE "-g -fprofile-arcs -ftest-coverage") -set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "-lgcov") -set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "-lgcov") +set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "-lgcov -lSystem") +set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "-lgcov -lSystem") # Work around bug in GNUInstallDirs if (EXISTS "/etc/debian_version") @@ -211,7 +214,9 @@ add_subdirectory(apt-private) add_subdirectory(apt-inst) add_subdirectory(cmdline) add_subdirectory(completions) +if (WITH_DOC) add_subdirectory(doc) +endif() add_subdirectory(dselect) add_subdirectory(ftparchive) add_subdirectory(methods) diff --git a/apt-pkg/CMakeLists.txt b/apt-pkg/CMakeLists.txt index e3e078b81..8d7fcd9e6 100644 --- a/apt-pkg/CMakeLists.txt +++ b/apt-pkg/CMakeLists.txt @@ -37,7 +37,7 @@ file(GLOB_RECURSE headers "*.h") # Create a library using the C++ files add_library(apt-pkg SHARED ${library}) -add_dependencies(apt-pkg apt-pkg-versionscript) +#add_dependencies(apt-pkg apt-pkg-versionscript) # Link the library and set the SONAME target_include_directories(apt-pkg PRIVATE ${ZLIB_INCLUDE_DIRS} diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 3c85f8adf..231c5eed9 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -266,7 +266,7 @@ static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const ms if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true) { - MessageInsecureRepository(false, msgstr, repo); + //MessageInsecureRepository(false, msgstr, repo); return true; } @@ -1294,7 +1294,6 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st { // for simplicity, the transaction manager is always InRelease // even if it doesn't exist. - TransactionManager->IMSHit = true; I->PartialFile = I->DestFile = I->GetFinalFilename(); } @@ -1412,7 +1411,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false) { // optional targets that we do not have in the Release file are skipped - if (hasHashes == true && Target.IsOptional) + if (Target.IsOptional) { new CleanupItem(Owner, TransactionManager, Target); continue; @@ -1536,6 +1535,13 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ } else { + // if the source wanted these files they should have given us a release file :/ + if (Target.IsOptional) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + // if we have no file to patch, no point in trying trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false); } @@ -3070,6 +3076,8 @@ void pkgAcqIndex::StageDownloadDone(string const &Message) { // copy FinalFile into partial/ so that we check the hash again string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI)); + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + unlink(DestFile.c_str()); if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0) _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str()); else @@ -3078,7 +3086,10 @@ void pkgAcqIndex::StageDownloadDone(string const &Message) Filename = DestFile; } Stage = STAGE_DECOMPRESS_AND_VERIFY; - Desc.URI = "store:" + Filename; + if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile)) + Desc.URI = "copy:" + Filename; + else + Desc.URI = "store:" + Filename; QueueURI(Desc); SetActiveSubprocess(::URI(Desc.URI).Access); return; @@ -3603,7 +3614,7 @@ std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls) should be so this could produce request order-dependent anomalies */ if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true) { - pkgTagFile TagFile(&rf, rf.Size()); + pkgTagFile TagFile(&rf); pkgTagSection Section; if (TagFile.Step(Section) == true) server = Section.FindS("Changelogs"); diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index b62c50c00..8df7a3239 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -435,6 +435,24 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) } else { FullQueueName = AccessSchema + U.Host; + + int parallel(_config->FindI("Acquire::"+U.Access+"::MaxParallel",8)); + if (parallel > 0) { + typedef map<string, int> indexmap; + static indexmap indices; + + pair<indexmap::iterator, bool> cache(indices.insert(indexmap::value_type(FullQueueName, -1))); + if (cache.second || cache.first->second == -1) { + int &index(indices[U.Access]); + if (index >= parallel) + index = 0; + cache.first->second = index++; + } + + ostringstream value; + value << U.Access << "::" << cache.first->second; + FullQueueName = value.str(); + } } unsigned int Instances = 0, SchemaLength = AccessSchema.length(); @@ -591,7 +609,6 @@ static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher) struct passwd const * const pw = getpwnam(SandboxUser.c_str()); if (pw == NULL) { - _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str()); _config->Set("APT::Sandbox::User", ""); return; } diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc index e16117b70..a0ba648b4 100644 --- a/apt-pkg/aptconfiguration.cc +++ b/apt-pkg/aptconfiguration.cc @@ -198,7 +198,7 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All, // FIXME: Remove support for the old APT::Acquire::Translation // it was undocumented and so it should be not very widthly used string const oldAcquire = _config->Find("APT::Acquire::Translation",""); - if (oldAcquire.empty() == false && oldAcquire != "environment") { + if (oldAcquire.empty() == false && oldAcquire != "environment" && !_config->Exists("Acquire::Languages")) { // TRANSLATORS: the two %s are APT configuration options _error->Notice("Option '%s' is deprecated. Please use '%s' instead, see 'man 5 apt.conf' for details.", "APT::Acquire::Translation", "Acquire::Languages"); diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc index b5f32fc29..90f803ad6 100644 --- a/apt-pkg/cachefile.cc +++ b/apt-pkg/cachefile.cc @@ -90,7 +90,7 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) return false; Cache.reset(new pkgCache(Map.get())); if (_error->PendingError() == true) - return false; + return _error->ReturnError(); this->Cache = Cache.release(); this->Map = Map.release(); @@ -102,7 +102,7 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) return false; if (_error->PendingError() == true) - return false; + return _error->ReturnError(); if (BuildSourceList(Progress) == false) return false; @@ -118,14 +118,8 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) if (Res == false) return _error->Error(_("The package lists or status file could not be parsed or opened.")); - /* This sux, remove it someday */ - if (_error->PendingError() == true) - _error->Warning(_("You may want to run apt-get update to correct these problems")); - if (Cache == nullptr) Cache.reset(new pkgCache(Map.get())); - if (_error->PendingError() == true) - return false; this->Map = Map.release(); this->Cache = Cache.release(); @@ -159,7 +153,7 @@ bool pkgCacheFile::BuildPolicy(OpProgress * /*Progress*/) Policy.reset(new pkgPolicy(Cache)); if (_error->PendingError() == true) - return false; + return _error->ReturnError(); if (ReadPinFile(*Policy) == false || ReadPinDir(*Policy) == false) return false; @@ -185,7 +179,7 @@ bool pkgCacheFile::BuildDepCache(OpProgress *Progress) DCache.reset(new pkgDepCache(Cache,Policy)); if (_error->PendingError() == true) - return false; + return _error->ReturnError(); if (DCache->Init(Progress) == false) return false; @@ -209,8 +203,6 @@ bool pkgCacheFile::Open(OpProgress *Progress, bool WithLock) if (Progress != NULL) Progress->Done(); - if (_error->PendingError() == true) - return false; return true; } @@ -256,7 +248,7 @@ bool pkgCacheFile::AddIndexFile(pkgIndexFile * const File) /*{{{*/ if (_error->PendingError() == true) { delete Cache; Cache = nullptr; - return false; + return _error->ReturnError(); } return true; } diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 62d0ab59c..e594f3e7d 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -57,7 +57,7 @@ template<typename Str, typename Itr> class pkgCache::Iterator : Str* OwnerPointer() const { return static_cast<Itr const*>(this)->OwnerPointer(); } protected: - Str *S; + Str *volatile S; pkgCache *Owner; public: @@ -214,6 +214,7 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;} inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;} + inline const char *Display() const {return S->Display == 0?0:Owner->StrP + S->Display;} /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;} @@ -231,6 +232,7 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> { DescIterator TranslatedDescription() const; inline DepIterator DependsList() const; inline PrvIterator ProvidesList() const; + inline TagIterator TagList() const; inline VerFileIterator FileList() const; bool Downloadable() const; inline const char *PriorityType() const {return Owner->Priority(S->Priority);} @@ -247,6 +249,33 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> { inline VerIterator() : Iterator<Version, VerIterator>() {} }; /*}}}*/ +// Tag Iterator /*{{{*/ +class pkgCache::TagIterator : public Iterator<Tag, TagIterator> { + public: + inline Tag* OwnerPointer() const { + return (Owner != 0) ? Owner->TagP : 0; + } + + // Iteration + void operator ++(int) {if (S != Owner->TagP) S = Owner->TagP + S->NextTag;}; + inline void operator ++() {operator ++(0);}; + + // Comparison + inline bool operator ==(const TagIterator &B) const {return S == B.S;}; + inline bool operator !=(const TagIterator &B) const {return S != B.S;}; + int CompareTag(const TagIterator &B) const; + + // Accessors + inline const char *Name() const {return Owner->StrP + S->Name;}; + inline unsigned long Index() const {return S - Owner->TagP;}; + + inline TagIterator(pkgCache &Owner,Tag *Trg = 0) : Iterator<Tag, TagIterator>(Owner, Trg) { + if (S == 0) + S = OwnerPointer(); + } + inline TagIterator() : Iterator<Tag, TagIterator>() {} +}; + /*}}}*/ // Description Iterator /*{{{*/ class pkgCache::DescIterator : public Iterator<Description, DescIterator> { public: @@ -515,6 +544,8 @@ inline pkgCache::DescIterator pkgCache::VerIterator::DescriptionList() const {return DescIterator(*Owner,Owner->DescP + S->DescriptionList);} inline pkgCache::PrvIterator pkgCache::VerIterator::ProvidesList() const {return PrvIterator(*Owner,Owner->ProvideP + S->ProvidesList,S);} +inline pkgCache::TagIterator pkgCache::VerIterator::TagList() const + {return TagIterator(*Owner,Owner->TagP + S->TagList);}; inline pkgCache::DepIterator pkgCache::VerIterator::DependsList() const {return DepIterator(*Owner,Owner->DepP + S->DependsList,S);} inline pkgCache::VerFileIterator pkgCache::VerIterator::FileList() const diff --git a/apt-pkg/contrib/error.cc b/apt-pkg/contrib/error.cc index c06ea8364..7d397d2c6 100644 --- a/apt-pkg/contrib/error.cc +++ b/apt-pkg/contrib/error.cc @@ -227,6 +227,15 @@ void GlobalError::Discard() { PendingFlag = false; } /*}}}*/ +// GlobalError::ReturnError - convert a stored error to a return code /*{{{*/ +bool GlobalError::ReturnError() { + for (auto &message : Messages) + if (message.Type == ERROR) + message.Type = WARNING; + PendingFlag = false; + return false; +} + /*}}}*/ // GlobalError::empty - does our error list include anything? /*{{{*/ bool GlobalError::empty(MsgType const &threshold) const { if (PendingFlag == true) diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h index b01a5fc1b..5ad408d25 100644 --- a/apt-pkg/contrib/error.h +++ b/apt-pkg/contrib/error.h @@ -227,6 +227,26 @@ public: /*{{{*/ */ inline bool PendingError() const APT_PURE {return PendingFlag;}; + /** \brief convert a stored error to a return code + * + * Put simply, the entire concept of PendingError() is flawed :/. + * + * The typical "if (PendingError()) return false;" check that is + * strewn throughout the codebase "compounds", making it impossible + * for there to be any nuance about the notion of "error" when a + * subsystem needs to fail but a higher-level system needs to work. + * + * However, the codebase is also horribly broken with respect to + * errors, as it fails to use C++ exceptions when warranted and + * instead relies on this insane indirect error mechanism to check + * the failure status of a constructor. What is thereby needed is + * a way to clear the PendingError() flag without also discarding + * the underlying errors, so we have to convert them to warnings. + * + * \return \b false + */ + bool ReturnError() APT_COLD; + /** \brief is the list empty? * * Can be used to check if the current stack level doesn't include diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index e4c40fb4f..96820c9b0 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -26,6 +26,7 @@ #include <apt-pkg/aptconfiguration.h> #include <apt-pkg/configuration.h> #include <apt-pkg/macros.h> +#include <apt-pkg/endian.h> #include <ctype.h> #include <stdarg.h> @@ -74,6 +75,14 @@ #endif #include <apti18n.h> + +//posix spawn +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <spawn.h> +#include <sys/wait.h> + /*}}}*/ using namespace std; @@ -81,6 +90,8 @@ using namespace std; /* Should be a multiple of the common page size (4096) */ static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024; +extern char **environ; + // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -154,7 +165,6 @@ bool RunScripts(const char *Cnf) return true; } - /*}}}*/ // CopyFile - Buffered copy of a file /*{{{*/ // --------------------------------------------------------------------- @@ -1956,12 +1966,6 @@ public: dup2(compressed_fd,STDIN_FILENO); dup2(Pipe[1],STDOUT_FILENO); } - int const nullfd = open("/dev/null", O_WRONLY); - if (nullfd != -1) - { - dup2(nullfd,STDERR_FILENO); - close(nullfd); - } SetCloseExec(STDOUT_FILENO,false); SetCloseExec(STDIN_FILENO,false); @@ -2792,7 +2796,7 @@ static std::string APT_NONNULL(1) GetTempDirEnv(char const * const env) /*{{{*/ stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0) // exists and is directory tmpdir = "/tmp"; else if (geteuid() != 0 && // root can do everything anyway - faccessat(AT_FDCWD, tmpdir, R_OK | W_OK | X_OK, AT_EACCESS) != 0) // current user has rwx access to directory + access(tmpdir, R_OK | W_OK | X_OK) != 0) // current user has rwx access to directory tmpdir = "/tmp"; return string(tmpdir); diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index dddeb70f5..bb05239c9 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -31,6 +31,28 @@ #include <zlib.h> +#include <errno.h> +static inline int _execvp(const char *file, char *const argv[]) { + int rv = execvp(file, argv); + fprintf(stderr, "execvp failed, trying shell\n"); + if (errno == ENOEXEC || errno == EPERM) { + int argc; + for (argc = 0; argv[argc] != NULL; argc++); + char *newargv[argc+4]; + newargv[0] = "/bin/sh"; + newargv[1] = "-c"; + newargv[2] = "exec \"$0\" \"$@\""; + for (int i = 0; i<argc; i++) { + newargv[i+3] = argv[i]; + } + newargv[argc+3] = NULL; + return execvp(newargv[0], newargv); + } + return rv; +} + +#define execvp(x, y) _execvp(x, y) + #ifndef APT_8_CLEANER_HEADERS using std::string; #endif @@ -158,6 +180,7 @@ class FileFd APT_HIDDEN bool FileFdError(const char* Description,...) APT_PRINTF(2) APT_COLD; }; + bool RunScripts(const char *Cnf); bool CopyFile(FileFd &From,FileFd &To); bool RemoveFile(char const * const Function, std::string const &FileName); diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc index cdf9481cb..d4ccf47c7 100644 --- a/apt-pkg/contrib/gpgv.cc +++ b/apt-pkg/contrib/gpgv.cc @@ -92,7 +92,7 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, #define EINTERNAL 111 std::string const aptkey = _config->Find("Dir::Bin::apt-key", CMAKE_INSTALL_FULL_BINDIR "/apt-key"); - bool const Debug = _config->FindB("Debug::Acquire::gpgv", false); + bool const Debug = _config->FindB("Debug::Acquire::gpgv", false); struct exiter { std::vector<const char *> files; void operator ()(int code) APT_NORETURN { @@ -103,8 +103,9 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, std::vector<const char *> Args; - Args.reserve(10); + Args.reserve(11); + Args.push_back("/bin/sh"); Args.push_back(aptkey.c_str()); Args.push_back("--quiet"); Args.push_back("--readonly"); @@ -214,6 +215,21 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, Args.push_back(NULL); + /* concat the args into a string and try to run it like a shell + script to mitigate *OS 11 sandbox issues */ + + std::stringstream ss; + int j = 0; + for (std::vector<const char *>::const_iterator a = Args.begin(); *a != NULL; ++a) + { + if(j != 0) + ss << " "; + ss << *a; + j++; + } + + std::string ArgString = ss.str(); + if (Debug == true) { std::clog << "Preparing to exec: "; @@ -238,8 +254,8 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, putenv((char *)"LC_ALL="); putenv((char *)"LC_MESSAGES="); } - - + + // We have created tempfiles we have to clean up // and we do an additional check, so fork yet another time โฆ pid_t pid = ExecFork(); @@ -251,8 +267,9 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, { if (statusfd != -1) dup2(fd[1], statusfd); - execvp(Args[0], (char **) &Args[0]); - apt_error(std::cerr, statusfd, fd, "Couldn't execute %s to check %s", Args[0], File.c_str()); + execlp("sh", "sh", "-c", ArgString.c_str(), NULL); //run as a shell script instead + //execvp(Args[0], (char **) &Args[0]); + apt_error(std::cerr, statusfd, fd, "Couldn't execute %s to check %s", Args[0], File.c_str()); local_exit(EINTERNAL); } diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 662c2bf8b..27e617751 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -141,8 +141,8 @@ APT_PURE bool HashString::usable() const /*{{{*/ { return ( (Type != "Checksum-FileSize") && - (Type != "MD5Sum") && - (Type != "SHA1") && + //(Type != "MD5Sum") && + //(Type != "SHA1") && !IsConfigured(Type.c_str(), "Untrusted") ); } diff --git a/apt-pkg/contrib/macros.h b/apt-pkg/contrib/macros.h index bc1f523ea..7262bc18b 100644 --- a/apt-pkg/contrib/macros.h +++ b/apt-pkg/contrib/macros.h @@ -118,7 +118,7 @@ #ifndef APT_10_CLEANER_HEADERS #if APT_GCC_VERSION >= 0x0300 #define __must_check __attribute__ ((warn_unused_result)) - #define __deprecated __attribute__ ((deprecated)) + #define __deprecated __attribute__((deprecated)) #define __attrib_const __attribute__ ((__const__)) #define __like_printf(n) __attribute__((format(printf, n, n + 1))) #else diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index cd24a2808..fa93c91af 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -40,7 +40,6 @@ MMap::MMap(FileFd &F,unsigned long Flags) : Flags(Flags), iSize(0), Base(nullptr), SyncToFd(nullptr) { - if ((Flags & NoImmMap) != NoImmMap) Map(F); } /*}}}*/ @@ -107,14 +106,14 @@ bool MMap::Map(FileFd &Fd) if (unlikely(Base == nullptr)) return _error->Errno("MMap-malloc", _("Couldn't make mmap of %llu bytes"), iSize); SyncToFd = new FileFd(); - return Fd.Read(Base, iSize); + return Fd.Seek(0L) && Fd.Read(Base, iSize); } // FIXME: Writing to compressed fd's ? int const dupped_fd = dup(Fd.Fd()); if (dupped_fd == -1) return _error->Errno("mmap", _("Couldn't duplicate file descriptor %i"), Fd.Fd()); - Base = calloc(iSize, 1); + Base = malloc(iSize); if (unlikely(Base == nullptr)) return _error->Errno("MMap-calloc", _("Couldn't make mmap of %llu bytes"), iSize); SyncToFd = new FileFd (dupped_fd); @@ -195,7 +194,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) { if (SyncToFd != 0) { - if (!SyncToFd->Seek(0) || + if (!SyncToFd->Seek(Start) || !SyncToFd->Write (((char *)Base)+Start, Stop-Start)) return false; } @@ -203,7 +202,8 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) { #ifdef _POSIX_SYNCHRONIZED_IO unsigned long long const PSize = sysconf(_SC_PAGESIZE); - if (msync((char *)Base+(Start/PSize)*PSize, Stop - Start, MS_SYNC) < 0) + Start = (Start/PSize)*PSize; + if (msync((char *)Base+Start, Stop - Start, MS_SYNC) < 0) return _error->Errno("msync", _("Unable to synchronize mmap")); #endif } @@ -217,7 +217,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) /* */ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Workspace, unsigned long const &Grow, unsigned long const &Limit) : - MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(Workspace), + MMap(Flags), Fd(&F), WorkSpace(Workspace), GrowFactor(Grow), Limit(Limit) { // disable Moveable if we don't grow @@ -251,7 +251,7 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Work and could come in handy later than we are able to grow such an mmap */ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace, unsigned long const &Grow, unsigned long const &Limit) : - MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace), + MMap(Flags | UnMapped), Fd(0), WorkSpace(WorkSpace), GrowFactor(Grow), Limit(Limit) { // disable Moveable if we don't grow @@ -307,10 +307,11 @@ DynamicMMap::~DynamicMMap() if (validData() == false) return; #ifdef _POSIX_MAPPED_FILES - munmap(Base, WorkSpace); -#else - free(Base); + if ((Flags & Fallback) != Fallback) { + munmap(Base, WorkSpace); + } else #endif + free(Base); return; } @@ -489,12 +490,14 @@ bool DynamicMMap::Grow() { if ((Flags & Moveable) != Moveable) return false; - Base = realloc(Base, newSize); - if (Base == NULL) + auto Temp = realloc(Base, newSize); + if (Temp == NULL) return false; - else + else { + Base = Temp; /* Set new memory to 0 */ memset((char*)Base + WorkSpace, 0, newSize - WorkSpace); + } } Pools =(Pool*) Base + poolOffset; diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index 62e64b95e..b776959c2 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -58,7 +58,7 @@ class MMap public: - enum OpenFlags {NoImmMap = (1<<0),Public = (1<<1),ReadOnly = (1<<2), + enum OpenFlags {Public = (1<<1),ReadOnly = (1<<2), UnMapped = (1<<3), Moveable = (1<<4), Fallback = (1 << 5)}; // Simple accessors diff --git a/apt-pkg/contrib/srvrec.cc b/apt-pkg/contrib/srvrec.cc index 327e59937..f2c45a458 100644 --- a/apt-pkg/contrib/srvrec.cc +++ b/apt-pkg/contrib/srvrec.cc @@ -12,6 +12,7 @@ #include <netinet/in.h> #include <arpa/nameser.h> +#include <apt-pkg/nameser_compat.h> #include <resolv.h> #include <time.h> @@ -50,7 +51,7 @@ bool GetSrvRecords(std::string host, int port, std::vector<SrvRec> &Result) bool GetSrvRecords(std::string name, std::vector<SrvRec> &Result) { - unsigned char answer[PACKETSZ]; + unsigned char answer[NS_PACKETSZ]; int answer_len, compressed_name_len; int answer_count; @@ -77,7 +78,7 @@ bool GetSrvRecords(std::string name, std::vector<SrvRec> &Result) return _error->Warning("dn_skipname failed %i", compressed_name_len); // pt points to the first answer record, go over all of them now - unsigned char *pt = answer+sizeof(HEADER)+compressed_name_len+QFIXEDSZ; + unsigned char *pt = answer+sizeof(HEADER)+compressed_name_len+NS_QFIXEDSZ; while ((int)Result.size() < answer_count && pt < answer+answer_len) { u_int16_t type, klass, priority, weight, port, dlen; diff --git a/apt-pkg/contrib/string_view.h b/apt-pkg/contrib/string_view.h index c504edd27..52ad71d5c 100644 --- a/apt-pkg/contrib/string_view.h +++ b/apt-pkg/contrib/string_view.h @@ -14,6 +14,7 @@ #include <string.h> #include <string> #include <apt-pkg/macros.h> +#include <apt-pkg/missing.h> namespace APT { @@ -112,18 +113,6 @@ public: constexpr size_t length() const { return size_; } }; -/** - * \brief Faster comparison for string views (compare size before data) - * - * Still stable, but faster than the normal ordering. */ -static inline int StringViewCompareFast(StringView a, StringView b) { - if (a.size() != b.size()) - return a.size() - b.size(); - - return memcmp(a.data(), b.data(), a.size()); -} - - } inline bool operator ==(const char *other, APT::StringView that); diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 73f27aa6c..ba41172fd 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -159,6 +159,27 @@ static inline int isspace_ascii_inline(int const c) return (c >= 9 && c <= 13) || c == ' '; } +// StringViewCompareFast - awkward attempt to optimize cache generation /*{{{*/ +#ifdef APT_PKG_EXPOSE_STRING_VIEW +/** + * \brief Faster comparison for string views (compare size before data) + * + * Still stable, but faster than the normal ordering. + * As this is used for package comparison this *MUST* be case insensitive, + * as the alternative is to lower case all dependency fields which is slow. */ +static inline int StringViewCompareFast(APT::StringView a, APT::StringView b) { + if (a.size() != b.size()) + return a.size() - b.size(); + auto l(a.data()), r(b.data()); + for (auto e(a.size()), i(decltype(e)(0)); i != e; ++i) + if (tolower_ascii_inline(l[i]) != tolower_ascii_inline(r[i])) + return tolower_ascii(l[i]) < tolower_ascii(r[i]) ? -1 : 1; + return 0; +} +#endif + /*}}}*/ + + std::string StripEpoch(const std::string &VerStr); #define APT_MKSTRCMP(name,func) \ diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index c55847305..6b162372d 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -134,6 +134,7 @@ pkgCacheListParser * debTranslationsIndex::CreateListParser(FileFd &Pkg) if (newError) { delete Parser; + _error->ReturnError(); return nullptr; } else @@ -168,6 +169,7 @@ pkgCacheListParser * debStatusIndex::CreateListParser(FileFd &Pkg) if (newError) { delete Parser; + _error->ReturnError(); return nullptr; } else @@ -250,6 +252,7 @@ pkgCacheListParser * debDebPkgFileIndex::CreateListParser(FileFd &Pkg) if (newError) { delete Parser; + _error->ReturnError(); return nullptr; } else diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 4e61f0fc2..3bcd381c0 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -63,6 +63,7 @@ debListParser::debListParser(FileFd *File) : else forceEssential.emplace_back("apt"); forceImportant = _config->FindVector("pkgCacheGen::ForceImportant"); + Arch = _config->Find("APT::architecture"); } /*}}}*/ // ListParser::Package - Return the package name /*{{{*/ @@ -87,7 +88,7 @@ string debListParser::Package() { } if(unlikely(Result.empty() == true)) - _error->Error("Encountered a section with no Package: header"); + _error->Warning("Encountered a section with no Package: header"); return Result; } /*}}}*/ @@ -159,6 +160,15 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) const char *Start; const char *Stop; + if (Section.Find("Name",Start,Stop) == true) + { + Ver->Display = WriteString(Start, Stop - Start); + } + else if (Section.Find("Maemo-Display-Name",Start,Stop) == true) + { + Ver->Display = WriteString(Start, Stop - Start); + } + // Parse the section if (Section.Find(pkgTagSection::Key::Section,Start,Stop) == true) { @@ -255,6 +265,8 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) if (ParseProvides(Ver) == false) return false; + if (ParseTag(Ver) == false) + return false; return true; } @@ -847,7 +859,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator &Ver, Start = ParseDepends(Start, Stop, Package, Version, Op, false, false, false); if (Start == 0) - return _error->Error("Problem parsing dependency %zu",static_cast<size_t>(Key)); // TODO + return _error->Warning("Problem parsing dependency %zu",static_cast<size_t>(Key)); // TODO size_t const found = Package.rfind(':'); if (found == string::npos) @@ -915,7 +927,7 @@ bool debListParser::ParseProvides(pkgCache::VerIterator &Ver) Start = ParseDepends(Start,Stop,Package,Version,Op, false, false, false); const size_t archfound = Package.rfind(':'); if (Start == 0) - return _error->Error("Problem parsing Provides line"); + return _error->Warning("Problem parsing Provides line"); if (unlikely(Op != pkgCache::Dep::NoOp && Op != pkgCache::Dep::Equals)) { _error->Warning("Ignoring Provides line with non-equal DepCompareOp for package %s", Package.to_string().c_str()); } else if (archfound != string::npos) { @@ -986,6 +998,46 @@ bool debListParser::ParseProvides(pkgCache::VerIterator &Ver) return true; } /*}}}*/ +// ListParser::ParseTag - Parse the tag list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debListParser::ParseTag(pkgCache::VerIterator &Ver) +{ + const char *Start; + const char *Stop; + if (Section.Find("Tag",Start,Stop) == false) + return true; + + while (1) { + while (1) { + if (Start == Stop) + return true; + if (Stop[-1] != ' ' && Stop[-1] != '\t') + break; + --Stop; + } + + const char *Begin = Stop - 1; + while (Begin != Start && Begin[-1] != ' ' && Begin[-1] != ',') + --Begin; + + if (NewTag(Ver, Begin, Stop - Begin) == false) + return false; + + while (1) { + if (Begin == Start) + return true; + if (Begin[-1] == ',') + break; + --Begin; + } + + Stop = Begin - 1; + } + + return true; +} + /*}}}*/ // ListParser::GrabWord - Matches a word and returns /*{{{*/ // --------------------------------------------------------------------- /* Looks for a word in a list of words - for ParseStatus */ diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 39f42915c..5e945812d 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -52,10 +52,13 @@ class APT_HIDDEN debListParser : public pkgCacheListParser pkgTagSection Section; map_filesize_t iOffset; + std::string Arch; + virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver); bool ParseDepends(pkgCache::VerIterator &Ver, pkgTagSection::Key Key, unsigned int Type); bool ParseProvides(pkgCache::VerIterator &Ver); + bool ParseTag(pkgCache::VerIterator &Ver); #ifdef APT_PKG_EXPOSE_STRING_VIEW APT_HIDDEN static bool GrabWord(APT::StringView Word,const WordList *List,unsigned char &Out); diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 879c941e1..f25906fba 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -376,7 +376,7 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro if (OpenMaybeClearSignedFile(Filename, Fd) == false) return false; - pkgTagFile TagFile(&Fd, Fd.Size()); + pkgTagFile TagFile(&Fd); if (Fd.IsOpen() == false || Fd.Failed()) { if (ErrorText != NULL) @@ -443,18 +443,15 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro bool AuthPossible = false; if(FoundHashSum == false) - _error->Warning(_("No Hash entry in Release file %s"), Filename.c_str()); + /*_error->Warning(_("No Hash entry in Release file %s"), Filename.c_str())*/; else if(FoundStrongHashSum == false) - _error->Warning(_("No Hash entry in Release file %s which is considered strong enough for security purposes"), Filename.c_str()); + /*_error->Warning(_("No Hash entry in Release file %s which is considered strong enough for security purposes"), Filename.c_str())*/; else AuthPossible = true; std::string const StrDate = Section.FindS("Date"); if (RFC1123StrToTime(StrDate.c_str(), Date) == false) - { - _error->Warning( _("Invalid '%s' entry in Release file %s"), "Date", Filename.c_str()); Date = 0; - } bool CheckValidUntil = _config->FindB("Acquire::Check-Valid-Until", true); if (d->CheckValidUntil == metaIndex::TRI_NO) @@ -804,7 +801,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/ File->Size = Buf.st_size; File->mtime = Buf.st_mtime; - pkgTagFile TagFile(&Rel, Rel.Size()); + pkgTagFile TagFile(&Rel); pkgTagSection Section; if (Rel.IsOpen() == false || Rel.Failed() || TagFile.Step(Section) == false) return false; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index f903617f7..879760934 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -34,8 +34,8 @@ class APT_HIDDEN debReleaseIndex : public metaIndex APT_HIDDEN std::string MetaIndexFile(const char *Types) const; APT_HIDDEN std::string MetaIndexURI(const char *Type) const; - debReleaseIndex(std::string const &URI, std::string const &Dist, std::map<std::string,std::string> const &Options); - debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted, std::map<std::string,std::string> const &Options); + debReleaseIndex(std::string const &URI, std::string const &Dist, std::map<std::string,std::string> const &Options = std::map<std::string,std::string>()); + debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted, std::map<std::string,std::string> const &Options = std::map<std::string,std::string>()); virtual ~debReleaseIndex(); virtual std::string ArchiveURI(std::string const &File) const APT_OVERRIDE {return URI + File;}; diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc index a132f34a6..696c55e6a 100644 --- a/apt-pkg/deb/debrecords.cc +++ b/apt-pkg/deb/debrecords.cc @@ -35,7 +35,7 @@ using std::string; // RecordParser::debRecordParser - Constructor /*{{{*/ debRecordParser::debRecordParser(string FileName,pkgCache &Cache) : debRecordParserBase(), d(NULL), File(FileName, FileFd::ReadOnly, FileFd::Extension), - Tags(&File, std::max(Cache.Head().MaxVerFileSize, Cache.Head().MaxDescFileSize) + 200) + Tags(&File) { } /*}}}*/ @@ -74,6 +74,15 @@ string debRecordParserBase::Name() return Result; } /*}}}*/ +// RecordParserBase::Display - Return the package homepage /*{{{*/ +string debRecordParserBase::Display() +{ + string display(Section.FindS("Name")); + if (display.empty()) + display = Section.FindS("Maemo-Display-Name"); + return display; +} + /*}}}*/ // RecordParserBase::Homepage - Return the package homepage /*{{{*/ string debRecordParserBase::Homepage() { @@ -153,7 +162,7 @@ string debRecordParserBase::LongDesc(std::string const &lang) } char const * const codeset = nl_langinfo(CODESET); - if (strcmp(codeset,"UTF-8") != 0) { + if (strcmp(codeset,"US-ASCII") != 0 && strcmp(codeset,"UTF-8") != 0) { string dest; UTF8ToCodeset(codeset, orig, &dest); return dest; @@ -201,6 +210,12 @@ void debRecordParserBase::GetRec(const char *&Start,const char *&Stop) Section.GetSection(Start,Stop); } /*}}}*/ +// RecordParserBase::Find - Locate a tag /*{{{*/ +bool debRecordParserBase::Find(const char *Tag,const char *&Start, const char *&End) +{ + return Section.Find(Tag,Start,End); +} + /*}}}*/ debRecordParserBase::~debRecordParserBase() {} bool debDebFileRecordParser::LoadContent() @@ -217,7 +232,7 @@ bool debDebFileRecordParser::LoadContent() content << "\n\n"; controlContent = content.str(); - if (Section.Scan(controlContent.c_str(), controlContent.length()) == false) + if (Section.Scan(controlContent.c_str(), controlContent.length(), false) == false) return _error->Error(_("Unable to parse package file %s (%d)"), debFileName.c_str(), 3); return true; } diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h index ae75a2b78..bc800bc89 100644 --- a/apt-pkg/deb/debrecords.h +++ b/apt-pkg/deb/debrecords.h @@ -44,12 +44,14 @@ class APT_HIDDEN debRecordParserBase : public pkgRecords::Parser virtual std::string ShortDesc(std::string const &lang) APT_OVERRIDE; virtual std::string LongDesc(std::string const &lang) APT_OVERRIDE; virtual std::string Name() APT_OVERRIDE; + virtual std::string Display() APT_OVERRIDE; virtual std::string Homepage() APT_OVERRIDE; // An arbitrary custom field virtual std::string RecordField(const char *fieldName) APT_OVERRIDE; virtual void GetRec(const char *&Start,const char *&Stop) APT_OVERRIDE; + virtual bool Find(const char *Tag,const char *&Start, const char *&End) APT_OVERRIDE; debRecordParserBase(); virtual ~debRecordParserBase(); diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index d664b609e..caaa53063 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -38,7 +38,7 @@ debSrcRecordParser::debSrcRecordParser(std::string const &File,pkgIndexFile cons if (File.empty() == false) { if (Fd.Open(File, FileFd::ReadOnly, FileFd::Extension)) - Tags.Init(&Fd, 102400); + Tags.Init(&Fd); } } std::string debSrcRecordParser::Package() const /*{{{*/ diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 80bee03dd..01282efcc 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1386,7 +1386,9 @@ static void cleanUpTmpDir(char * const tmpdir) /*{{{*/ if (unlikely(Ent->d_type != DT_LNK && Ent->d_type != DT_UNKNOWN)) continue; #endif - if (unlikely(unlinkat(dfd, Ent->d_name, 0) != 0)) + char path[strlen(tmpdir) + 1 + strlen(Ent->d_name) + 1]; + sprintf(path, "%s/%s", tmpdir, Ent->d_name); + if (unlikely(unlink(path) != 0)) break; } closedir(D); @@ -1697,7 +1699,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) bool dpkgMultiArch = debSystem::SupportsMultiArch(); // start pty magic before the loop - StartPtyMagic(); + //StartPtyMagic(); or not... // Tell the progress that its starting and fork dpkg d->progress->Start(d->master); @@ -1758,6 +1760,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) case Item::Remove: case Item::Purge: ADDARGC("--force-depends"); + ADDARGC("--force-remove-reinstreq"); if (std::any_of(I, J, ItemIsEssential)) ADDARGC("--force-remove-essential"); ADDARGC("--remove"); @@ -2096,7 +2099,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) } } // dpkg is done at this point - StopPtyMagic(); + //StopPtyMagic(); CloseLog(); if (d->dpkg_error.empty() == false) diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc index a2520441b..59c7720c3 100644 --- a/apt-pkg/edsp.cc +++ b/apt-pkg/edsp.cc @@ -629,7 +629,7 @@ bool EDSP::ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progres FileFd in; in.OpenDescriptor(input, FileFd::ReadOnly, true); - pkgTagFile response(&in, 100); + pkgTagFile response(&in); pkgTagSection section; std::set<decltype(Cache.PkgBegin()->ID)> seenOnce; @@ -1337,7 +1337,7 @@ bool EIPP::ReadResponse(int const input, pkgPackageManager * const PM, OpProgres FileFd in; in.OpenDescriptor(input, FileFd::ReadOnly); - pkgTagFile response(&in, 100); + pkgTagFile response(&in); pkgTagSection section; while (response.Step(section) == true) { diff --git a/apt-pkg/endian.h b/apt-pkg/endian.h new file mode 100644 index 000000000..e89694a44 --- /dev/null +++ b/apt-pkg/endian.h @@ -0,0 +1,118 @@ +// "License": Public Domain +// I, Mathias Panzenbรถck, place this file hereby into the public domain. Use it at your own risk for whatever you like. +// In case there are jurisdictions that don't support putting things in the public domain you can also consider it to +// be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it +// an example on how to get the endian conversion functions on different platforms. + +#ifndef PORTABLE_ENDIAN_H__ +#define PORTABLE_ENDIAN_H__ + +#if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) + +# define __WINDOWS__ + +#endif + +#if defined(__linux__) || defined(__CYGWIN__) + +# include <endian.h> + +#elif defined(__APPLE__) + +# include <libkern/OSByteOrder.h> + +# define htobe16(x) OSSwapHostToBigInt16(x) +# define htole16(x) OSSwapHostToLittleInt16(x) +# define be16toh(x) OSSwapBigToHostInt16(x) +# define le16toh(x) OSSwapLittleToHostInt16(x) + +# define htobe32(x) OSSwapHostToBigInt32(x) +# define htole32(x) OSSwapHostToLittleInt32(x) +# define be32toh(x) OSSwapBigToHostInt32(x) +# define le32toh(x) OSSwapLittleToHostInt32(x) + +# define htobe64(x) OSSwapHostToBigInt64(x) +# define htole64(x) OSSwapHostToLittleInt64(x) +# define be64toh(x) OSSwapBigToHostInt64(x) +# define le64toh(x) OSSwapLittleToHostInt64(x) + +# define __BYTE_ORDER BYTE_ORDER +# define __BIG_ENDIAN BIG_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __PDP_ENDIAN PDP_ENDIAN + +#elif defined(__OpenBSD__) + +# include <sys/endian.h> + +#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) + +# include <sys/endian.h> + +# define be16toh(x) betoh16(x) +# define le16toh(x) letoh16(x) + +# define be32toh(x) betoh32(x) +# define le32toh(x) letoh32(x) + +# define be64toh(x) betoh64(x) +# define le64toh(x) letoh64(x) + +#elif defined(__WINDOWS__) + +# include <winsock2.h> +# include <sys/param.h> + +# if BYTE_ORDER == LITTLE_ENDIAN + +# define htobe16(x) htons(x) +# define htole16(x) (x) +# define be16toh(x) ntohs(x) +# define le16toh(x) (x) + +# define htobe32(x) htonl(x) +# define htole32(x) (x) +# define be32toh(x) ntohl(x) +# define le32toh(x) (x) + +# define htobe64(x) htonll(x) +# define htole64(x) (x) +# define be64toh(x) ntohll(x) +# define le64toh(x) (x) + +# elif BYTE_ORDER == BIG_ENDIAN + + /* that would be xbox 360 */ +# define htobe16(x) (x) +# define htole16(x) __builtin_bswap16(x) +# define be16toh(x) (x) +# define le16toh(x) __builtin_bswap16(x) + +# define htobe32(x) (x) +# define htole32(x) __builtin_bswap32(x) +# define be32toh(x) (x) +# define le32toh(x) __builtin_bswap32(x) + +# define htobe64(x) (x) +# define htole64(x) __builtin_bswap64(x) +# define be64toh(x) (x) +# define le64toh(x) __builtin_bswap64(x) + +# else + +# error byte order not supported + +# endif + +# define __BYTE_ORDER BYTE_ORDER +# define __BIG_ENDIAN BIG_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __PDP_ENDIAN PDP_ENDIAN + +#else + +# error platform not supported + +#endif + +#endif diff --git a/apt-pkg/getservbyport_r.cc b/apt-pkg/getservbyport_r.cc new file mode 100644 index 000000000..cf78ad514 --- /dev/null +++ b/apt-pkg/getservbyport_r.cc @@ -0,0 +1,59 @@ +#define _GNU_SOURCE +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netdb.h> +#include <inttypes.h> +#include <errno.h> +#include <string.h> + +#ifndef HAVE_GETSERVBYPORT_R + +extern "C" int getservbyport_r(int port, const char *prots, + struct servent *se, char *buf, size_t buflen, struct servent **res) +{ + int i; + struct sockaddr_in sin = { + .sin_family = AF_INET, + .sin_port = (in_port_t) port, + }; + + if (!prots) { + int r = getservbyport_r(port, "tcp", se, buf, buflen, res); + if (r) r = getservbyport_r(port, "udp", se, buf, buflen, res); + return r; + } + + /* Align buffer */ + i = (uintptr_t)buf & sizeof(char *)-1; + if (!i) i = sizeof(char *); + if (buflen < 3*sizeof(char *)-i) + return ERANGE; + buf += sizeof(char *)-i; + buflen -= sizeof(char *)-i; + + if (strcmp(prots, "tcp") && strcmp(prots, "udp")) return EINVAL; + + se->s_port = port; + se->s_proto = (char *)prots; + se->s_aliases = (char **)buf; + buf += 2*sizeof(char *); + buflen -= 2*sizeof(char *); + se->s_aliases[1] = 0; + se->s_aliases[0] = se->s_name = buf; + + switch (getnameinfo((const struct sockaddr *) &sin, sizeof sin, 0, 0, buf, buflen, + strcmp(prots, "udp") ? 0 : NI_DGRAM)) { + case EAI_MEMORY: + case EAI_SYSTEM: + return ENOMEM; + default: + return ENOENT; + case 0: + break; + } + + *res = se; + return 0; +} +#endif diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index ca5c42cb7..cf3bb058d 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -550,7 +550,7 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, if(Debug) cout << "Signature verify for: " << *I << endl; - metaIndex *MetaIndex = new debReleaseIndex("","", {}); + metaIndex *MetaIndex = new debReleaseIndex("",""); string prefix = *I; string const releasegpg = *I+"Release.gpg"; diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc index 934943205..21765388f 100644 --- a/apt-pkg/indexfile.cc +++ b/apt-pkg/indexfile.cc @@ -344,6 +344,7 @@ pkgCacheListParser * pkgDebianIndexFile::CreateListParser(FileFd &Pkg) if (newError) { delete Parser; + _error->ReturnError(); return nullptr; } else diff --git a/apt-pkg/memrchr.cc b/apt-pkg/memrchr.cc new file mode 100644 index 000000000..edf8f346a --- /dev/null +++ b/apt-pkg/memrchr.cc @@ -0,0 +1,157 @@ +/* memrchr -- find the last occurrence of a byte in a memory block + + Copyright (C) 1991, 1993, 1996-1997, 1999-2000, 2003-2015 Free Software + Foundation, Inc. + + Based on strlen implementation by Torbjorn Granlund (tege@sics.se), + with help from Dan Sahlin (dan@sics.se) and + commentary by Jim Blandy (jimb@ai.mit.edu); + adaptation to memchr suggested by Dick Karpinski (dick@cca.ucsf.edu), + and implemented by Roland McGrath (roland@ai.mit.edu). + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ + +#include <config.h> + +#ifndef HAVE_MEMRCHR +#define reg_char char + +#include <string.h> +#include <limits.h> + +#undef __memrchr +#ifdef _LIBC +# undef memrchr +#endif + +#ifndef weak_alias +# define __memrchr memrchr +#endif + +/* Search no more than N bytes of S for C. */ +extern "C" void * +memrchr (const void *s, int c_in, size_t n) +{ + /* On 32-bit hardware, choosing longword to be a 32-bit unsigned + long instead of a 64-bit uintmax_t tends to give better + performance. On 64-bit hardware, unsigned long is generally 64 + bits already. Change this typedef to experiment with + performance. */ + typedef unsigned long int longword; + + const unsigned char *char_ptr; + const longword *longword_ptr; + longword repeated_one; + longword repeated_c; + unsigned reg_char c; + + c = (unsigned char) c_in; + + /* Handle the last few bytes by reading one byte at a time. + Do this until CHAR_PTR is aligned on a longword boundary. */ + for (char_ptr = (const unsigned char *) s + n; + n > 0 && (size_t) char_ptr % sizeof (longword) != 0; + --n) + if (*--char_ptr == c) + return (void *) char_ptr; + + longword_ptr = (const longword *) char_ptr; + + /* All these elucidatory comments refer to 4-byte longwords, + but the theory applies equally well to any size longwords. */ + + /* Compute auxiliary longword values: + repeated_one is a value which has a 1 in every byte. + repeated_c has c in every byte. */ + repeated_one = 0x01010101; + repeated_c = c | (c << 8); + repeated_c |= repeated_c << 16; + if (0xffffffffU < (longword) -1) + { + repeated_one |= repeated_one << 31 << 1; + repeated_c |= repeated_c << 31 << 1; + if (8 < sizeof (longword)) + { + size_t i; + + for (i = 64; i < sizeof (longword) * 8; i *= 2) + { + repeated_one |= repeated_one << i; + repeated_c |= repeated_c << i; + } + } + } + + /* Instead of the traditional loop which tests each byte, we will test a + longword at a time. The tricky part is testing if *any of the four* + bytes in the longword in question are equal to c. We first use an xor + with repeated_c. This reduces the task to testing whether *any of the + four* bytes in longword1 is zero. + + We compute tmp = + ((longword1 - repeated_one) & ~longword1) & (repeated_one << 7). + That is, we perform the following operations: + 1. Subtract repeated_one. + 2. & ~longword1. + 3. & a mask consisting of 0x80 in every byte. + Consider what happens in each byte: + - If a byte of longword1 is zero, step 1 and 2 transform it into 0xff, + and step 3 transforms it into 0x80. A carry can also be propagated + to more significant bytes. + - If a byte of longword1 is nonzero, let its lowest 1 bit be at + position k (0 <= k <= 7); so the lowest k bits are 0. After step 1, + the byte ends in a single bit of value 0 and k bits of value 1. + After step 2, the result is just k bits of value 1: 2^k - 1. After + step 3, the result is 0. And no carry is produced. + So, if longword1 has only non-zero bytes, tmp is zero. + Whereas if longword1 has a zero byte, call j the position of the least + significant zero byte. Then the result has a zero at positions 0, ..., + j-1 and a 0x80 at position j. We cannot predict the result at the more + significant bytes (positions j+1..3), but it does not matter since we + already have a non-zero bit at position 8*j+7. + + So, the test whether any byte in longword1 is zero is equivalent to + testing whether tmp is nonzero. */ + + while (n >= sizeof (longword)) + { + longword longword1 = *--longword_ptr ^ repeated_c; + + if ((((longword1 - repeated_one) & ~longword1) + & (repeated_one << 7)) != 0) + { + longword_ptr++; + break; + } + n -= sizeof (longword); + } + + char_ptr = (const unsigned char *) longword_ptr; + + /* At this point, we know that either n < sizeof (longword), or one of the + sizeof (longword) bytes starting at char_ptr is == c. On little-endian + machines, we could determine the first such byte without any further + memory accesses, just by looking at the tmp result from the last loop + iteration. But this does not work on big-endian machines. Choose code + that works in both cases. */ + + while (n-- > 0) + { + if (*--char_ptr == c) + return (void *) char_ptr; + } + + return NULL; +} +#endif diff --git a/apt-pkg/missing.h b/apt-pkg/missing.h new file mode 100644 index 000000000..441b47ce1 --- /dev/null +++ b/apt-pkg/missing.h @@ -0,0 +1,26 @@ +#define _GNU_SOURCE +#include <sys/socket.h> +#include <netinet/in.h> +#include <netdb.h> +#include <inttypes.h> +#include <errno.h> +#include <string.h> + +#ifndef PKGLIB_MISSING_H +#define PKGLIB_MISSING_H + +extern "C" { + void *memrchr(const void *s, int c, size_t n); + void *rawmemchr(const void *s, int c); + char *strchrnul(const char *s, int c); + int getservbyport_r(int port, const char *prots, struct servent *se, char *buf, size_t buflen, struct servent **res); +} + +typedef void (*sighandler_t)(int); + +extern char **environ; + +#define AI_IDN 0x0040 + +#endif + diff --git a/apt-pkg/nameser_compat.h b/apt-pkg/nameser_compat.h new file mode 100644 index 000000000..b2cf2ffaa --- /dev/null +++ b/apt-pkg/nameser_compat.h @@ -0,0 +1,187 @@ +/* Copyright (c) 1983, 1989 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/*% + * from nameser.h 8.1 (Berkeley) 6/2/93 + * $BINDId: nameser_compat.h,v 8.11 1999/01/02 08:00:58 vixie Exp $ + */ + +#ifndef _ARPA_NAMESER_COMPAT_ +#define _ARPA_NAMESER_COMPAT_ + +#define __BIND 19950621 /*%< (DEAD) interface version stamp. */ + +#include <apt-pkg/endian.h> + +/*% + * Structure for query header. The order of the fields is machine- and + * compiler-dependent, depending on the byte/bit order and the layout + * of bit fields. We use bit fields only in int variables, as this + * is all ANSI requires. This requires a somewhat confusing rearrangement. + */ + +typedef struct { + unsigned id :16; /*%< query identification number */ +#if BYTE_ORDER == BIG_ENDIAN + /* fields in third byte */ + unsigned qr: 1; /*%< response flag */ + unsigned opcode: 4; /*%< purpose of message */ + unsigned aa: 1; /*%< authoritive answer */ + unsigned tc: 1; /*%< truncated message */ + unsigned rd: 1; /*%< recursion desired */ + /* fields in fourth byte */ + unsigned ra: 1; /*%< recursion available */ + unsigned unused :1; /*%< unused bits (MBZ as of 4.9.3a3) */ + unsigned ad: 1; /*%< authentic data from named */ + unsigned cd: 1; /*%< checking disabled by resolver */ + unsigned rcode :4; /*%< response code */ +#endif +#if BYTE_ORDER == LITTLE_ENDIAN || BYTE_ORDER == PDP_ENDIAN + /* fields in third byte */ + unsigned rd :1; /*%< recursion desired */ + unsigned tc :1; /*%< truncated message */ + unsigned aa :1; /*%< authoritive answer */ + unsigned opcode :4; /*%< purpose of message */ + unsigned qr :1; /*%< response flag */ + /* fields in fourth byte */ + unsigned rcode :4; /*%< response code */ + unsigned cd: 1; /*%< checking disabled by resolver */ + unsigned ad: 1; /*%< authentic data from named */ + unsigned unused :1; /*%< unused bits (MBZ as of 4.9.3a3) */ + unsigned ra :1; /*%< recursion available */ +#endif + /* remaining bytes */ + unsigned qdcount :16; /*%< number of question entries */ + unsigned ancount :16; /*%< number of answer entries */ + unsigned nscount :16; /*%< number of authority entries */ + unsigned arcount :16; /*%< number of resource entries */ +} HEADER; + +#define PACKETSZ NS_PACKETSZ +#define MAXDNAME NS_MAXDNAME +#define MAXCDNAME NS_MAXCDNAME +#define MAXLABEL NS_MAXLABEL +#define HFIXEDSZ NS_HFIXEDSZ +#define QFIXEDSZ NS_QFIXEDSZ +#define RRFIXEDSZ NS_RRFIXEDSZ +#define INT32SZ NS_INT32SZ +#define INT16SZ NS_INT16SZ +#define INT8SZ NS_INT8SZ +#define INADDRSZ NS_INADDRSZ +#define IN6ADDRSZ NS_IN6ADDRSZ +#define INDIR_MASK NS_CMPRSFLGS +#define NAMESERVER_PORT NS_DEFAULTPORT + +#define S_ZONE ns_s_zn +#define S_PREREQ ns_s_pr +#define S_UPDATE ns_s_ud +#define S_ADDT ns_s_ar + +#define QUERY ns_o_query +#define IQUERY ns_o_iquery +#define STATUS ns_o_status +#define NS_NOTIFY_OP ns_o_notify +#define NS_UPDATE_OP ns_o_update + +#define NOERROR ns_r_noerror +#define FORMERR ns_r_formerr +#define SERVFAIL ns_r_servfail +#define NXDOMAIN ns_r_nxdomain +#define NOTIMP ns_r_notimpl +#define REFUSED ns_r_refused +#define YXDOMAIN ns_r_yxdomain +#define YXRRSET ns_r_yxrrset +#define NXRRSET ns_r_nxrrset +#define NOTAUTH ns_r_notauth +#define NOTZONE ns_r_notzone +/*#define BADSIG ns_r_badsig*/ +/*#define BADKEY ns_r_badkey*/ +/*#define BADTIME ns_r_badtime*/ + + +#define DELETE ns_uop_delete +#define ADD ns_uop_add + +#define T_A ns_t_a +#define T_NS ns_t_ns +#define T_MD ns_t_md +#define T_MF ns_t_mf +#define T_CNAME ns_t_cname +#define T_SOA ns_t_soa +#define T_MB ns_t_mb +#define T_MG ns_t_mg +#define T_MR ns_t_mr +#define T_NULL ns_t_null +#define T_WKS ns_t_wks +#define T_PTR ns_t_ptr +#define T_HINFO ns_t_hinfo +#define T_MINFO ns_t_minfo +#define T_MX ns_t_mx +#define T_TXT ns_t_txt +#define T_RP ns_t_rp +#define T_AFSDB ns_t_afsdb +#define T_X25 ns_t_x25 +#define T_ISDN ns_t_isdn +#define T_RT ns_t_rt +#define T_NSAP ns_t_nsap +#define T_NSAP_PTR ns_t_nsap_ptr +#define T_SIG ns_t_sig +#define T_KEY ns_t_key +#define T_PX ns_t_px +#define T_GPOS ns_t_gpos +#define T_AAAA ns_t_aaaa +#define T_LOC ns_t_loc +#define T_NXT ns_t_nxt +#define T_EID ns_t_eid +#define T_NIMLOC ns_t_nimloc +#define T_SRV ns_t_srv +#define T_ATMA ns_t_atma +#define T_NAPTR ns_t_naptr +#define T_A6 ns_t_a6 +#define T_DNAME ns_t_dname +#define T_TSIG ns_t_tsig +#define T_IXFR ns_t_ixfr +#define T_AXFR ns_t_axfr +#define T_MAILB ns_t_mailb +#define T_MAILA ns_t_maila +#define T_ANY ns_t_any + +#define C_IN ns_c_in +#define C_CHAOS ns_c_chaos +#define C_HS ns_c_hs +/* BIND_UPDATE */ +#define C_NONE ns_c_none +#define C_ANY ns_c_any + +#define GETSHORT NS_GET16 +#define GETLONG NS_GET32 +#define PUTSHORT NS_PUT16 +#define PUTLONG NS_PUT32 + +#endif /* _ARPA_NAMESER_COMPAT_ */ +/*! \file */ diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index c4bf34022..c5ed1b916 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -154,6 +154,7 @@ bool pkgCache::ReMap(bool const &Errorchecks) VerP = (Version *)Map.Data(); DescP = (Description *)Map.Data(); ProvideP = (Provides *)Map.Data(); + TagP = (Tag *)Map.Data(); DepP = (Dependency *)Map.Data(); DepDataP = (DependencyData *)Map.Data(); StrP = (char *)Map.Data(); diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 91228f713..83bf8961d 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -123,6 +123,7 @@ class pkgCache /*{{{*/ struct StringItem; struct VerFile; struct DescFile; + struct Tag; // Iterators template<typename Str, typename Itr> class Iterator; @@ -136,6 +137,7 @@ class pkgCache /*{{{*/ class PkgFileIterator; class VerFileIterator; class DescFileIterator; + class TagIterator; class Namespace; @@ -214,6 +216,7 @@ class pkgCache /*{{{*/ ReleaseFile *RlsFileP; PackageFile *PkgFileP; Version *VerP; + Tag *TagP; Description *DescP; Provides *ProvideP; Dependency *DepP; @@ -320,6 +323,7 @@ struct pkgCache::Header map_number_t ReleaseFileSz; map_number_t PackageFileSz; map_number_t VersionSz; + map_number_t TagSz; map_number_t DescriptionSz; map_number_t DependencySz; map_number_t DependencyDataSz; @@ -335,6 +339,7 @@ struct pkgCache::Header map_id_t GroupCount; map_id_t PackageCount; map_id_t VersionCount; + map_id_t TagCount; map_id_t DescriptionCount; map_id_t DependsCount; map_id_t DependsDataCount; @@ -585,6 +590,16 @@ struct pkgCache::VerFile map_filesize_t Size; }; /*}}}*/ +// TagFile structure /*{{{*/ +/** \brief associates a tag with something */ +struct pkgCache::Tag +{ + /** \brief name of this tag */ + map_stringitem_t Name; + /** \brief next step in the linked list */ + map_pointer_t NextTag; // Tag +}; + /*}}}*/ // DescFile structure /*{{{*/ /** \brief associates a description with a Translation file */ struct pkgCache::DescFile @@ -612,6 +627,8 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; + /** \brief high-level name used to display package */ + map_stringitem_t Display; /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ map_stringitem_t SourcePkgName; @@ -656,6 +673,8 @@ struct pkgCache::Version map_pointer_t ParentPkg; // Package /** \brief list of pkgCache::Provides */ map_pointer_t ProvidesList; // Provides + /** \brief list of pkgCache::Tag */ + map_pointer_t TagList; // Tag /** \brief archive size for this version @@ -808,6 +827,7 @@ class pkgCache::Namespace /*{{{*/ typedef pkgCache::GrpIterator GrpIterator; typedef pkgCache::PkgIterator PkgIterator; typedef pkgCache::VerIterator VerIterator; + typedef pkgCache::TagIterator TagIterator; typedef pkgCache::DescIterator DescIterator; typedef pkgCache::DepIterator DepIterator; typedef pkgCache::PrvIterator PrvIterator; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index e52667fbc..678b5f660 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -70,7 +70,7 @@ bool pkgCacheGenerator::Start() bool const newError = _error->PendingError(); _error->MergeWithStack(); if (newError) - return false; + return _error->ReturnError(); if (Map.Size() <= 0) return false; @@ -134,7 +134,7 @@ bool pkgCacheGenerator::Start() advoid a problem during a crash */ pkgCacheGenerator::~pkgCacheGenerator() { - if (_error->PendingError() == true || Map.validData() == false) + if (Map.validData() == false) return; if (Map.Sync() == false) return; @@ -175,6 +175,10 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i) if (std::get<1>(seen.insert(*i)) == true) (*i)->ReMap(oldMap, newMap); + for (std::vector<pkgCache::TagIterator*>::const_iterator i = Dynamic<pkgCache::TagIterator>::toReMap.begin(); + i != Dynamic<pkgCache::TagIterator>::toReMap.end(); ++i) + if (std::get<1>(seen.insert(*i)) == true) + (*i)->ReMap(oldMap, newMap); for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin(); i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i) if (std::get<1>(seen.insert(*i)) == true) @@ -250,7 +254,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, { string const PackageName = List.Package(); if (PackageName.empty() == true) - return false; + continue; Counter++; if (Counter % 100 == 0 && Progress != 0) @@ -264,24 +268,26 @@ bool pkgCacheGenerator::MergeList(ListParser &List, { // package descriptions if (MergeListGroup(List, PackageName) == false) - return false; + continue; continue; } // Get a pointer to the package structure pkgCache::PkgIterator Pkg; Dynamic<pkgCache::PkgIterator> DynPkg(Pkg); - if (NewPackage(Pkg, PackageName, Arch) == false) + if (NewPackage(Pkg, PackageName, Arch) == false) { // TRANSLATOR: The first placeholder is a package name, // the other two should be copied verbatim as they include debug info - return _error->Error(_("Error occurred while processing %s (%s%d)"), + _error->Error(_("Error occurred while processing %s (%s%d)"), PackageName.c_str(), "NewPackage", 1); + continue; + } if (Version.empty() == true) { if (MergeListPackage(List, Pkg) == false) - return false; + continue; } else { @@ -1274,6 +1280,38 @@ bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site, return true; } /*}}}*/ +// ListParser::NewTag - Create a Tag element /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver, + const char *NameStart, + unsigned int NameSize) +{ + return Owner->NewTag(Ver, NameStart, NameSize); +} +bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver, + const char *NameStart, + unsigned int NameSize) +{ + // Get a structure + map_pointer_t const idxTag = AllocateInMap(sizeof(pkgCache::Tag)); + if (unlikely(idxTag == 0)) + return false; + + // Fill it in + pkgCache::TagIterator Tg(Cache,Cache.TagP + idxTag); + map_pointer_t const idxName = StoreString(TAG,NameStart,NameSize); + if (idxName == 0) + return false; + Tg->Name = idxName; + + Tg->NextTag = Ver->TagList; + Ver->TagList = Tg.Index(); + Cache.HeaderP->TagCount++; + + return true; +} + /*}}}*/ // CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/ // --------------------------------------------------------------------- /* This is used to select which file is to be associated with all newly @@ -1341,6 +1379,7 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons case PKGNAME: strings = &strPkgNames; break; case VERSIONNUMBER: strings = &strVersions; break; case SECTION: strings = &strSections; break; + case TAG: strings = &strTags; break; default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0; } @@ -1480,7 +1519,7 @@ static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName, std::clog << "Validity failed because of pending errors:" << std::endl; _error->DumpErrors(std::clog, GlobalError::DEBUG, false); } - return false; + return _error->ReturnError(); } if (OutMap != 0) @@ -1518,16 +1557,14 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator } /*}}}*/ // BuildCache - Merge the list of index files into the cache /*{{{*/ -static bool BuildCache(pkgCacheGenerator &Gen, +static void BuildCache(pkgCacheGenerator &Gen, OpProgress * const Progress, map_filesize_t &CurrentSize,map_filesize_t TotalSize, pkgSourceList const * const List, FileIterator const Start, FileIterator const End) { - bool mergeFailure = false; - auto const indexFileMerge = [&](pkgIndexFile * const I) { - if (I->HasPackages() == false || mergeFailure) + if (I->HasPackages() == false) return; if (I->Exists() == false) @@ -1545,8 +1582,10 @@ static bool BuildCache(pkgCacheGenerator &Gen, Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists")); CurrentSize += Size; - if (I->Merge(Gen,Progress) == false) - mergeFailure = true; + if (I->Merge(Gen,Progress) == false) { + _error->ReturnError(); + return; + } }; if (List != NULL) @@ -1560,14 +1599,14 @@ static bool BuildCache(pkgCacheGenerator &Gen, continue; } - if ((*i)->Merge(Gen, Progress) == false) - return false; + if ((*i)->Merge(Gen, Progress) == false) { + _error->ReturnError(); + continue; + } std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles(); if (Indexes != NULL) std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge); - if (mergeFailure) - return false; } } @@ -1575,10 +1614,7 @@ static bool BuildCache(pkgCacheGenerator &Gen, { Gen.SelectReleaseFile("", ""); std::for_each(Start, End, indexFileMerge); - if (mergeFailure) - return false; } - return true; } /*}}}*/ // CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ @@ -1637,7 +1673,7 @@ static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen, bool const newError = _error->PendingError(); _error->MergeWithStack(); if (alloc == 0 && newError) - return false; + return _error->ReturnError(); if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false) return false; Gen.reset(new pkgCacheGenerator(Map.get(),Progress)); @@ -1753,9 +1789,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress return false; TotalSize += ComputeSize(&List, Files.begin(),Files.end()); - if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List, - Files.end(),Files.end()) == false) - return false; + BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List, + Files.end(),Files.end()); if (Writeable == true && SrcCacheFileName.empty() == false) if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFileName) == false) @@ -1766,9 +1801,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress { if (Debug == true) std::clog << "Building status cache in pkgcache.bin now" << std::endl; - if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(), Files.end()) == false) - return false; + BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()); if (Writeable == true && CacheFileName.empty() == false) if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFileName) == false) @@ -1789,9 +1823,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress } Files = List.GetVolatileFiles(); - if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(), Files.end()) == false) - return false; + BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()); } if (OutMap != nullptr) @@ -1828,14 +1861,15 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (Progress != NULL) Progress->OverallProgress(0,1,1,_("Reading package lists")); pkgCacheGenerator Gen(Map.get(),Progress); - if (Gen.Start() == false || _error->PendingError() == true) - return false; - if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL, - Files.begin(), Files.end()) == false) + if (Gen.Start() == false) return false; - if (_error->PendingError() == true) - return false; + return _error->ReturnError(); + BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL, + Files.begin(), Files.end()); + // We've passed the point of no return + _error->ReturnError(); + *OutMap = Map.release(); return true; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index cb51c113a..27fd8d4d0 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -78,6 +78,7 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ std::unordered_set<string_pointer, hash> strPkgNames; std::unordered_set<string_pointer, hash> strVersions; std::unordered_set<string_pointer, hash> strSections; + std::unordered_set<string_pointer, hash> strTags; #endif friend class pkgCacheListParser; @@ -128,10 +129,11 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ uint8_t const Type, map_pointer_t* &OldDepLast); bool NewProvides(pkgCache::VerIterator &Ver, pkgCache::PkgIterator &Pkg, map_stringitem_t const ProvidesVersion, uint8_t const Flags); + bool NewTag(pkgCache::VerIterator &Ver,const char *NameStart,unsigned int NameSize); public: - enum StringType { MIXED, PKGNAME, VERSIONNUMBER, SECTION }; + enum StringType { MIXED, PKGNAME, VERSIONNUMBER, SECTION, TAG }; map_stringitem_t StoreString(StringType const type, const char * S, unsigned int const Size); #ifdef APT_PKG_EXPOSE_STRING_VIEW @@ -209,6 +211,7 @@ class APT_HIDDEN pkgCacheListParser uint8_t const Flags); bool NewProvidesAllArch(pkgCache::VerIterator &Ver, APT::StringView Package, APT::StringView Version, uint8_t const Flags); + bool NewTag(pkgCache::VerIterator &Ver,const char *NameStart,unsigned int NameSize); #endif public: diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index 7c50c5d41..5d4ba37d7 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -92,6 +92,7 @@ class pkgRecords::Parser /*{{{*/ std::string LongDesc() {return LongDesc("");}; virtual std::string Name() {return std::string();}; + virtual std::string Display() {return std::string();} virtual std::string Homepage() {return std::string();} // An arbitrary custom field @@ -100,6 +101,9 @@ class pkgRecords::Parser /*{{{*/ // The record in binary form virtual void GetRec(const char *&Start,const char *&Stop) {Start = Stop = 0;}; + // Locate a tag + virtual bool Find(const char *Tag,const char *&Start, const char *&End) {Start = End = 0; return false;}; + Parser(); virtual ~Parser(); diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc index 3dd6ddac4..a13510e66 100644 --- a/apt-pkg/policy.cc +++ b/apt-pkg/policy.cc @@ -320,7 +320,7 @@ bool ReadPinDir(pkgPolicy &Plcy,string Dir) bool const PendingErrors = _error->PendingError(); _error->MergeWithStack(); if (PendingErrors) - return false; + return _error->ReturnError(); // Read the files for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I) @@ -391,6 +391,7 @@ bool ReadPinFile(pkgPolicy &Plcy,string File) if (priority < std::numeric_limits<short>::min() || priority > std::numeric_limits<short>::max() || newError) { + _error->ReturnError(); return _error->Error(_("%s: Value %s is outside the range of valid pin priorities (%d to %d)"), File.c_str(), Tags.FindS("Pin-Priority").c_str(), std::numeric_limits<short>::min(), diff --git a/apt-pkg/rawmemchr.cc b/apt-pkg/rawmemchr.cc new file mode 100644 index 000000000..8f7669b6c --- /dev/null +++ b/apt-pkg/rawmemchr.cc @@ -0,0 +1,139 @@ +/* Searching in a string. + Copyright (C) 2008-2015 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ + +#include <config.h> + +#ifndef HAVE_RAWMEMCHR + +/* Specification. */ +#include <string.h> + +/* Find the first occurrence of C in S. */ +extern "C" void * +rawmemchr (const void *s, int c_in) +{ + /* On 32-bit hardware, choosing longword to be a 32-bit unsigned + long instead of a 64-bit uintmax_t tends to give better + performance. On 64-bit hardware, unsigned long is generally 64 + bits already. Change this typedef to experiment with + performance. */ + typedef unsigned long int longword; + + const unsigned char *char_ptr; + const longword *longword_ptr; + longword repeated_one; + longword repeated_c; + unsigned char c; + + c = (unsigned char) c_in; + + /* Handle the first few bytes by reading one byte at a time. + Do this until CHAR_PTR is aligned on a longword boundary. */ + for (char_ptr = (const unsigned char *) s; + (size_t) char_ptr % sizeof (longword) != 0; + ++char_ptr) + if (*char_ptr == c) + return (void *) char_ptr; + + longword_ptr = (const longword *) char_ptr; + + /* All these elucidatory comments refer to 4-byte longwords, + but the theory applies equally well to any size longwords. */ + + /* Compute auxiliary longword values: + repeated_one is a value which has a 1 in every byte. + repeated_c has c in every byte. */ + repeated_one = 0x01010101; + repeated_c = c | (c << 8); + repeated_c |= repeated_c << 16; + if (0xffffffffU < (longword) -1) + { + repeated_one |= repeated_one << 31 << 1; + repeated_c |= repeated_c << 31 << 1; + if (8 < sizeof (longword)) + { + size_t i; + + for (i = 64; i < sizeof (longword) * 8; i *= 2) + { + repeated_one |= repeated_one << i; + repeated_c |= repeated_c << i; + } + } + } + + /* Instead of the traditional loop which tests each byte, we will + test a longword at a time. The tricky part is testing if *any of + the four* bytes in the longword in question are equal to NUL or + c. We first use an xor with repeated_c. This reduces the task + to testing whether *any of the four* bytes in longword1 is zero. + + We compute tmp = + ((longword1 - repeated_one) & ~longword1) & (repeated_one << 7). + That is, we perform the following operations: + 1. Subtract repeated_one. + 2. & ~longword1. + 3. & a mask consisting of 0x80 in every byte. + Consider what happens in each byte: + - If a byte of longword1 is zero, step 1 and 2 transform it into 0xff, + and step 3 transforms it into 0x80. A carry can also be propagated + to more significant bytes. + - If a byte of longword1 is nonzero, let its lowest 1 bit be at + position k (0 <= k <= 7); so the lowest k bits are 0. After step 1, + the byte ends in a single bit of value 0 and k bits of value 1. + After step 2, the result is just k bits of value 1: 2^k - 1. After + step 3, the result is 0. And no carry is produced. + So, if longword1 has only non-zero bytes, tmp is zero. + Whereas if longword1 has a zero byte, call j the position of the least + significant zero byte. Then the result has a zero at positions 0, ..., + j-1 and a 0x80 at position j. We cannot predict the result at the more + significant bytes (positions j+1..3), but it does not matter since we + already have a non-zero bit at position 8*j+7. + + The test whether any byte in longword1 is zero is equivalent + to testing whether tmp is nonzero. + + This test can read beyond the end of a string, depending on where + C_IN is encountered. However, this is considered safe since the + initialization phase ensured that the read will be aligned, + therefore, the read will not cross page boundaries and will not + cause a fault. */ + + while (1) + { + longword longword1 = *longword_ptr ^ repeated_c; + + if ((((longword1 - repeated_one) & ~longword1) + & (repeated_one << 7)) != 0) + break; + longword_ptr++; + } + + char_ptr = (const unsigned char *) longword_ptr; + + /* At this point, we know that one of the sizeof (longword) bytes + starting at char_ptr is == c. On little-endian machines, we + could determine the first such byte without any further memory + accesses, just by looking at the tmp result from the last loop + iteration. But this does not work on big-endian machines. + Choose code that works in both cases. */ + + char_ptr = (unsigned char *) longword_ptr; + while (*char_ptr != c) + char_ptr++; + return (void *) char_ptr; +} +#endif diff --git a/apt-pkg/strchrnul.cc b/apt-pkg/strchrnul.cc new file mode 100644 index 000000000..b68b8501b --- /dev/null +++ b/apt-pkg/strchrnul.cc @@ -0,0 +1,147 @@ +/* Searching in a string. + Copyright (C) 2003, 2007-2015 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ + +#include <config.h> + +#ifndef HAVE_STRCHRNUL + +/* Specification. */ +#include <string.h> + +#include <apt-pkg/missing.h> + +/* Find the first occurrence of C in S or the final NUL byte. */ +extern "C" char * +strchrnul (const char *s, int c_in) +{ + /* On 32-bit hardware, choosing longword to be a 32-bit unsigned + long instead of a 64-bit uintmax_t tends to give better + performance. On 64-bit hardware, unsigned long is generally 64 + bits already. Change this typedef to experiment with + performance. */ + typedef unsigned long int longword; + + const unsigned char *char_ptr; + const longword *longword_ptr; + longword repeated_one; + longword repeated_c; + unsigned char c; + + c = (unsigned char) c_in; + if (!c) + return (char*) rawmemchr (s, 0); + + /* Handle the first few bytes by reading one byte at a time. + Do this until CHAR_PTR is aligned on a longword boundary. */ + for (char_ptr = (const unsigned char *) s; + (size_t) char_ptr % sizeof (longword) != 0; + ++char_ptr) + if (!*char_ptr || *char_ptr == c) + return (char *) char_ptr; + + longword_ptr = (const longword *) char_ptr; + + /* All these elucidatory comments refer to 4-byte longwords, + but the theory applies equally well to any size longwords. */ + + /* Compute auxiliary longword values: + repeated_one is a value which has a 1 in every byte. + repeated_c has c in every byte. */ + repeated_one = 0x01010101; + repeated_c = c | (c << 8); + repeated_c |= repeated_c << 16; + if (0xffffffffU < (longword) -1) + { + repeated_one |= repeated_one << 31 << 1; + repeated_c |= repeated_c << 31 << 1; + if (8 < sizeof (longword)) + { + size_t i; + + for (i = 64; i < sizeof (longword) * 8; i *= 2) + { + repeated_one |= repeated_one << i; + repeated_c |= repeated_c << i; + } + } + } + + /* Instead of the traditional loop which tests each byte, we will + test a longword at a time. The tricky part is testing if *any of + the four* bytes in the longword in question are equal to NUL or + c. We first use an xor with repeated_c. This reduces the task + to testing whether *any of the four* bytes in longword1 or + longword2 is zero. + + Let's consider longword1. We compute tmp = + ((longword1 - repeated_one) & ~longword1) & (repeated_one << 7). + That is, we perform the following operations: + 1. Subtract repeated_one. + 2. & ~longword1. + 3. & a mask consisting of 0x80 in every byte. + Consider what happens in each byte: + - If a byte of longword1 is zero, step 1 and 2 transform it into 0xff, + and step 3 transforms it into 0x80. A carry can also be propagated + to more significant bytes. + - If a byte of longword1 is nonzero, let its lowest 1 bit be at + position k (0 <= k <= 7); so the lowest k bits are 0. After step 1, + the byte ends in a single bit of value 0 and k bits of value 1. + After step 2, the result is just k bits of value 1: 2^k - 1. After + step 3, the result is 0. And no carry is produced. + So, if longword1 has only non-zero bytes, tmp is zero. + Whereas if longword1 has a zero byte, call j the position of the least + significant zero byte. Then the result has a zero at positions 0, ..., + j-1 and a 0x80 at position j. We cannot predict the result at the more + significant bytes (positions j+1..3), but it does not matter since we + already have a non-zero bit at position 8*j+7. + + The test whether any byte in longword1 or longword2 is zero is equivalent + to testing whether tmp1 is nonzero or tmp2 is nonzero. We can combine + this into a single test, whether (tmp1 | tmp2) is nonzero. + + This test can read more than one byte beyond the end of a string, + depending on where the terminating NUL is encountered. However, + this is considered safe since the initialization phase ensured + that the read will be aligned, therefore, the read will not cross + page boundaries and will not cause a fault. */ + + while (1) + { + longword longword1 = *longword_ptr ^ repeated_c; + longword longword2 = *longword_ptr; + + if (((((longword1 - repeated_one) & ~longword1) + | ((longword2 - repeated_one) & ~longword2)) + & (repeated_one << 7)) != 0) + break; + longword_ptr++; + } + + char_ptr = (const unsigned char *) longword_ptr; + + /* At this point, we know that one of the sizeof (longword) bytes + starting at char_ptr is == 0 or == c. On little-endian machines, + we could determine the first such byte without any further memory + accesses, just by looking at the tmp result from the last loop + iteration. But this does not work on big-endian machines. + Choose code that works in both cases. */ + + char_ptr = (unsigned char *) longword_ptr; + while (*char_ptr && (*char_ptr != c)) + char_ptr++; + return (char *) char_ptr; +} +#endif diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index d9519175e..943575956 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -13,6 +13,7 @@ // Include Files /*{{{*/ #include<config.h> +#include <apt-pkg/mmap.h> #include <apt-pkg/tagfile.h> #include <apt-pkg/tagfile-keys.h> #include <apt-pkg/error.h> @@ -37,10 +38,11 @@ using APT::StringView; class APT_HIDDEN pkgTagFilePrivate /*{{{*/ { public: - void Reset(FileFd * const pFd, unsigned long long const pSize, pkgTagFile::Flags const pFlags) + void Reset(FileFd * const pFd, pkgTagFile::Flags const pFlags) { - if (Buffer != NULL) - free(Buffer); + if (Map != NULL) + delete Map; + Map = NULL; Buffer = NULL; Fd = pFd; Flags = pFlags; @@ -48,14 +50,11 @@ public: End = NULL; Done = false; iOffset = 0; - Size = pSize; - isCommentedLine = false; - chunks.clear(); } - pkgTagFilePrivate(FileFd * const pFd, unsigned long long const Size, pkgTagFile::Flags const pFlags) : Buffer(NULL) + pkgTagFilePrivate(FileFd * const pFd, pkgTagFile::Flags const pFlags) : Map(NULL) { - Reset(pFd, Size, pFlags); + Reset(pFd, pFlags); } FileFd * Fd; pkgTagFile::Flags Flags; @@ -64,20 +63,12 @@ public: char *End; bool Done; unsigned long long iOffset; - unsigned long long Size; - bool isCommentedLine; - struct FileChunk - { - bool const good; - size_t length; - FileChunk(bool const pgood, size_t const plength) : good(pgood), length(plength) {} - }; - std::list<FileChunk> chunks; + MMap *Map; ~pkgTagFilePrivate() { - if (Buffer != NULL) - free(Buffer); + if (Map != NULL) + delete Map; } }; /*}}}*/ @@ -116,42 +107,38 @@ static unsigned long BetaHash(const char *Text, size_t Length) /*{{{*/ /*}}}*/ // TagFile::pkgTagFile - Constructor /*{{{*/ -pkgTagFile::pkgTagFile(FileFd * const pFd,pkgTagFile::Flags const pFlags, unsigned long long const Size) - : d(new pkgTagFilePrivate(pFd, Size + 4, pFlags)) +pkgTagFile::pkgTagFile(FileFd * const pFd,pkgTagFile::Flags const pFlags) + : d(new pkgTagFilePrivate(pFd, pFlags)) { - Init(pFd, pFlags, Size); + Init(pFd, pFlags); } -pkgTagFile::pkgTagFile(FileFd * const pFd,unsigned long long const Size) - : pkgTagFile(pFd, pkgTagFile::STRICT, Size) +pkgTagFile::pkgTagFile(FileFd * const pFd) + : pkgTagFile(pFd, pkgTagFile::STRICT) { } -void pkgTagFile::Init(FileFd * const pFd, pkgTagFile::Flags const pFlags, unsigned long long Size) +void pkgTagFile::Init(FileFd * const pFd, pkgTagFile::Flags const pFlags) { - /* The size is increased by 4 because if we start with the Size of the - filename we need to try to read 1 char more to see an EOF faster, 1 - char the end-pointer can be on and maybe 2 newlines need to be added - to the end of the file -> 4 extra chars */ - Size += 4; - d->Reset(pFd, Size, pFlags); - - if (d->Fd->IsOpen() == false) - d->Start = d->End = d->Buffer = 0; - else - d->Buffer = (char*)malloc(sizeof(char) * Size); + d->Reset(pFd, pFlags); + + if (d->Fd->IsOpen() == false || d->Fd->Size() == 0) + _error->Discard(); + else { + d->Map = new MMap(*d->Fd, MMap::ReadOnly); + d->Buffer = static_cast<char *>(d->Map->Data()); + } if (d->Buffer == NULL) d->Done = true; - else + else { d->Done = false; + d->End = d->Buffer + d->Map->Size(); + } - d->Start = d->End = d->Buffer; - d->iOffset = 0; - if (d->Done == false) - Fill(); + d->Start = d->Buffer; } -void pkgTagFile::Init(FileFd * const pFd,unsigned long long Size) +void pkgTagFile::Init(FileFd * const pFd) { - Init(pFd, pkgTagFile::STRICT, Size); + Init(pFd, pkgTagFile::STRICT); } /*}}}*/ // TagFile::~pkgTagFile - Destructor /*{{{*/ @@ -166,36 +153,6 @@ APT_PURE unsigned long pkgTagFile::Offset() return d->iOffset; } /*}}}*/ -// TagFile::Resize - Resize the internal buffer /*{{{*/ -// --------------------------------------------------------------------- -/* Resize the internal buffer (double it in size). Fail if a maximum size - * size is reached. - */ -bool pkgTagFile::Resize() -{ - // fail is the buffer grows too big - if(d->Size > 1024*1024+1) - return false; - - return Resize(d->Size * 2); -} -bool pkgTagFile::Resize(unsigned long long const newSize) -{ - unsigned long long const EndSize = d->End - d->Start; - - // get new buffer and use it - char* const newBuffer = static_cast<char*>(realloc(d->Buffer, sizeof(char) * newSize)); - if (newBuffer == NULL) - return false; - d->Buffer = newBuffer; - d->Size = newSize; - - // update the start/end pointers to the new buffer - d->Start = d->Buffer; - d->End = d->Start + EndSize; - return true; -} - /*}}}*/ // TagFile::Step - Advance to the next section /*{{{*/ // --------------------------------------------------------------------- /* If the Section Scanner fails we refill the buffer and try again. @@ -204,221 +161,18 @@ bool pkgTagFile::Resize(unsigned long long const newSize) */ bool pkgTagFile::Step(pkgTagSection &Tag) { - if(Tag.Scan(d->Start,d->End - d->Start) == false) + if(Tag.Scan(d->Start,d->End - d->Start,(d->Flags & SUPPORT_COMMENTS) != 0) == false) { - do - { - if (Fill() == false) - return false; - - if(Tag.Scan(d->Start,d->End - d->Start, false)) - break; - - if (Resize() == false) - return _error->Error(_("Unable to parse package file %s (%d)"), + if (d->Start == d->End) + return false; + else + return _error->Warning(_("Unable to parse package file %s (%d)"), d->Fd->Name().c_str(), 1); - - } while (Tag.Scan(d->Start,d->End - d->Start, false) == false); } size_t tagSize = Tag.size(); d->Start += tagSize; - - if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0) - d->iOffset += tagSize; - else - { - auto first = d->chunks.begin(); - for (; first != d->chunks.end(); ++first) - { - if (first->good == false) - d->iOffset += first->length; - else - { - if (tagSize < first->length) - { - first->length -= tagSize; - d->iOffset += tagSize; - break; - } - else - { - tagSize -= first->length; - d->iOffset += first->length; - } - } - } - d->chunks.erase(d->chunks.begin(), first); - } - - Tag.Trim(); - return true; -} - /*}}}*/ -// TagFile::Fill - Top up the buffer /*{{{*/ -// --------------------------------------------------------------------- -/* This takes the bit at the end of the buffer and puts it at the start - then fills the rest from the file */ -static bool FillBuffer(pkgTagFilePrivate * const d) -{ - unsigned long long Actual = 0; - // See if only a bit of the file is left - unsigned long long const dataSize = d->Size - ((d->End - d->Buffer) + 1); - if (d->Fd->Read(d->End, dataSize, &Actual) == false) - return false; - if (Actual != dataSize) - d->Done = true; - d->End += Actual; - return true; -} -static void RemoveCommentsFromBuffer(pkgTagFilePrivate * const d) -{ - // look for valid comments in the buffer - char * good_start = nullptr, * bad_start = nullptr; - char * current = d->Start; - if (d->isCommentedLine == false) - { - if (d->Start == d->Buffer) - { - // the start of the buffer is a newline as a record can't start - // in the middle of a line by definition. - if (*d->Start == '#') - { - d->isCommentedLine = true; - ++current; - if (current > d->End) - d->chunks.emplace_back(false, 1); - } - } - if (d->isCommentedLine == false) - good_start = d->Start; - else - bad_start = d->Start; - } - else - bad_start = d->Start; - - std::vector<std::pair<char*, size_t>> good_parts; - while (current <= d->End) - { - size_t const restLength = (d->End - current); - if (d->isCommentedLine == false) - { - current = static_cast<char*>(memchr(current, '#', restLength)); - if (current == nullptr) - { - size_t const goodLength = d->End - good_start; - d->chunks.emplace_back(true, goodLength); - if (good_start != d->Start) - good_parts.push_back(std::make_pair(good_start, goodLength)); - break; - } - bad_start = current; - --current; - // ensure that this is really a comment and not a '#' in the middle of a line - if (*current == '\n') - { - size_t const goodLength = (current - good_start) + 1; - d->chunks.emplace_back(true, goodLength); - good_parts.push_back(std::make_pair(good_start, goodLength)); - good_start = nullptr; - d->isCommentedLine = true; - } - current += 2; - } - else // the current line is a comment - { - current = static_cast<char*>(memchr(current, '\n', restLength)); - if (current == nullptr) - { - d->chunks.emplace_back(false, (d->End - bad_start)); - break; - } - ++current; - // is the next line a comment, too? - if (current >= d->End || *current != '#') - { - d->chunks.emplace_back(false, (current - bad_start)); - good_start = current; - bad_start = nullptr; - d->isCommentedLine = false; - } - ++current; - } - } - - if (good_parts.empty() == false) - { - // we found comments, so move later parts over them - current = d->Start; - for (auto const &good: good_parts) - { - memmove(current, good.first, good.second); - current += good.second; - } - d->End = current; - } - - if (d->isCommentedLine == true) - { - // deal with a buffer containing only comments - // or an (unfinished) comment at the end - if (good_parts.empty() == true) - d->End = d->Start; - else - d->Start = d->End; - } - else - { - // the buffer was all comment, but ended with the buffer - if (good_parts.empty() == true && good_start >= d->End) - d->End = d->Start; - else - d->Start = d->End; - } -} -bool pkgTagFile::Fill() -{ - unsigned long long const EndSize = d->End - d->Start; - if (EndSize != 0) - { - memmove(d->Buffer,d->Start,EndSize); - d->Start = d->End = d->Buffer + EndSize; - } - else - d->Start = d->End = d->Buffer; - - unsigned long long Actual = 0; - while (d->Done == false && d->Size > (Actual + 1)) - { - if (FillBuffer(d) == false) - return false; - if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) != 0) - RemoveCommentsFromBuffer(d); - Actual = d->End - d->Buffer; - } - d->Start = d->Buffer; - - if (d->Done == true) - { - if (EndSize <= 3 && Actual == 0) - return false; - if (d->Size - (d->End - d->Buffer) < 4) - return true; - - // Append a double new line if one does not exist - unsigned int LineCount = 0; - for (const char *E = d->End - 1; E - d->End < 6 && (*E == '\n' || *E == '\r'); E--) - if (*E == '\n') - ++LineCount; - if (LineCount < 2) - { - if (static_cast<unsigned long long>(d->End - d->Buffer) >= d->Size) - Resize(d->Size + 3); - for (; LineCount < 2; ++LineCount) - *d->End++ = '\n'; - } - } + d->iOffset += tagSize; return true; } /*}}}*/ @@ -428,43 +182,16 @@ bool pkgTagFile::Fill() that is there */ bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset) { - if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0 && - // We are within a buffer space of the next hit.. - Offset >= d->iOffset && d->iOffset + (d->End - d->Start) > Offset) - { - unsigned long long Dist = Offset - d->iOffset; - d->Start += Dist; - d->iOffset += Dist; - // if we have seen the end, don't ask for more - if (d->Done == true) - return Tag.Scan(d->Start, d->End - d->Start); - else - return Step(Tag); - } + unsigned int Size(d->Map->Size()); + if (Offset >= Size) + return false; // Reposition and reload.. d->iOffset = Offset; d->Done = false; - if (d->Fd->Seek(Offset) == false) - return false; - d->End = d->Start = d->Buffer; - d->isCommentedLine = false; - d->chunks.clear(); - - if (Fill() == false) - return false; + d->Start = d->Buffer + d->iOffset; - if (Tag.Scan(d->Start, d->End - d->Start) == true) - return true; - - // This appends a double new line (for the real eof handling) - if (Fill() == false) - return false; - - if (Tag.Scan(d->Start, d->End - d->Start, false) == false) - return _error->Error(_("Unable to parse package file %s (%d)"),d->Fd->Name().c_str(), 2); - - return true; + return Step(Tag); } /*}}}*/ // pkgTagSection::pkgTagSection - Constructor /*{{{*/ @@ -480,32 +207,20 @@ pkgTagSection::pkgTagSection() APT_IGNORE_DEPRECATED_POP /*}}}*/ // TagSection::Scan - Scan for the end of the header information /*{{{*/ -bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const Restart) +bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength,bool const SupportComments) { Section = Start; const char *End = Start + MaxLength; - if (Restart == false && d->Tags.empty() == false) + Stop = Section; + if (d->Tags.empty() == false) { - Stop = Section + d->Tags.back().StartTag; - if (End <= Stop) - return false; - Stop = (const char *)memchr(Stop,'\n',End - Stop); - if (Stop == NULL) - return false; - ++Stop; - } - else - { - Stop = Section; - if (d->Tags.empty() == false) - { - memset(&AlphaIndexes, 0, sizeof(AlphaIndexes)); - memset(&BetaIndexes, 0, sizeof(BetaIndexes)); - d->Tags.clear(); - } - d->Tags.reserve(0x100); + memset(&AlphaIndexes, 0, sizeof(AlphaIndexes)); + memset(&BetaIndexes, 0, sizeof(BetaIndexes)); + d->Tags.clear(); } + d->Tags.reserve(0x100); + unsigned int TagCount = d->Tags.size(); if (Stop == 0) @@ -517,12 +232,12 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R unsigned int lastTagHash = 0; while (Stop < End) { - TrimRecord(true,End); + TrimRecord(true,End,SupportComments); // this can happen when TrimRecord trims away the entire Record // (e.g. because it just contains comments) if(Stop == End) - return true; + goto end; // Start a new index and add it to the hash if (isspace_ascii(Stop[0]) == 0) @@ -561,17 +276,17 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R // find the beginning of the value Stop = Colon + 1; for (; Stop < End && isspace_ascii(*Stop) != 0; ++Stop) - if (*Stop == '\n' && Stop[1] != ' ') + if (*Stop == '\n' && (Stop+1 == End || Stop[1] != ' ')) break; - if (Stop >= End) - return false; lastTagData.StartValue = Stop - Section; } Stop = (const char *)memchr(Stop,'\n',End - Stop); - if (Stop == 0) - return false; + if (Stop == 0) { + Stop = End; + goto end; + } for (; Stop+1 < End && Stop[1] == '\r'; Stop++) /* nothing */ @@ -579,7 +294,7 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R // Double newline marks the end of the record if (Stop+1 < End && Stop[1] == '\n') - { + end: { if (lastTagData.EndTag != 0) { if (lastTagKey != Key::Unknown) { @@ -592,26 +307,32 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R d->Tags.push_back(lastTagData); } + if (d->Tags.empty()) + return false; + pkgTagSectionPrivate::TagData const td(Stop - Section); d->Tags.push_back(td); - TrimRecord(false,End); + TrimRecord(false,End,SupportComments); return true; } Stop++; } - return false; + goto end; } /*}}}*/ // TagSection::TrimRecord - Trim off any garbage before/after a record /*{{{*/ // --------------------------------------------------------------------- /* There should be exactly 2 newline at the end of the record, no more. */ -void pkgTagSection::TrimRecord(bool BeforeRecord, const char*& End) -{ - if (BeforeRecord == true) - return; - for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r'); Stop++); +void pkgTagSection::TrimRecord(bool BeforeRecord, const char*& End, bool SupportComments) +{ trim: + if (BeforeRecord == false) + for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r'); Stop++); + if (SupportComments && Stop < End && Stop[0] == '#') { + Stop = (const char*) memchr(Stop,'\n',End-Stop) ?: End; + goto trim; + } } /*}}}*/ // TagSection::Trim - Trim off any trailing garbage /*{{{*/ @@ -621,7 +342,7 @@ void pkgTagSection::Trim() { for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--); } - /*}}}*/ + // TagSection::Exists - return True if a tag exists /*{{{*/ bool pkgTagSection::Exists(StringView Tag) const { @@ -676,7 +397,7 @@ bool pkgTagSection::FindInternal(unsigned int Pos, const char *&Start, if (unlikely(Start > End)) return _error->Error("Internal parsing error"); - for (; isspace_ascii(End[-1]) != 0 && End > Start; --End); + for (; End > Start && isspace_ascii(End[-1]) != 0; --End); return true; } @@ -1062,14 +783,6 @@ bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::v } /*}}}*/ -void pkgUserTagSection::TrimRecord(bool /*BeforeRecord*/, const char* &End)/*{{{*/ -{ - for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++) - if (Stop[0] == '#') - Stop = (const char*) memchr(Stop,'\n',End-Stop); -} - /*}}}*/ - #include "tagfile-order.c" // TFRewrite - Rewrite a control record /*{{{*/ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index e02e7332e..04e20a5e8 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -138,11 +138,11 @@ class pkgTagSection * @return \b true if section end was found, \b false otherwise. * Beware that internal state will be inconsistent if \b false is returned! */ - APT_MUSTCHECK bool Scan(const char *Start, unsigned long MaxLength, bool const Restart = true); + APT_MUSTCHECK bool Scan(const char *Start, unsigned long MaxLength, bool const SupportComments = true); inline unsigned long size() const {return Stop - Section;}; void Trim(); - virtual void TrimRecord(bool BeforeRecord, const char* &End); + void TrimRecord(bool BeforeRecord, const char* &End, bool SupportComments); /** \brief amount of Tags in the current section * @@ -187,11 +187,6 @@ class pkgTagSection }; -class APT_DEPRECATED_MSG("Use pkgTagFile with the SUPPORT_COMMENTS flag instead") pkgUserTagSection : public pkgTagSection -{ - virtual void TrimRecord(bool BeforeRecord, const char* &End) APT_OVERRIDE; -}; - /** \class pkgTagFile reads and prepares a deb822 formatted file for parsing * via #pkgTagSection. The default mode tries to be as fast as possible and * assumes perfectly valid (machine generated) files like Packages. Support @@ -200,10 +195,6 @@ class pkgTagFile { pkgTagFilePrivate * const d; - APT_HIDDEN bool Fill(); - APT_HIDDEN bool Resize(); - APT_HIDDEN bool Resize(unsigned long long const newSize); - public: bool Step(pkgTagSection &Section); @@ -216,11 +207,11 @@ public: SUPPORT_COMMENTS = 1 << 0, }; - void Init(FileFd * const F, pkgTagFile::Flags const Flags, unsigned long long Size = 32*1024); - void Init(FileFd * const F,unsigned long long const Size = 32*1024); + void Init(FileFd * const F, pkgTagFile::Flags const Flags); + void Init(FileFd * const F); - pkgTagFile(FileFd * const F, pkgTagFile::Flags const Flags, unsigned long long Size = 32*1024); - pkgTagFile(FileFd * const F,unsigned long long Size = 32*1024); + pkgTagFile(FileFd * const F, pkgTagFile::Flags const Flags); + pkgTagFile(FileFd * const F); virtual ~pkgTagFile(); }; diff --git a/debian/libapt-pkg5.0.symbols b/debian/libapt-pkg5.0.symbols index 030ad35e4..2f625d2e1 100644 --- a/debian/libapt-pkg5.0.symbols +++ b/debian/libapt-pkg5.0.symbols @@ -1294,7 +1294,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER# (c++)"pkgTagSection::Tag::Rename(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)@APTPKG_5.0" 1.1~exp9 (c++)"pkgTagSection::Tag::Rewrite(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)@APTPKG_5.0" 1.1~exp9 (c++)"pkgTagSection::Write(FileFd&, char const* const*, std::vector<pkgTagSection::Tag, std::allocator<pkgTagSection::Tag> > const&) const@APTPKG_5.0" 1.1~exp9 - (c++)"pkgUserTagSection::TrimRecord(bool, char const*&)@APTPKG_5.0" 1.1~exp9 (c++)"pkgVersioningSystem::~pkgVersioningSystem()@APTPKG_5.0" 1.1~exp9 (c++)"SigVerify::~SigVerify()@APTPKG_5.0" 1.1~exp9 (c++)"SigVerify::SigVerify()@APTPKG_5.0" 1.1~exp9 @@ -1323,7 +1322,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER# (c++)"typeinfo for pkgProblemResolver@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo for pkgRecords@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo for pkgSourceList@APTPKG_5.0" 1.1~exp9 - (c++)"typeinfo for pkgUserTagSection@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo for SigVerify@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo for TranslationsCopy@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo name for APT::PackageUniverse@APTPKG_5.0" 1.1~exp9 @@ -1347,7 +1345,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER# (c++)"typeinfo name for pkgProblemResolver@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo name for pkgRecords@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo name for pkgSourceList@APTPKG_5.0" 1.1~exp9 - (c++)"typeinfo name for pkgUserTagSection@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo name for SigVerify@APTPKG_5.0" 1.1~exp9 (c++)"typeinfo name for TranslationsCopy@APTPKG_5.0" 1.1~exp9 (c++)"URI::ArchiveOnly(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)@APTPKG_5.0" 1.1~exp9 @@ -1372,7 +1369,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER# (c++)"vtable for pkgProblemResolver@APTPKG_5.0" 1.1~exp9 (c++)"vtable for pkgRecords@APTPKG_5.0" 1.1~exp9 (c++)"vtable for pkgSourceList@APTPKG_5.0" 1.1~exp9 - (c++)"vtable for pkgUserTagSection@APTPKG_5.0" 1.1~exp9 (c++)"vtable for SigVerify@APTPKG_5.0" 1.1~exp9 (c++)"vtable for TranslationsCopy@APTPKG_5.0" 1.1~exp9 ### dpkg selection state changer & general dpkg interfacing diff --git a/ftparchive/byhash.cc b/ftparchive/byhash.cc index 354d089c3..358fd27b4 100644 --- a/ftparchive/byhash.cc +++ b/ftparchive/byhash.cc @@ -17,6 +17,8 @@ #include <unistd.h> #include <sys/stat.h> +#define st_mtim st_mtimespec + #include <apt-pkg/fileutl.h> #include <apt-pkg/hashes.h> #include "byhash.h" diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index 868029abd..930bb5f94 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -22,6 +22,7 @@ #include <apt-pkg/debfile.h> #include <apt-pkg/gpgv.h> #include <apt-pkg/hashes.h> +#include <apt-pkg/missing.h> #include <netinet/in.h> // htonl, etc #include <ctype.h> diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index a74c2ce07..128c61005 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -4,8 +4,9 @@ add_executable(copy copy.cc) add_executable(store store.cc) add_executable(gpgv gpgv.cc) add_executable(cdrom cdrom.cc) -add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc basehttp.cc) -add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc basehttp.cc) +#add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc basehttp.cc) +add_executable(http http.cc rfc2553emu.cc connect.cc basehttp.cc) +#add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc basehttp.cc) add_executable(https https.cc basehttp.cc) add_executable(ftp ftp.cc rfc2553emu.cc connect.cc) add_executable(rred rred.cc) @@ -20,15 +21,16 @@ target_link_libraries(copy apt-pkg) target_link_libraries(store apt-pkg) target_link_libraries(gpgv apt-pkg) target_link_libraries(cdrom apt-pkg) -target_link_libraries(http apt-pkg) -target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES}) +#target_link_libraries(http apt-pkg) +#target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES}) target_link_libraries(https apt-pkg ${CURL_LIBRARIES}) +target_link_libraries(http apt-pkg lockdown "-framework CoreFoundation" "-framework CFNetwork" "-framework SystemConfiguration") target_link_libraries(ftp apt-pkg) target_link_libraries(rred apt-pkg) target_link_libraries(rsh apt-pkg) # Install the library -install(TARGETS file copy store gpgv cdrom http https ftp rred rsh mirror +install(TARGETS file copy store gpgv cdrom http https ftp rred rsh #mirror RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store gzip lzma bzip2 xz) diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 51c268d02..83135c220 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -67,7 +67,7 @@ struct Digest { static constexpr Digest Digests[] = { {Digest::State::Untrusted, "Invalid digest"}, {Digest::State::Untrusted, "MD5"}, - {Digest::State::Untrusted, "SHA1"}, + {Digest::State::Trusted, "SHA1"}, {Digest::State::Untrusted, "RIPE-MD/160"}, {Digest::State::Trusted, "Reserved digest"}, {Digest::State::Trusted, "Reserved digest"}, diff --git a/methods/http.cc b/methods/http.cc index 9f5959548..9549b23ac 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -25,7 +25,7 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ -#include <config.h> +// #include <config.h> #include <apt-pkg/fileutl.h> #include <apt-pkg/configuration.h> @@ -39,6 +39,7 @@ #include <stdlib.h> #include <sys/select.h> #include <cstring> +#include <sys/sysctl.h> #include <sys/stat.h> #include <sys/time.h> #include <unistd.h> @@ -52,956 +53,503 @@ #include "connect.h" #include "http.h" -#include <apti18n.h> +// #include <apti18n.h> + +#include <netdb.h> +#include <dlfcn.h> +#include <lockdown.h> +#include <CoreFoundation/CoreFoundation.h> +#include <CFNetwork/CFNetwork.h> +extern "C" CFDictionaryRef SCDynamicStoreCopyProxies(void *); /*}}}*/ using namespace std; -unsigned long long CircleBuf::BwReadLimit=0; -unsigned long long CircleBuf::BwTickReadData=0; -struct timeval CircleBuf::BwReadTick={0,0}; -const unsigned int CircleBuf::BW_HZ=10; - -// CircleBuf::CircleBuf - Circular input buffer /*{{{*/ -// --------------------------------------------------------------------- -/* */ -CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size) - : Size(Size), Hash(NULL), TotalWriten(0) -{ - Buf = new unsigned char[Size]; - Reset(); - - CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024; -} - /*}}}*/ -// CircleBuf::Reset - Reset to the default state /*{{{*/ -// --------------------------------------------------------------------- -/* */ -void CircleBuf::Reset() -{ - InP = 0; - OutP = 0; - StrPos = 0; - TotalWriten = 0; - MaxGet = (unsigned long long)-1; - OutQueue = string(); - if (Hash != NULL) - { - delete Hash; - Hash = NULL; - } -} - /*}}}*/ -// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/ -// --------------------------------------------------------------------- -/* This fills up the buffer with as much data as is in the FD, assuming it - is non-blocking.. */ -bool CircleBuf::Read(int Fd) -{ - while (1) - { - // Woops, buffer is full - if (InP - OutP == Size) - return true; - - // what's left to read in this tick - unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ; - - if(CircleBuf::BwReadLimit) { - struct timeval now; - gettimeofday(&now,0); - - unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 + - now.tv_usec-CircleBuf::BwReadTick.tv_usec; - if(d > 1000000/BW_HZ) { - CircleBuf::BwReadTick = now; - CircleBuf::BwTickReadData = 0; - } - - if(CircleBuf::BwTickReadData >= BwReadMax) { - usleep(1000000/BW_HZ); - return true; - } +#define _(str) str + +CFStringRef Firmware_; +const char *Machine_; +CFStringRef UniqueID_; + +void CfrsError(const char *name, CFReadStreamRef rs) { + CFStreamError se = CFReadStreamGetError(rs); + + if (se.domain == kCFStreamErrorDomainCustom) { + } else if (se.domain == kCFStreamErrorDomainPOSIX) { + _error->Error("POSIX: %s", strerror(se.error)); + } else if (se.domain == kCFStreamErrorDomainMacOSStatus) { + _error->Error("MacOSStatus: %d", (int)se.error); + } else if (se.domain == kCFStreamErrorDomainNetDB) { + _error->Error("NetDB: %s %s", name, gai_strerror(se.error)); + } else if (se.domain == kCFStreamErrorDomainMach) { + _error->Error("Mach: %d", (int)se.error); + } else if (se.domain == kCFStreamErrorDomainHTTP) { + switch (se.error) { + case kCFStreamErrorHTTPParseFailure: + _error->Error("Parse failure"); + break; + + case kCFStreamErrorHTTPRedirectionLoop: + _error->Error("Redirection loop"); + break; + + case kCFStreamErrorHTTPBadURL: + _error->Error("Bad URL"); + break; + + default: + _error->Error("Unknown HTTP error: %d", (int)se.error); + break; } - - // Write the buffer segment - ssize_t Res; - if(CircleBuf::BwReadLimit) { - Res = read(Fd,Buf + (InP%Size), - BwReadMax > LeftRead() ? LeftRead() : BwReadMax); - } else - Res = read(Fd,Buf + (InP%Size),LeftRead()); - - if(Res > 0 && BwReadLimit > 0) - CircleBuf::BwTickReadData += Res; - - if (Res == 0) - return false; - if (Res < 0) - { - if (errno == EAGAIN) - return true; - return false; - } - - if (InP == 0) - gettimeofday(&Start,0); - InP += Res; + } else if (se.domain == kCFStreamErrorDomainSOCKS) { + _error->Error("SOCKS: %d", (int)se.error); + } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) { + _error->Error("SystemConfiguration: %d", (int)se.error); + } else if (se.domain == kCFStreamErrorDomainSSL) { + _error->Error("SSL: %d", (int)se.error); + } else { + _error->Error("Domain #%d: %d", (int)se.domain, (int)se.error); } } - /*}}}*/ -// CircleBuf::Read - Put the string into the buffer /*{{{*/ -// --------------------------------------------------------------------- -/* This will hold the string in and fill the buffer with it as it empties */ -bool CircleBuf::Read(string const &Data) -{ - OutQueue.append(Data); - FillOut(); - return true; -} - /*}}}*/ -// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/ -// --------------------------------------------------------------------- -/* */ -void CircleBuf::FillOut() -{ - if (OutQueue.empty() == true) - return; - while (1) - { - // Woops, buffer is full - if (InP - OutP == Size) - return; - - // Write the buffer segment - unsigned long long Sz = LeftRead(); - if (OutQueue.length() - StrPos < Sz) - Sz = OutQueue.length() - StrPos; - memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz); - - // Advance - StrPos += Sz; - InP += Sz; - if (OutQueue.length() == StrPos) - { - StrPos = 0; - OutQueue = ""; - return; - } + +unsigned long TimeOut = 120; + +static const CFOptionFlags kNetworkEvents = + kCFStreamEventOpenCompleted | + kCFStreamEventHasBytesAvailable | + kCFStreamEventEndEncountered | + kCFStreamEventErrorOccurred | +0; + +static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) { + switch (event) { + case kCFStreamEventOpenCompleted: + break; + + case kCFStreamEventHasBytesAvailable: + case kCFStreamEventEndEncountered: + *reinterpret_cast<int *>(arg) = 1; + CFRunLoopStop(CFRunLoopGetCurrent()); + break; + + case kCFStreamEventErrorOccurred: + *reinterpret_cast<int *>(arg) = -1; + CFRunLoopStop(CFRunLoopGetCurrent()); + break; } } - /*}}}*/ -// CircleBuf::Write - Write from the buffer into a FD /*{{{*/ -// --------------------------------------------------------------------- -/* This empties the buffer into the FD. */ -bool CircleBuf::Write(int Fd) -{ - while (1) - { - FillOut(); - - // Woops, buffer is empty - if (OutP == InP) - return true; - - if (OutP == MaxGet) - return true; - - // Write the buffer segment - ssize_t Res; - Res = write(Fd,Buf + (OutP%Size),LeftWrite()); - if (Res == 0) - return false; - if (Res < 0) - { - if (errno == EAGAIN) - return true; - - return false; - } +/* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */ +int CFReadStreamOpen(CFReadStreamRef stream, double timeout) { + CFStreamClientContext context; + int value(0); - TotalWriten += Res; - - if (Hash != NULL) - Hash->Add(Buf + (OutP%Size),Res); - - OutP += Res; + memset(&context, 0, sizeof(context)); + context.info = &value; + + if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) { + CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes); + if (CFReadStreamOpen(stream)) + CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false); + else + value = -1; + CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL); } + + return value; } - /*}}}*/ -// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/ + +// HttpMethod::SendReq - Send the HTTP request /*{{{*/ // --------------------------------------------------------------------- -/* This copies till the first empty line */ -bool CircleBuf::WriteTillEl(string &Data,bool Single) +/* This places the http request in the outbound buffer */ +void HttpMethod::SendReq(FetchItem *Itm) { - // We cheat and assume it is unneeded to have more than one buffer load - for (unsigned long long I = OutP; I < InP; I++) - { - if (Buf[I%Size] != '\n') - continue; - ++I; - - if (Single == false) - { - if (I < InP && Buf[I%Size] == '\r') - ++I; - if (I >= InP || Buf[I%Size] != '\n') - continue; - ++I; - } - - Data = ""; - while (OutP < I) - { - unsigned long long Sz = LeftWrite(); - if (Sz == 0) - return false; - if (I - OutP < Sz) - Sz = I - OutP; - Data += string((char *)(Buf + (OutP%Size)),Sz); - OutP += Sz; - } - return true; - } - return false; } /*}}}*/ -// CircleBuf::Stats - Print out stats information /*{{{*/ -// --------------------------------------------------------------------- -/* */ -void CircleBuf::Stats() +std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/ { - if (InP == 0) - return; - - struct timeval Stop; - gettimeofday(&Stop,0); -/* float Diff = Stop.tv_sec - Start.tv_sec + - (float)(Stop.tv_usec - Start.tv_usec)/1000000; - clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/ + return NULL; } /*}}}*/ -CircleBuf::~CircleBuf() -{ - delete [] Buf; - delete Hash; -} - -// HttpServerState::HttpServerState - Constructor /*{{{*/ -HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024) +void HttpMethod::RotateDNS() /*{{{*/ { - TimeOut = Owner->ConfigFindI("Timeout", TimeOut); - Reset(); } /*}}}*/ -// HttpServerState::Open - Open a connection to the server /*{{{*/ -// --------------------------------------------------------------------- -/* This opens a connection to the server. */ -static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, - char const * const type, bool const ReadWrite, uint8_t * const ToFrom, - unsigned int const Size, unsigned int const Timeout) -{ - if (WaitFd(ServerFd, ReadWrite, Timeout) == false) - return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type); - if (ReadWrite == false) - { - if (FileFd::Read(ServerFd, ToFrom, Size) == false) - return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); - } - else - { - if (FileFd::Write(ServerFd, ToFrom, Size) == false) - return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); - } - return true; -} -bool HttpServerState::Open() +BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/ { - // Use the already open connection if possible. - if (ServerFd != -1) - return true; - - Close(); - In.Reset(); - Out.Reset(); - Persistent = true; - - // Determine the proxy setting - AutoDetectProxy(ServerName); - string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, ""); - if (!SpecificProxy.empty()) - { - if (SpecificProxy == "DIRECT") - Proxy = ""; - else - Proxy = SpecificProxy; - } - else - { - string DefProxy = Owner->ConfigFind("Proxy", ""); - if (!DefProxy.empty()) - { - Proxy = DefProxy; - } - else - { - char* result = getenv("http_proxy"); - Proxy = result ? result : ""; - } - } - - // Parse no_proxy, a , separated list of domains - if (getenv("no_proxy") != 0) - { - if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) - Proxy = ""; - } + auto ret = BaseHttpMethod::DealWithHeaders(Res, Req); + if (ret != BaseHttpMethod::FILE_IS_OPEN) + return ret; + if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false) + return ERROR_NOT_FROM_SERVER; - if (Proxy.empty() == false) - Owner->AddProxyAuth(Proxy, ServerName); + FailFile = Queue->DestFile; + FailFile.c_str(); // Make sure we don't do a malloc in the signal handler + FailFd = Req.File.Fd(); + FailTime = Req.Date; - if (Proxy.Access == "socks5h") - { - if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false) - return false; - - /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect - * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). - * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ - std::string const ProxyInfo = URI::SiteOnly(Proxy); - Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str()); - auto const Timeout = Owner->ConfigFindI("TimeOut", 120); - #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false - #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false - if (ServerName.Host.length() > 255) - return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str()); - if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) - return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); - if (Proxy.User.empty()) - { - uint8_t greeting[] = { 0x05, 0x01, 0x00 }; - APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); - } - else - { - uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 }; - APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); - } - uint8_t greeting[2]; - APT_ReadOrFail("greet back", greeting, sizeof(greeting)); - if (greeting[0] != 0x05) - return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); - if (greeting[1] == 0x00) - ; // no auth has no method-dependent sub-negotiations - else if (greeting[1] == 0x02) - { - if (Proxy.User.empty()) - return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); - // user&pass auth sub-negotiations are defined by RFC1929 - std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }}; - std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); - auth.push_back(static_cast<uint8_t>(Proxy.Password.length())); - std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); - APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); - uint8_t authstatus[2]; - APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); - if (authstatus[0] != 0x01) - return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); - if (authstatus[1] != 0x00) - return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); - } - else - return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); - union { uint16_t * i; uint8_t * b; } portu; - uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port)); - portu.i = &port; - std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }}; - std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request)); - request.push_back(portu.b[0]); - request.push_back(portu.b[1]); - APT_WriteOrFail("request", request.data(), request.size()); - uint8_t response[4]; - APT_ReadOrFail("first part of response", response, sizeof(response)); - if (response[0] != 0x05) - return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); - if (response[2] != 0x00) - return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); - std::string bindaddr; - if (response[3] == 0x01) // IPv4 address - { - uint8_t ip4port[6]; - APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); - portu.b[0] = ip4port[4]; - portu.b[1] = ip4port[5]; - port = ntohs(*portu.i); - strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); - } - else if (response[3] == 0x03) // hostname - { - uint8_t namelength; - APT_ReadOrFail("hostname length of response", &namelength, 1); - uint8_t hostname[namelength + 2]; - APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); - portu.b[0] = hostname[namelength]; - portu.b[1] = hostname[namelength + 1]; - port = ntohs(*portu.i); - hostname[namelength] = '\0'; - strprintf(bindaddr, "%s:%d", hostname, port); - } - else if (response[3] == 0x04) // IPv6 address - { - uint8_t ip6port[18]; - APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); - portu.b[0] = ip6port[16]; - portu.b[1] = ip6port[17]; - port = ntohs(*portu.i); - strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", - ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], - ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], - port); - } - else - return _error->Error("SOCKS proxy %s destination address is of unknown type: %d", - ProxyInfo.c_str(), response[3]); - if (response[1] != 0x00) - { - char const * errstr = nullptr; - auto errcode = response[1]; - // Tor error reporting can be a bit arcane, lets try to detect & fix it up - if (bindaddr == "0.0.0.0:0") - { - auto const lastdot = ServerName.Host.rfind('.'); - if (lastdot == std::string::npos || ServerName.Host.substr(lastdot) != ".onion") - ; - else if (errcode == 0x01) - { - auto const prevdot = ServerName.Host.rfind('.', lastdot - 1); - if (lastdot == 16 && prevdot == std::string::npos) - ; // valid .onion address - else if (prevdot != std::string::npos && (lastdot - prevdot) == 17) - ; // valid .onion address with subdomain(s) - else - { - errstr = "Invalid hostname: onion service name must be 16 characters long"; - Owner->SetFailReason("SOCKS"); - } - } - // in all likelihood the service is either down or the address has - // a typo and so "Host unreachable" is the better understood error - // compared to the technically correct "TLL expired". - else if (errcode == 0x06) - errcode = 0x04; - } - if (errstr == nullptr) - { - switch (errcode) - { - case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break; - case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break; - case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; - case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; - case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break; - case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break; - case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break; - case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break; - default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break; - } - } - return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str(), errstr, response[1]); - } - else if (Owner->DebugEnabled()) - ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - - if (WaitFd(ServerFd, true, Timeout) == false) - return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - #undef APT_ReadOrFail - #undef APT_WriteOrFail - } - else + if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false) { - // Determine what host and port to use based on the proxy settings - int Port = 0; - string Host; - if (Proxy.empty() == true || Proxy.Host.empty() == true) - { - if (ServerName.Port != 0) - Port = ServerName.Port; - Host = ServerName.Host; - } - else if (Proxy.Access != "http") - return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str()); - else - { - if (Proxy.Port != 0) - Port = Proxy.Port; - Host = Proxy.Host; - } - return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner); + _error->Errno("read",_("Problem hashing file")); + return ERROR_NOT_FROM_SERVER; } - return true; -} - /*}}}*/ -// HttpServerState::Close - Close a connection to the server /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool HttpServerState::Close() -{ - close(ServerFd); - ServerFd = -1; - return true; + if (Req.StartPos > 0) + Res.ResumePoint = Req.StartPos; + + SetNonBlock(Req.File.Fd(),true); + return FILE_IS_OPEN; } - /*}}}*/ -// HttpServerState::RunData - Transfer the data from the socket /*{{{*/ -bool HttpServerState::RunData(RequestState &Req) + +// HttpMethod::Loop - Main loop /*{{{*/ +int HttpMethod::Loop() { - Req.State = RequestState::Data; + signal(SIGTERM,SigTerm); + signal(SIGINT,SigTerm); - // Chunked transfer encoding is fun.. - if (Req.Encoding == RequestState::Chunked) - { - while (1) + Server = 0; + + std::set<std::string> cached; + + int FailCounter = 0; + while (1) + { + // We have no commands, wait for some to arrive + if (Queue == 0) { - // Grab the block size - bool Last = true; - string Data; - In.Limit(-1); - do - { - if (In.WriteTillEl(Data,true) == true) - break; - } - while ((Last = Go(false, Req)) == true); - - if (Last == false) - return false; - - // See if we are done - unsigned long long Len = strtoull(Data.c_str(),0,16); - if (Len == 0) - { - In.Limit(-1); - - // We have to remove the entity trailer - Last = true; - do - { - if (In.WriteTillEl(Data,true) == true && Data.length() <= 2) - break; - } - while ((Last = Go(false, Req)) == true); - if (Last == false) - return false; - return !_error->PendingError(); - } - - // Transfer the block - In.Limit(Len); - while (Go(true, Req) == true) - if (In.IsLimit() == true) - break; - - // Error - if (In.IsLimit() == false) - return false; - - // The server sends an extra new line before the next block specifier.. - In.Limit(-1); - Last = true; - do - { - if (In.WriteTillEl(Data,true) == true) - break; - } - while ((Last = Go(false, Req)) == true); - if (Last == false) - return false; + if (WaitFd(STDIN_FILENO) == false) + return 0; } - } - else - { - /* Closes encoding is used when the server did not specify a size, the - loss of the connection means we are done */ - if (Req.JunkSize != 0) - In.Limit(Req.JunkSize); - else if (Req.DownloadSize != 0) - In.Limit(Req.DownloadSize); - else if (Persistent == false) - In.Limit(-1); - // Just transfer the whole block. - do + /* Run messages, we can accept 0 (no message) if we didn't + do a WaitFd above.. Otherwise the FD is closed. */ + int Result = Run(true); + if (Result != -1 && (Result != 0 || Queue == 0)) { - if (In.IsLimit() == false) - continue; - - In.Limit(-1); - return !_error->PendingError(); + if(FailReason.empty() == false || + ConfigFindB("DependOnSTDIN", true) == true) + return 100; + else + return 0; } - while (Go(true, Req) == true); - } - return Flush(&Req.File) && !_error->PendingError(); -} - /*}}}*/ -bool HttpServerState::RunDataToDevNull(RequestState &Req) /*{{{*/ -{ - // no need to clean up if we discard the connection anyhow - if (Persistent == false) - return true; - Req.File.Open("/dev/null", FileFd::WriteOnly); - return RunData(Req); -} - /*}}}*/ -bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/ -{ - return In.WriteTillEl(Data); -} - /*}}}*/ -bool HttpServerState::LoadNextResponse(bool const ToFile, RequestState &Req)/*{{{*/ -{ - return Go(ToFile, Req); -} - /*}}}*/ -bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/ -{ - return Out.Read(Data); -} - /*}}}*/ -APT_PURE bool HttpServerState::IsOpen() /*{{{*/ -{ - return (ServerFd != -1); -} - /*}}}*/ -bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ -{ - delete In.Hash; - In.Hash = new Hashes(ExpectedHashes); - return true; -} - /*}}}*/ -void HttpServerState::Reset() /*{{{*/ -{ - ServerState::Reset(); - ServerFd = -1; -} - /*}}}*/ + if (Queue == 0) + continue; -APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/ -{ - return In.Hash; -} - /*}}}*/ -// HttpServerState::Die - The server has closed the connection. /*{{{*/ -bool HttpServerState::Die(RequestState &Req) -{ - unsigned int LErrno = errno; + CFStringEncoding se = kCFStringEncodingUTF8; - // Dump the buffer to the file - if (Req.State == RequestState::Data) - { - if (Req.File.IsOpen() == false) - return true; - // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking - // can't be set - if (Req.File.Name() != "/dev/null") - SetNonBlock(Req.File.Fd(),false); - while (In.WriteSpace() == true) - { - if (In.Write(Req.File.Fd()) == false) - return _error->Errno("write",_("Error writing to the file")); + URI uri2 = Queue->Uri; + string uriString = static_cast<string>(uri2); + + char *url = strdup(uriString.c_str()); + url: + URI uri = std::string(url); + std::string hs = uri.Host; + + if (cached.find(hs) != cached.end()) { + _error->Error("Cached Failure"); + Fail(true); + free(url); + FailCounter = 0; + continue; + } + + std::string urs = uri; - // Done - if (In.IsLimit() == true) - return true; + for (;;) { + size_t bad = urs.find_first_of("+"); + if (bad == std::string::npos) + break; + // XXX: generalize + urs = urs.substr(0, bad) + "%2b" + urs.substr(bad + 1); } - } - // See if this is because the server finished the data stream - if (In.IsLimit() == false && Req.State != RequestState::Header && - Persistent == true) - { - Close(); - if (LErrno == 0) - return _error->Error(_("Error reading from server. Remote end closed connection")); - errno = LErrno; - return _error->Errno("read",_("Error reading from server")); - } - else - { - In.Limit(-1); + CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se); + CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL); + CFRelease(sr); + CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1); + CFRelease(ur); + + struct stat SBuf; + if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) { + sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1); + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr); + CFRelease(sr); + + sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime, false).c_str(), se); + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr); + CFRelease(sr); + + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache")); + } else if (Queue->LastModified != 0) { + sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified, true).c_str(), se); + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr); + CFRelease(sr); + + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache")); + } else + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("max-age=0")); - // Nothing left in the buffer - if (In.WriteSpace() == false) - return false; + if (Firmware_ != NULL) + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_); - // We may have got multiple responses back in one packet.. - Close(); - return true; - } + sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se); + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr); + CFRelease(sr); - return false; -} - /*}}}*/ -// HttpServerState::Flush - Dump the buffer into the file /*{{{*/ -// --------------------------------------------------------------------- -/* This takes the current input buffer from the Server FD and writes it - into the file */ -bool HttpServerState::Flush(FileFd * const File) -{ - if (File != nullptr) - { - // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking - // can't be set - if (File->Name() != "/dev/null") - SetNonBlock(File->Fd(),false); - if (In.WriteSpace() == false) - return true; - - while (In.WriteSpace() == true) - { - if (In.Write(File->Fd()) == false) - return _error->Errno("write",_("Error writing to file")); - if (In.IsLimit() == true) - return true; + if (UniqueID_ != NULL) + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_); + + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.592")); + + CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm); + CFRelease(hm); + +#define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout") +#define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout") +#define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut") + + /*SInt32 to(TimeOut); + CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));os_log(OS_LOG_DEFAULT, "[%{public}s:%{public}d]",__BASE_FILE__,__LINE__);*/ + double to = TimeOut; + CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to)); + + CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm); + CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm); + CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm); + CFRelease(nm); + + CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL); + CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr); + CFRelease(dr); + + //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue); + CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue); + + FetchResult Res; + CFIndex rd; + UInt32 sc; + + uint8_t data[10240]; + size_t offset = 0; + + Status("Connecting to %s", hs.c_str()); + + switch (CFReadStreamOpen(rs, to)) { + case -1: + CfrsError("Open", rs); + goto fail; + + case 0: + _error->Error("Host Unreachable"); + cached.insert(hs); + goto fail; + + case 1: + /* success */ + break; + + fail: + Fail(true); + goto done; } - if (In.IsLimit() == true || Persistent == false) - return true; - } - return false; -} - /*}}}*/ -// HttpServerState::Go - Run a single loop /*{{{*/ -// --------------------------------------------------------------------- -/* This runs the select loop over the server FDs, Output file FDs and - stdin. */ -bool HttpServerState::Go(bool ToFile, RequestState &Req) -{ - // Server has closed the connection - if (ServerFd == -1 && (In.WriteSpace() == false || - ToFile == false)) - return false; - - fd_set rfds,wfds; - FD_ZERO(&rfds); - FD_ZERO(&wfds); - - /* Add the server. We only send more requests if the connection will - be persisting */ - if (Out.WriteSpace() == true && ServerFd != -1 - && Persistent == true) - FD_SET(ServerFd,&wfds); - if (In.ReadSpace() == true && ServerFd != -1) - FD_SET(ServerFd,&rfds); - - // Add the file - int FileFD = -1; - if (Req.File.IsOpen()) - FileFD = Req.File.Fd(); - - if (In.WriteSpace() == true && ToFile == true && FileFD != -1) - FD_SET(FileFD,&wfds); - - // Add stdin - if (Owner->ConfigFindB("DependOnSTDIN", true) == true) - FD_SET(STDIN_FILENO,&rfds); - - // Figure out the max fd - int MaxFd = FileFD; - if (MaxFd < ServerFd) - MaxFd = ServerFd; - - // Select - struct timeval tv; - tv.tv_sec = TimeOut; - tv.tv_usec = 0; - int Res = 0; - if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0) - { - if (errno == EINTR) - return true; - return _error->Errno("select",_("Select failed")); - } - - if (Res == 0) - { - _error->Error(_("Connection timed out")); - return Die(Req); - } - - // Handle server IO - if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds)) - { - errno = 0; - if (In.Read(ServerFd) == false) - return Die(Req); - } - - if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds)) - { - errno = 0; - if (Out.Write(ServerFd) == false) - return Die(Req); - } + rd = CFReadStreamRead(rs, data, sizeof(data)); - // Send data to the file - if (FileFD != -1 && FD_ISSET(FileFD,&wfds)) - { - if (In.Write(FileFD) == false) - return _error->Errno("write",_("Error writing to output file")); - } + if (rd == -1) { + CfrsError(uri.Host.c_str(), rs); + cached.insert(hs); + Fail(true); + goto done; + } - if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize) - { - Owner->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", - Req.File.Tell(), Req.MaximumSize); - } + Res.Filename = Queue->DestFile; + + hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader); + sc = CFHTTPMessageGetResponseStatusCode(hm); + + if (sc == 301 || sc == 302) { + sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location")); + if (sr == NULL) { + Fail(); + goto done_; + } else { + size_t ln = CFStringGetLength(sr) + 1; + free(url); + url = static_cast<char *>(malloc(ln)); + + if (!CFStringGetCString(sr, url, ln, se)) { + Fail(); + goto done_; + } + + CFRelease(sr); + goto url; + } + } - // Handle commands from APT - if (FD_ISSET(STDIN_FILENO,&rfds)) - { - if (Owner->Run(true) != -1) - exit(100); - } - - return true; -} - /*}}}*/ + sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range")); + if (sr != NULL) { + size_t ln = CFStringGetLength(sr) + 1; + char cr[ln]; + + if (!CFStringGetCString(sr, cr, ln, se)) { + Fail(); + goto done_; + } + + CFRelease(sr); + + if (sscanf(cr, "bytes %lu-%*u/%llu", &offset, &Res.Size) != 2) { + _error->Error(_("The HTTP server sent an invalid Content-Range header")); + Fail(); + goto done_; + } + + if (offset > Res.Size) { + _error->Error(_("This HTTP server has broken range support")); + Fail(); + goto done_; + } + } else { + sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length")); + if (sr != NULL) { + Res.Size = CFStringGetIntValue(sr); + CFRelease(sr); + } + } -// HttpMethod::SendReq - Send the HTTP request /*{{{*/ -// --------------------------------------------------------------------- -/* This places the http request in the outbound buffer */ -void HttpMethod::SendReq(FetchItem *Itm) -{ - URI Uri = Itm->Uri; - { - auto const plus = Binary.find('+'); - if (plus != std::string::npos) - Uri.Access = Binary.substr(plus + 1); - } + time(&Res.LastModified); - // The HTTP server expects a hostname with a trailing :port - std::stringstream Req; - string ProperHost; - - if (Uri.Host.find(':') != string::npos) - ProperHost = '[' + Uri.Host + ']'; - else - ProperHost = Uri.Host; - - /* RFC 2616 ยง5.1.2 requires absolute URIs for requests to proxies, - but while its a must for all servers to accept absolute URIs, - it is assumed clients will sent an absolute path for non-proxies */ - std::string requesturi; - if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty()) - requesturi = Uri.Path; - else - requesturi = Uri; - - // The "+" is encoded as a workaround for a amazon S3 bug - // see LP bugs #1003633 and #1086997. - requesturi = QuoteString(requesturi, "+~ "); - - /* Build the request. No keep-alive is included as it is the default - in 1.1, can cause problems with proxies, and we are an HTTP/1.1 - client anyway. - C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */ - Req << "GET " << requesturi << " HTTP/1.1\r\n"; - if (Uri.Port != 0) - Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n"; - else - Req << "Host: " << ProperHost << "\r\n"; - - // generate a cache control header (if needed) - if (ConfigFindB("No-Cache",false) == true) - Req << "Cache-Control: no-cache\r\n" - << "Pragma: no-cache\r\n"; - else if (Itm->IndexFile == true) - Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n"; - else if (ConfigFindB("No-Store", false) == true) - Req << "Cache-Control: no-store\r\n"; - - // If we ask for uncompressed files servers might respond with content- - // negotiation which lets us end up with compressed files we do not support, - // see 657029, 657560 and co, so if we have no extension on the request - // ask for text only. As a sidenote: If there is nothing to negotate servers - // seem to be nice and ignore it. - if (ConfigFindB("SendAccept", true) == true) - { - size_t const filepos = Itm->Uri.find_last_of('/'); - string const file = Itm->Uri.substr(filepos + 1); - if (flExtension(file) == file) - Req << "Accept: text/*\r\n"; - } + sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified")); + if (sr != NULL) { + size_t ln = CFStringGetLength(sr) + 1; + char cr[ln]; - // Check for a partial file and send if-queries accordingly - struct stat SBuf; - if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) - Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n" - << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n"; - else if (Itm->LastModified != 0) - Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n"; + if (!CFStringGetCString(sr, cr, ln, se)) { + Fail(); + goto done_; + } - if (Server->Proxy.Access == "http" && - (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)) - Req << "Proxy-Authorization: Basic " - << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n"; + CFRelease(sr); - maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc")); - if (Uri.User.empty() == false || Uri.Password.empty() == false) - Req << "Authorization: Basic " - << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n"; + if (!RFC1123StrToTime(cr, Res.LastModified)) { + _error->Error(_("Unknown date format")); + Fail(); + goto done_; + } + } - Req << "User-Agent: " << ConfigFind("User-Agent", - "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n"; + if (sc < 200 || (sc >= 300 && sc != 304)) { + sr = CFHTTPMessageCopyResponseStatusLine(hm); - Req << "\r\n"; + size_t ln = CFStringGetLength(sr) + 1; + char cr[ln]; - if (Debug == true) - cerr << Req.str() << endl; + if (!CFStringGetCString(sr, cr, ln, se)) { + Fail(); + goto done; + } - Server->WriteResponse(Req.str()); -} - /*}}}*/ -std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/ -{ - return std::unique_ptr<ServerState>(new HttpServerState(uri, this)); -} - /*}}}*/ -void HttpMethod::RotateDNS() /*{{{*/ -{ - ::RotateDNS(); -} - /*}}}*/ -BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/ -{ - auto ret = BaseHttpMethod::DealWithHeaders(Res, Req); - if (ret != BaseHttpMethod::FILE_IS_OPEN) - return ret; - if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false) - return ERROR_NOT_FROM_SERVER; + CFRelease(sr); - FailFile = Queue->DestFile; - FailFile.c_str(); // Make sure we don't do a malloc in the signal handler - FailFd = Req.File.Fd(); - FailTime = Req.Date; + _error->Error("%s", cr); - if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false) - { - _error->Errno("read",_("Problem hashing file")); - return ERROR_NOT_FROM_SERVER; - } - if (Req.StartPos > 0) - Res.ResumePoint = Req.StartPos; + Fail(); + goto done_; + } - SetNonBlock(Req.File.Fd(),true); - return FILE_IS_OPEN; + CFRelease(hm); + + if (sc == 304) { + unlink(Queue->DestFile.c_str()); + Res.IMSHit = true; + Res.LastModified = Queue->LastModified; + URIDone(Res); + } else { + Hashes hash; + + File = new FileFd(Queue->DestFile, FileFd::WriteAny); + if (_error->PendingError() == true) { + delete File; + File = NULL; + Fail(); + goto done; + } + + FailFile = Queue->DestFile; + FailFile.c_str(); // Make sure we dont do a malloc in the signal handler + FailFd = File->Fd(); + FailTime = Res.LastModified; + + Res.ResumePoint = offset; + ftruncate(File->Fd(), offset); + + if (offset != 0) { + lseek(File->Fd(), 0, SEEK_SET); + if (!hash.AddFD(File->Fd(), offset)) { + _error->Errno("read", _("Problem hashing file")); + delete File; + File = NULL; + Fail(); + goto done; + } + } + + lseek(File->Fd(), 0, SEEK_END); + + URIStart(Res); + + read: if (rd == -1) { + CfrsError("rd", rs); + Fail(true); + } else if (rd == 0) { + if (Res.Size == 0) + Res.Size = File->Size(); + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = Res.LastModified; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(Queue->DestFile.c_str(), times); + + Res.TakeHashes(hash); + URIDone(Res); + } else { + hash.Add(data, rd); + + uint8_t *dt = data; + while (rd != 0) { + int sz = write(File->Fd(), dt, rd); + + if (sz == -1) { + delete File; + File = NULL; + Fail(); + goto done; + } + + dt += sz; + rd -= sz; + } + + rd = CFReadStreamRead(rs, data, sizeof(data)); + goto read; + } + } + + goto done; + + done_: + CFRelease(hm); + done: + CFReadStreamClose(rs); + CFRelease(rs); + free(url); + + FailCounter = 0; + } + + return 0; } - /*}}}*/ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ { auto addName = std::inserter(methodNames, methodNames.begin()); @@ -1010,5 +558,53 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2 auto const plus = Binary.find('+'); if (plus != std::string::npos) addName = Binary.substr(0, plus); + File = 0; + Server = 0; } /*}}}*/ + +int main(int, const char *argv[]) +{ + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + + size_t size; + sysctlbyname("hw.machine", NULL, &size, NULL, 0); + char *machine = new char[size]; + sysctlbyname("hw.machine", machine, &size, NULL, 0); + Machine_ = machine; + + const char *path = "/System/Library/CoreServices/SystemVersion.plist"; + CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false); + + CFPropertyListRef plist; { + CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url); + CFReadStreamOpen(stream); + plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL); + CFReadStreamClose(stream); + } + + CFRelease(url); + + if (plist != NULL) { + Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion"))); + CFRelease(plist); + } + + if (UniqueID_ == NULL) + if (void *libMobileGestalt = dlopen("/usr/lib/libMobileGestalt.dylib", RTLD_GLOBAL | RTLD_LAZY)) + if (CFStringRef (*$MGCopyAnswer)(CFStringRef) = (CFStringRef (*)(CFStringRef)) dlsym(libMobileGestalt, "MGCopyAnswer")) + UniqueID_ = $MGCopyAnswer(CFSTR("UniqueDeviceID")); + + if (UniqueID_ == NULL) + if (void *lockdown = lockdown_connect()) { + UniqueID_ = (CFStringRef)lockdown_copy_value(lockdown, NULL, kLockdownUniqueDeviceIDKey); + lockdown_disconnect(lockdown); + } + + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "http") + Binary.append("+http"); + return HttpMethod(std::move(Binary)).Loop(); +} diff --git a/methods/http.h b/methods/http.h index c79a6454e..3b491dfda 100644 --- a/methods/http.h +++ b/methods/http.h @@ -26,101 +26,6 @@ class FileFd; class HttpMethod; class Hashes; -class CircleBuf -{ - unsigned char *Buf; - unsigned long long Size; - unsigned long long InP; - unsigned long long OutP; - std::string OutQueue; - unsigned long long StrPos; - unsigned long long MaxGet; - struct timeval Start; - - static unsigned long long BwReadLimit; - static unsigned long long BwTickReadData; - static struct timeval BwReadTick; - static const unsigned int BW_HZ; - - unsigned long long LeftRead() const - { - unsigned long long Sz = Size - (InP - OutP); - if (Sz > Size - (InP%Size)) - Sz = Size - (InP%Size); - return Sz; - } - unsigned long long LeftWrite() const - { - unsigned long long Sz = InP - OutP; - if (InP > MaxGet) - Sz = MaxGet - OutP; - if (Sz > Size - (OutP%Size)) - Sz = Size - (OutP%Size); - return Sz; - } - void FillOut(); - - public: - Hashes *Hash; - // total amount of data that got written so far - unsigned long long TotalWriten; - - // Read data in - bool Read(int Fd); - bool Read(std::string const &Data); - - // Write data out - bool Write(int Fd); - bool WriteTillEl(std::string &Data,bool Single = false); - - // Control the write limit - void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;} - bool IsLimit() const {return MaxGet == OutP;}; - void Print() const {cout << MaxGet << ',' << OutP << endl;}; - - // Test for free space in the buffer - bool ReadSpace() const {return Size - (InP - OutP) > 0;}; - bool WriteSpace() const {return InP - OutP > 0;}; - - void Reset(); - // Dump everything - void Stats(); - - CircleBuf(HttpMethod const * const Owner, unsigned long long Size); - ~CircleBuf(); -}; - -struct HttpServerState: public ServerState -{ - // This is the connection itself. Output is data FROM the server - CircleBuf In; - CircleBuf Out; - int ServerFd; - - protected: - virtual bool ReadHeaderLines(std::string &Data) APT_OVERRIDE; - virtual bool LoadNextResponse(bool const ToFile, RequestState &Req) APT_OVERRIDE; - virtual bool WriteResponse(std::string const &Data) APT_OVERRIDE; - - public: - virtual void Reset() APT_OVERRIDE; - - virtual bool RunData(RequestState &Req) APT_OVERRIDE; - virtual bool RunDataToDevNull(RequestState &Req) APT_OVERRIDE; - - virtual bool Open() APT_OVERRIDE; - virtual bool IsOpen() APT_OVERRIDE; - virtual bool Close() APT_OVERRIDE; - virtual bool InitHashes(HashStringList const &ExpectedHashes) APT_OVERRIDE; - virtual Hashes * GetHashes() APT_OVERRIDE; - virtual bool Die(RequestState &Req) APT_OVERRIDE; - virtual bool Flush(FileFd * const File) APT_OVERRIDE; - virtual bool Go(bool ToFile, RequestState &Req) APT_OVERRIDE; - - HttpServerState(URI Srv, HttpMethod *Owner); - virtual ~HttpServerState() {Close();}; -}; - class HttpMethod : public BaseHttpMethod { public: @@ -132,10 +37,13 @@ class HttpMethod : public BaseHttpMethod protected: std::string AutoDetectProxyCmd; + FileFd *File; public: friend struct HttpServerState; + int Loop(); + explicit HttpMethod(std::string &&pProg); }; |